@empiricalrun/test-gen 0.43.3 → 0.45.0

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @empiricalrun/test-gen
 
+## 0.45.0
+
+### Minor Changes
+
+- 698ad31: feat: add approval in skill execute
+
+### Patch Changes
+
+- 23f875f: fix: abstract the communication method of human in loop
+
+## 0.44.0
+
+### Minor Changes
+
+- c206ea8: feat: Add suites CLI flag for test generation
+
+### Patch Changes
+
+- 57ea65d: fix: prompt for create test using code agent to honour the file name and test case name
+
 ## 0.43.3
 
 ### Patch Changes

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAqD9B,wBAAwB,IAAI,YAAY,EAAE;IAoB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,WAAW,EACjB,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IA0D9B,wBAAwB,IAAI,YAAY,EAAE;IAoB1C,sBAAsB,IAAI,YAAY,EAAE;IAWxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;IAWV,iBAAiB;IAIjB,iBAAiB,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAItD"}

package/dist/actions/index.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PlaywrightActions = void 0;
 const logger_1 = require("../bin/logger");
+const errors_1 = require("../errors");
 const reporter_1 = require("../reporter");
 const assert_1 = require("./assert");
 const click_1 = require("./click");
@@ -81,7 +82,13 @@ class PlaywrightActions {
                 },
             });
             logger.log(`action: ${name} \nreason: ${args.reason}`);
-            throw Error(`Error executing ${name} action: ${e.message}`);
+            // if its already parsed error throw it as is
+            if (e instanceof errors_1.TestGenError) {
+                throw e;
+            }
+            else {
+                throw new errors_1.TestGenError(`Error executing ${name} action: ${e.message}`);
+            }
         }
     }
     getBrowsingActionSchemas() {

package/dist/actions/skill.d.ts

@@ -13,6 +13,9 @@ declare class TestCaseSkills {
     updateSkills(skills: Skill[]): void;
 }
 export declare const testCaseSkills: TestCaseSkills;
+export declare function extractSkillFromArgs(args: Record<string, any>): {
+    skillDetails: Skill | undefined;
+};
 export declare const skillActionGenerator: PlaywrightActionGenerator;
 export {};
 //# sourceMappingURL=skill.d.ts.map

package/dist/actions/skill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,MAAM,MAAM,KAAK,GAAG;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,eAAO,MAAM,oBAAoB,EAAE,yBAoHlC,CAAC"}
+{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,MAAM,MAAM,KAAK,GAAG;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;;EAO7D;AAED,eAAO,MAAM,oBAAoB,EAAE,yBAmHlC,CAAC"}

package/dist/actions/skill.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.skillActionGenerator = exports.testCaseSkills = exports.SKILL_USAGE = void 0;
+exports.skillActionGenerator = exports.extractSkillFromArgs = exports.testCaseSkills = exports.SKILL_USAGE = void 0;
 const promises_1 = __importDefault(require("fs/promises"));
 const api_1 = __importDefault(require("tsx/cjs/api"));
 const use_skill_1 = require("../agent/codegen/use-skill");
@@ -23,12 +23,19 @@ class TestCaseSkills {
     }
 }
 exports.testCaseSkills = new TestCaseSkills([]);
+function extractSkillFromArgs(args) {
+    const availableSkills = exports.testCaseSkills.getAvailableSkills();
+    const { skill } = args;
+    const [skillDetails] = availableSkills.filter((skillDetails) => skillDetails.testStep === skill);
+    return { skillDetails };
+}
+exports.extractSkillFromArgs = extractSkillFromArgs;
 const skillActionGenerator = (page, options) => {
     const availableSkills = exports.testCaseSkills.getAvailableSkills();
     return {
         execute: async ({ args, trace }) => {
             const { skill, action } = args;
-            const [skillDetails] = availableSkills.filter((skillDetails) => skillDetails.testStep === skill);
+            const { skillDetails } = extractSkillFromArgs(args);
             if (!skillDetails) {
                 throw new Error(`No skill found for skill: ${skill}`);
             }
@@ -42,6 +49,7 @@ const skillActionGenerator = (page, options) => {
             const code = await (0, use_skill_1.generateSkillUsageCode)({
                 task: action,
                 sampleUsageMethod: skillDetails.usageExample,
+                importPathForSkill: skillFilePath,
                 scopeVariablesMapStr: JSON.stringify(options.stateVariables || {}, null, 2),
                 pageVariableName: (0, utils_1.getPageVarName)(),
                 skillMethodDefinition: skillFile,

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAmBhD,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;;;;GAuDA;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CA8D5B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAmBhD,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;;;;GAyDA;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CA8D5B"}

package/dist/agent/codegen/run.js

@@ -10,7 +10,7 @@ const logger_1 = require("../../bin/logger");
 const context_1 = require("../../bin/utils/context");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
-const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write test cases in a Playwright test repository.\nYou will be provided with current test files, fixtures and page object models for you to use and update code as per the\ntask provided to you.\nYou need to respond with file path, the new test code block to be updated in the file and the reason for the change.\n\nHere is the list of files:\n\n{{repoFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nTask: {{task}}\n\n## Follow these instructions before responding with output:\n- Respond with <file_path></file_path>, <code_block></code_block> and <change></change> as xml tags\n- Only respond with single code_block in the response containing only the test code block to be updated\n- If the test already exists, do not update the file\n{{/section}}";
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write test cases in a Playwright test repository.\nYou will be provided with current test files, fixtures and page object models for you to use and update code as per the\ntask provided to you.\nYou need to respond with file path, the new test code block to be updated in the file and the reason for the change.\n\nHere is the list of files:\n\n{{repoFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nTask: {{task}}\n\n## Follow these instructions before responding with output:\n- Scan the file '{{testFilePath}}' and check whether test case with name '{{testCaseName}}' exists in the file\n- If it exists do not update the file\n- else respond with <file_path></file_path>, <code_block></code_block> and <change></change> as xml tags\n- Only respond with single code_block in the response containing only the test code block to be updated\n{{/section}}";
 const reporter_1 = require("../../reporter");
 const update_flow_1 = require("./update-flow");
 const utils_1 = require("./utils");
@@ -27,6 +27,8 @@ ${testCase.steps.join("\n")}`;
     const prompt = (0, llm_1.compilePrompt)(promptTemplate_0, {
         task,
         repoFiles,
+        testCaseName: testCase.name,
+        testFilePath: file,
     });
     const createTestSpan = trace?.span({
         name: "create-test",

package/dist/agent/codegen/use-skill.d.ts

@@ -5,6 +5,7 @@ export declare function generateSkillUsageCode({ task, sampleUsageMethod, scopeV
     scopeVariablesMapStr: string;
     pageVariableName: string;
     skillMethodDefinition: string;
+    importPathForSkill: string;
     trace?: TraceClient;
 }): Promise<string>;
 //# sourceMappingURL=use-skill.d.ts.map

package/dist/agent/codegen/use-skill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"use-skill.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/use-skill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAWpE,wBAAsB,sBAAsB,CAAC,EAC3C,IAAI,EACJ,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,mBA6CA"}
+{"version":3,"file":"use-skill.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/use-skill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAWpE,wBAAsB,sBAAsB,CAAC,EAC3C,IAAI,EACJ,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,mBA6CA"}

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA+WA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAqBlC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA6BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA4XA"}

package/dist/agent/master/run.js

@@ -7,9 +7,12 @@ const skill_1 = require("../../actions/skill");
 const utils_1 = require("../../actions/utils");
 const logger_1 = require("../../bin/logger");
 const constants_1 = require("../../constants");
+const errors_1 = require("../../errors");
+const human_in_the_loop_1 = require("../../human-in-the-loop");
 const page_1 = require("../../page");
 const reporter_1 = require("../../reporter");
 const session_1 = require("../../session");
+const env_1 = require("../../utils/env");
 const browsing_1 = require("../browsing");
 const utils_2 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
@@ -176,6 +179,15 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                 void testGenReporter.sendMessage(output.reason);
                 logger.log(`Next Action: ${output.action}`);
                 if (toolCall.actionType === skill_1.SKILL_USAGE) {
+                    if (!(0, env_1.isRunningOnCloud)()) {
+                        const { skillDetails } = (0, skill_1.extractSkillFromArgs)(args);
+                        const feedback = await human_in_the_loop_1.humanLoop.getFeedback({
+                            message: `Should I use this skill?\n\n${skillDetails?.usageExample}\n\nfrom: ${skillDetails?.filePath}\n\nAnswer with "yes" or "no"`,
+                        });
+                        if (feedback.toLowerCase().trim().includes("no")) {
+                            throw new errors_1.HumanApprovalDenied("Skill usage approval not provided");
+                        }
+                    }
                     const code = await actions.executeAction(toolCall.actionType, args, masterAgentActionSpan);
                     if (code) {
                         generatedCodeSteps.push(code);
@@ -311,13 +323,15 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                         errorSummary: e.message,
                     },
                 });
-                logger.error("Failed to run master agent's next task", JSON.stringify(toolCall, null, 2), e);
-                failedActions.push(output.action);
-                if (failedActions.length >= MAX_ERROR_COUNT) {
-                    const error = "Agent is not able to figure out next action, marking task as done";
-                    logger.error(error);
-                    await testgenUpdatesReporter.sendMessage(error);
-                    break;
+                if (!(e instanceof errors_1.HumanApprovalDenied)) {
+                    logger.error("Failed to run master agent's next task", JSON.stringify(toolCall, null, 2), e);
+                    failedActions.push(output.action);
+                    if (failedActions.length >= MAX_ERROR_COUNT) {
+                        const error = "Agent is not able to figure out next action, marking task as done";
+                        logger.error(error);
+                        await testgenUpdatesReporter.sendMessage(error);
+                        break;
+                    }
                 }
             }
         }

package/dist/agent/planner/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAapE,wBAAsB,QAAQ,CAAC,EAC7B,IAAI,EACJ,QAAQ,EACR,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,gBA0HA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAcpE,wBAAsB,QAAQ,CAAC,EAC7B,IAAI,EACJ,QAAQ,EACR,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,gBAwHA"}

package/dist/agent/planner/run.js

@@ -5,6 +5,7 @@ const llm_1 = require("@empiricalrun/llm");
 const context_1 = require("../../bin/utils/context");
 const human_in_the_loop_1 = require("../../human-in-the-loop");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert software engineer in test and have capability to collaborate with user to generate a test plan.\nYou are given a task to provide a high level plan to create a test for a given scenario.\n\nYou will be provided with already added Playwright tests and page object object models which you can use generate the test plan.\nTest plan should be a list of bullet points and each bullet point representing each test step.\nYou will be provided with app knowledge which can help you groom the steps in the plan.\n\n{{#if humanLoopRequired}}\nDo NOT finish until you have a complete understanding of the plan and have all the minute details of how the test should look like.\n\nALWAYS VERIFY THE PLAN WITH USER BEFORE FINISHING. THE USER SHOULD BE ABLE TO REVIEW AND GIVE GO AHEAD TO THE PLAN.\n\nDO NOT FINISH UNTIL USER EXPLICITLY CONFIRMS THE PLAN IS GOOD TO GO.\n{{/if}}\n\n### Test Plan Creation Guidelines\n\n- **Create Subtasks:**\n  - Read the task step by step and create subtasks from the given task.\n  - Do not add any new steps that are not mentioned in the task.\n\n- **Enrich Subtasks:**\n  - Read the provided app knowledge.\n  - Enrich the subtasks based on the information from the app knowledge.\n\n- **Plan:**\n  - Once all subtasks are enriched, list them as bullet points.\n  - Each bullet point must start with one of the following actions:  \n    - Open page  \n    - Click on  \n    - Fill in  \n    - Assert  \n    - Hover on  \n    - Press  \n    - Extract textContent\n  - Do not include any bullet points that do not start with the above actions.\n\n- **Additional Notes:**\n  - The final output should only contain bullet points listing the test steps.\n  - Do not add any assertions that are not mentioned in the task or app knowledge.\n  - The plan should not reference the knowledge base used to generate it.\n\n{{/section}}\n\n{{#section \"user\"}}\n\nApp knowledge:\n{{appKnowledge}}\n\n-----\n\nCurrent tests and page object model references:\n{{fileContext}}\n\n------\n\nTask:\n{{task}}\n\n------\n{{/section}}";
+const env_1 = require("../../utils/env");
 async function planTask({ task, specPath, trace, }) {
     let fileContext = "";
     try {
@@ -24,7 +25,7 @@ ${pomPrompt}
         },
     });
     // if the code in running in ci worker or github actions, disable human loop
-    const humanLoopRequired = process.env.CI !== "true";
+    const humanLoopRequired = !(0, env_1.isRunningOnCloud)();
     const appKnowledge = await (0, context_1.fetchAppKnowledge)();
     const messages = (0, llm_1.compilePrompt)(promptTemplate_0, {
         appKnowledge,
@@ -37,8 +38,6 @@ ${pomPrompt}
         defaultModel: "gpt-4o",
     });
     let maxRetries = 10;
-    // TODO: this should be environment driven
-    const hl = new human_in_the_loop_1.HumanLoop({ method: "ipc" });
     const tools = [
         {
             type: "function",
@@ -96,7 +95,7 @@ ${pomPrompt}
         const resp = JSON.parse(respStr || "{}");
         // TODO: handle this as a part of actions handler
         if (toolCall?.function.name === "send_user_message") {
-            const feedback = await hl.getFeedback({
+            const feedback = await human_in_the_loop_1.humanLoop.getFeedback({
                 message: resp.message,
             });
             messages.push(response);

package/dist/bin/index.js

@@ -186,6 +186,7 @@ async function runAgent(testGenConfig, testGenToken, span) {
                 .option("--name <test-name>", "Name of the test case")
                 .option("--prompt <prompt>", "Prompt for the test case")
                 .option("--file <test-file>", "File path of the test case (inside tests dir)")
+                .option("--suites <suites>", "Comma separated list of describe blocks")
                 .parse(process.argv);
             const options = program.opts();
             (0, utils_2.validateCliOptions)(options);

package/dist/bin/utils/index.d.ts

@@ -5,6 +5,7 @@ export interface CliOptions {
     name?: string;
     file?: string;
     prompt?: string;
+    suites?: string;
 }
 export declare function validateCliOptions(options: CliOptions): void;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW,oBAA2B,CAAC;AACpD,eAAO,MAAM,OAAO,oBAA6B,CAAC;AAElD,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAS5D"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW,oBAA2B,CAAC;AACpD,eAAO,MAAM,OAAO,oBAA6B,CAAC;AAElD,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAS5D"}

package/dist/bin/utils/scenarios/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/scenarios/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAIV,aAAa,EAEd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAetC,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,GACjC,MAAM,CAgBR;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,GACjC,aAAa,CAYf;AAED,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,aAAa,CAmBnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/scenarios/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAIV,aAAa,EAEd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAetC,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,GACjC,MAAM,CAgBR;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,GACjC,aAAa,CAaf;AAED,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,aAAa,CAmBnE"}

package/dist/bin/utils/scenarios/index.js

@@ -27,8 +27,9 @@ function buildTestConfigFromOptions(options) {
             name: options.name,
             steps: [options.prompt],
             filePath: options.file,
-            // TODO: Support suites in this flow
-            suites: [],
+            suites: options.suites
+                ? options.suites.split(",").map((s) => s.trim())
+                : [],
         },
     };
 }

package/dist/errors/index.d.ts

@@ -0,0 +1,5 @@
+export declare class TestGenError extends Error {
+}
+export declare class HumanApprovalDenied extends TestGenError {
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/errors/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/errors/index.ts"],"names":[],"mappings":"AAAA,qBAAa,YAAa,SAAQ,KAAK;CAAG;AAC1C,qBAAa,mBAAoB,SAAQ,YAAY;CAAG"}

package/dist/errors/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HumanApprovalDenied = exports.TestGenError = void 0;
+class TestGenError extends Error {
+}
+exports.TestGenError = TestGenError;
+class HumanApprovalDenied extends TestGenError {
+}
+exports.HumanApprovalDenied = HumanApprovalDenied;

package/dist/file/client.d.ts

@@ -1,6 +1,8 @@
 declare class TestFileService {
     baseUrl: string;
-    constructor(port: number);
+    port: number | undefined;
+    constructor();
+    static isAvailable(): boolean;
     updateTest({ generatedCode, task, importPaths, }: {
         generatedCode: string;
         task: string;

package/dist/file/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/file/client.ts"],"names":[],"mappings":"AAAA,cAAM,eAAe;IACnB,OAAO,EAAE,MAAM,CAAC;gBACJ,IAAI,EAAE,MAAM;IAIlB,UAAU,CAAC,EACf,aAAa,EACb,IAAI,EACJ,WAAW,GACZ,EAAE;QACD,aAAa,EAAE,MAAM,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAgBK,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG;CAgBnC;AAED,eAAe,eAAe,CAAC"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/file/client.ts"],"names":[],"mappings":"AAAA,cAAM,eAAe;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;;IASzB,MAAM,CAAC,WAAW;IAIZ,UAAU,CAAC,EACf,aAAa,EACb,IAAI,EACJ,WAAW,GACZ,EAAE;QACD,aAAa,EAAE,MAAM,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAgBK,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG;CAgBnC;AAED,eAAe,eAAe,CAAC"}

package/dist/file/client.js

@@ -2,9 +2,17 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 class TestFileService {
     baseUrl;
-    constructor(port) {
+    port;
+    constructor() {
+        const port = Number(process.env.APP_PORT);
+        if (port && !isNaN(port)) {
+            this.port = port;
+        }
         this.baseUrl = `http://localhost:${port}`;
     }
+    static isAvailable() {
+        return !!Number(process.env.APP_PORT);
+    }
     async updateTest({ generatedCode, task, importPaths, }) {
         const resp = await fetch(`${this.baseUrl}/test`, {
             method: "POST",

package/dist/human-in-the-loop/index.d.ts

@@ -1,10 +1,12 @@
-export declare class HumanLoop {
-    private options;
-    constructor(options: {
+declare class HumanLoop {
+    options: {
         method: "ipc" | "cli";
-    });
+    } | undefined;
+    constructor();
     getFeedback({ message }: {
         message: string;
     }): Promise<string>;
 }
+export declare const humanLoop: HumanLoop;
+export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/human-in-the-loop/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/human-in-the-loop/index.ts"],"names":[],"mappings":"AAQA,qBAAa,SAAS;IAER,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE;QAAE,MAAM,EAAE,KAAK,GAAG,KAAK,CAAA;KAAE;IAChD,WAAW,CAAC,EAAE,OAAO,EAAE,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE;CAOnD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/human-in-the-loop/index.ts"],"names":[],"mappings":"AASA,cAAM,SAAS;IAEb,OAAO,EAAE;QAAE,MAAM,EAAE,KAAK,GAAG,KAAK,CAAA;KAAE,GAAG,SAAS,CAAC;;IAMzC,WAAW,CAAC,EAAE,OAAO,EAAE,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE;CAOnD;AAED,eAAO,MAAM,SAAS,WAAkB,CAAC"}

package/dist/human-in-the-loop/index.js

@@ -1,6 +1,10 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HumanLoop = void 0;
+exports.humanLoop = void 0;
+const client_1 = __importDefault(require("../file/client"));
 const cli_1 = require("./cli");
 const ipc_1 = require("./ipc");
 // TODO: how to make this more conversational
@@ -8,17 +12,19 @@ const ipc_1 = require("./ipc");
 // we need a layer to put the feedback in the context again
 // and then we need to generate final output which the user can review
 class HumanLoop {
-    options;
     // TODO: have a better name for this
-    constructor(options) {
-        this.options = options;
+    options;
+    constructor() {
+        this.options = {
+            method: client_1.default.isAvailable() ? "ipc" : "cli",
+        };
     }
     async getFeedback({ message }) {
         // this needs to have a provider based on environment
-        if (this.options.method === "ipc") {
+        if (this.options?.method === "ipc") {
             return await (0, ipc_1.getFeedbackUsingIPC)(message);
         }
         return await (0, cli_1.getFeedbackUsingCli)(message);
     }
 }
-exports.HumanLoop = HumanLoop;
+exports.humanLoop = new HumanLoop();

package/dist/human-in-the-loop/ipc.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ipc.d.ts","sourceRoot":"","sources":["../../src/human-in-the-loop/ipc.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAKvC,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,QAwB1C;AAED,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAa1E"}
+{"version":3,"file":"ipc.d.ts","sourceRoot":"","sources":["../../src/human-in-the-loop/ipc.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAKvC,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,QAwB1C;AAED,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAY1E"}

package/dist/human-in-the-loop/ipc.js

@@ -34,8 +34,7 @@ function humanLoopRoute(app) {
 }
 exports.humanLoopRoute = humanLoopRoute;
 async function getFeedbackUsingIPC(message) {
-    const port = process.env.APP_PORT || 3030;
-    const fileService = new client_1.default(Number(port));
+    const fileService = new client_1.default();
     try {
         const resp = await fileService.post("/converse", { message });
         return resp.feedback;

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,cAAc,CAAC;AAItB,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAapC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAuC3E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,cAAc,CAAC;AAItB,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAapC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,SAAS,iBAsC3E"}

package/dist/index.js

@@ -45,7 +45,6 @@ process.on("exit", async () => await flushEvents());
 process.on("SIGINT", async () => await flushEvents());
 process.on("SIGTERM", async () => await flushEvents());
 async function createTest(task, page, scope) {
-    const port = process.env.APP_PORT || 3030;
     const testConfigArg = process.env.TEST_GEN_TOKEN;
     const testGenConfig = (0, scenarios_1.loadTestConfigs)(testConfigArg);
     (0, reporter_1.setReporterConfig)({
@@ -59,7 +58,7 @@ async function createTest(task, page, scope) {
         testCaseId: testGenConfig.testCase.id,
         projectRepoName: testGenConfig.options?.metadata.projectRepoName,
     });
-    const fileService = new client_1.default(Number(port));
+    const fileService = new client_1.default();
     const { testCase, specPath } = testGenConfig;
     const { code, importPaths } = await (0, run_1.createTestUsingMasterAgent)({
         testCase,

package/dist/session/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAkBA,iBAAS,iBAAiB;;;;;EAOzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,EACZ,UAAU,EACV,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB,QAIA;AAED,wBAAsB,iBAAiB,qBAItC;AAED,wBAAsB,eAAe,0DA0BpC;AAED,wBAAsB,UAAU,kBAyB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC;AAE7B,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;IACP,MAAM,EAAE,MAAM,CAAC;CAChB,iBAmBF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAmBA,iBAAS,iBAAiB;;;;;EAOzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,EACZ,UAAU,EACV,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB,QAIA;AAED,wBAAsB,iBAAiB,qBAItC;AAED,wBAAsB,eAAe,0DA0BpC;AAED,wBAAsB,UAAU,kBAyB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC;AAE7B,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;IACP,MAAM,EAAE,MAAM,CAAC;CAChB,iBAmBF"}

package/dist/session/index.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.updateSessionStatus = exports.getSessionDetails = exports.endSession = exports.getSessionState = exports.shouldStopSession = exports.setSessionDetails = void 0;
 const package_json_1 = __importDefault(require("../../package.json"));
+const env_1 = require("../utils/env");
 const sessionDetails = {
     sessionId: undefined,
     version: package_json_1.default.version,
@@ -12,7 +13,7 @@ const sessionDetails = {
     testUrl: undefined,
 };
 const DASHBOARD_DOMAIN = process.env.DASHBOARD_DOMAIN ||
-    (process.env.CI === "true" ? "https://dash.empirical.run" : "");
+    ((0, env_1.isRunningOnCloud)() ? "https://dash.empirical.run" : "");
 function getSessionDetails() {
     return {
         generationId: sessionDetails.generationId,

package/dist/utils/env.d.ts

@@ -0,0 +1,2 @@
+export declare function isRunningOnCloud(): boolean;
+//# sourceMappingURL=env.d.ts.map

package/dist/utils/env.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"env.d.ts","sourceRoot":"","sources":["../../src/utils/env.ts"],"names":[],"mappings":"AAAA,wBAAgB,gBAAgB,YAE/B"}

package/dist/utils/env.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isRunningOnCloud = void 0;
+function isRunningOnCloud() {
+    return process.env.CI === "true";
+}
+exports.isRunningOnCloud = isRunningOnCloud;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.43.3",
+  "version": "0.45.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -74,8 +74,8 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "@empiricalrun/llm": "^0.9.35",
-    "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/reporter": "^0.23.1"
+    "@empiricalrun/reporter": "^0.23.1",
+    "@empiricalrun/r2-uploader": "^0.3.8"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",