@empiricalrun/test-gen 0.37.3 → 0.37.4

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @empiricalrun/test-gen
 
+## 0.37.4
+
+### Patch Changes
+
+- 46e6f61: test: added test for fetch pom skills
+- 5315b91: fix: remove support for page.reload action
+- 9b4894a: fix: identification of agent stuck in loop
+- 8c6bab2: fix: regex for extracting test steps suggestion
+- 0128934: fix: added code in trace for master agent
+
 ## 0.37.3
 
 ### Patch Changes

package/dist/actions/index.d.ts

@@ -7,7 +7,7 @@ export declare class PlaywrightActions {
     private actionGenerators;
     private recordedActions;
     constructor(page: Page, stateVariables?: Record<string, any>);
-    executeAction(name: string | undefined, args: Record<string, any>, trace?: TraceClient): Promise<void>;
+    executeAction(name: string | undefined, args: Record<string, any>, trace?: TraceClient): Promise<string | undefined>;
     getBrowsingActionSchemas(): ActionSchema[];
     getMasterActionSchemas(): ActionSchema[];
     generateCode(): {

package/dist/actions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAYnE,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,IAAI,EACV,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAiB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW;IAgDrB,wBAAwB,IAAI,YAAY,EAAE;IAmB1C,sBAAsB,IAAI,YAAY,EAAE;IAUxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAO9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;CAUX"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,YAAY,EAA6B,MAAM,UAAU,CAAC;AAWnE,qBAAa,iBAAiB;IAQ1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,cAAc;IARxB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAInB;gBAEM,IAAI,EAAE,IAAI,EACV,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAgB5C,aAAa,CACjB,IAAI,oBAAa,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAgD9B,wBAAwB,IAAI,YAAY,EAAE;IAkB1C,sBAAsB,IAAI,YAAY,EAAE;IAUxC,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAUD,gBAAgB,CAAC,KAAK,EAAE,MAAM;IAK9B;;;;;;OAMG;IACH,aAAa,IAAI,OAAO;IAQxB,UAAU;CAUX"}

package/dist/actions/index.js

@@ -10,7 +10,6 @@ const fill_1 = require("./fill");
 const goto_1 = require("./goto");
 const hover_1 = require("./hover");
 const press_1 = require("./press");
-const reload_page_1 = require("./reload-page");
 const skill_1 = require("./skill");
 const text_content_1 = require("./text-content");
 class PlaywrightActions {
@@ -29,7 +28,6 @@ class PlaywrightActions {
             press_1.pressActionGenerator,
             done_1.doneActionGenerator,
             assert_1.assertTextVisibilityActionGenerator,
-            reload_page_1.reloadActionGenerator,
             text_content_1.textContentActionGenerator,
             skill_1.skillActionGenerator,
         ];
@@ -71,6 +69,7 @@ class PlaywrightActions {
                 logger.log(`action: ${name} \ncode: ${code} \nreason: ${args.reason}`);
                 void testgenUpdatesReporter.sendMessage("```ts\n" + code + "\n```");
             }
+            return code;
         }
         catch (e) {
             actionSpan?.end({
@@ -90,7 +89,6 @@ class PlaywrightActions {
             press_1.pressActionGenerator,
             done_1.doneActionGenerator,
             assert_1.assertTextVisibilityActionGenerator,
-            reload_page_1.reloadActionGenerator,
             text_content_1.textContentActionGenerator,
         ]
             .map((a) => a(this.page, {
@@ -115,7 +113,7 @@ class PlaywrightActions {
         };
     }
     getLastCodeLines(count) {
-        const actions = this.recordedActions.filter((a) => a.name !== done_1.PLAYWRIGHT_DONE_ACTION_NAME);
+        const actions = this.recordedActions;
         return actions.slice(-count).map((a) => a.code);
     }
     /**

package/dist/agent/browsing/index.d.ts

@@ -16,5 +16,5 @@ export declare function executeTaskUsingBrowsingAgent({ trace, action, logger, p
     options: BrowsingAgentOptions;
     llm: LLM;
     actions: PlaywrightActions;
-}): Promise<void>;
+}): Promise<string[] | undefined>;
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,iBAqIA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,CAiJhC"}

package/dist/agent/browsing/index.js

@@ -12,11 +12,15 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
     let isTaskDone = false;
     const executedActions = [];
     let lastActionExecTrace = "";
+    let generatedCodeSteps = [];
     const tools = actions.getBrowsingActionSchemas();
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     while (!isTaskDone) {
         const browsingAgentSpan = trace?.span({
             name: `browsing-agent`,
+            input: {
+                action,
+            },
         });
         if (await (0, session_1.shouldStopSession)()) {
             break;
@@ -48,7 +52,11 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
             if (isTaskDone) {
                 browsingAgentSpan?.event({ name: "task-done" });
                 browsingAgentSpan?.end({
-                    output: { taskDone: true, reason: verificationAgentResp.reason },
+                    output: {
+                        taskDone: true,
+                        reason: verificationAgentResp.reason,
+                        code: generatedCodeSteps,
+                    },
                 });
                 break;
             }
@@ -98,7 +106,10 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
                 break;
             }
             try {
-                await actions.executeAction(toolCall.function.name, JSON.parse(toolCall.function.arguments), toolCallsSpan);
+                const code = await actions.executeAction(toolCall.function.name, JSON.parse(toolCall.function.arguments), toolCallsSpan);
+                if (code) {
+                    generatedCodeSteps.push(code);
+                }
                 executedActions.push({
                     isError: false,
                     action: JSON.parse(toolCall.function.arguments)?.reason,
@@ -132,5 +143,6 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
             }
         }
     }
+    return generatedCodeSteps;
 }
 exports.executeTaskUsingBrowsingAgent = executeTaskUsingBrowsingAgent;

package/dist/agent/browsing/o1-completion.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"o1-completion.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/o1-completion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAKpC,wBAAsB,eAAe,CAAC,EACpC,QAAQ,EACR,KAAK,EACL,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CA6D7C"}
+{"version":3,"file":"o1-completion.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/o1-completion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAKpC,wBAAsB,eAAe,CAAC,EACpC,QAAQ,EACR,KAAK,EACL,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CA8D7C"}

package/dist/agent/browsing/o1-completion.js

@@ -33,6 +33,7 @@ async function getO1Completion({ messages, tools, trace, }) {
     - Do not respond with any backticks.
     - The reason for action should also include what was been executed in the action.
   `;
+        o1Span?.update({ input: [userInstruction] });
         const response = (await llm.createChatCompletion({
             messages: [userInstruction],
             modelParameters: {

package/dist/agent/codegen/skills-retriever.d.ts

@@ -10,6 +10,7 @@ export declare const fetchPomSkills: ({ testCase, pomFiles, options, trace, }: {
     filePath: string;
     usageExample: string;
     reason: string;
+    methodName: string;
 }[]>;
 export declare function getAppropriateSkills({ testCase, options, trace, }: {
     testCase: TestCase;
@@ -20,5 +21,6 @@ export declare function getAppropriateSkills({ testCase, options, trace, }: {
     filePath: string;
     usageExample: string;
     reason: string;
+    methodName: string;
 }[]>;
 //# sourceMappingURL=skills-retriever.d.ts.map

package/dist/agent/codegen/skills-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"skills-retriever.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/skills-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAYhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,eAAO,MAAM,cAAc;cAMf,QAAQ;;;;;;;;;IA8CnB,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,EACzC,QAAQ,EACR,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;;;;;KA6BA"}
+{"version":3,"file":"skills-retriever.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/skills-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAYhE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,eAAO,MAAM,cAAc;cAMf,QAAQ;;;;;;;;;;IA8CnB,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,EACzC,QAAQ,EACR,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;;;;;;KA6BA"}

package/dist/agent/codegen/utils.d.ts

@@ -43,5 +43,6 @@ export declare function extractTestStepsSuggestions(input: string): {
     filePath: string;
     usageExample: string;
     reason: string;
+    methodName: string;
 }[];
 //# sourceMappingURL=utils.d.ts.map

package/dist/agent/codegen/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,EAAE,CAeF"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF"}

package/dist/agent/codegen/utils.js

@@ -66,15 +66,16 @@ function extractAppendTestUpdates(input) {
 exports.extractAppendTestUpdates = extractAppendTestUpdates;
 function extractTestStepsSuggestions(input) {
     const result = [];
-    const regex = /<subtask>(.*?)<\/subtask>[\s\S]*?<file_import_path>([\s\S]*?)<\/file_import_path>[\s\S]*?<usage_example>([\s\S]*?)<\/usage_example>[\s\S]*?<reason>([\s\S]*?)<\/reason>/g;
+    const regex = /<subtask>(.*?)<\/subtask>[\s\S]*?<reason>([\s\S]*?)<\/reason>[\s\S]*?<file_import_path>([\s\S]*?)<\/file_import_path>[\s\S]*?<usage_example>([\s\S]*?)<\/usage_example>[\s\S]*?<method_name>([\s\S]*?)<\/method_name>/g;
     let match;
     while ((match = regex.exec(input)) !== null) {
-        const [, testStep, filePath, usageExample, reason] = match;
+        const [, testStep, filePath, usageExample, reason, methodName] = match;
         result.push({
             testStep: testStep?.trim() || "",
             filePath: filePath?.trim() || "",
             usageExample: usageExample?.trim() || "",
             reason: reason?.trim() || "",
+            methodName: methodName?.trim() || "",
         });
     }
     return result.filter((r) => !!r.filePath && !!r.usageExample);

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAwFA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA8SA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAwFA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwTA"}

package/dist/agent/master/run.js

@@ -178,6 +178,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
         const pageScreenshot = buffer.toString("base64");
         let output;
+        let generatedCodeSteps = [];
         let annotations;
         let testGenAnnotatedSnapshotUpdatePromise;
         let annotatedPageScreenshot;
@@ -269,23 +270,29 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                 void testGenReporter.sendMessage(output.reason);
                 logger.log(`Next Action: ${output.action}`);
                 if (toolCall.function.name === skill_1.SKILL_USAGE) {
-                    await actions.executeAction(toolCall.function.name, args, masterAgentActionSpan);
+                    const code = await actions.executeAction(toolCall.function.name, args, masterAgentActionSpan);
+                    if (code) {
+                        generatedCodeSteps.push(code);
+                    }
                 }
                 else if (shouldTriggerHintsFlow && hintsExecutionCompletion) {
                     const toolCalls = hintsExecutionCompletion?.tool_calls || [];
                     for (const i in toolCalls) {
                         const currentToolCall = toolCalls[i];
-                        await actions.executeAction(currentToolCall.function.name, {
+                        const code = await actions.executeAction(currentToolCall.function.name, {
                             ...JSON.parse(currentToolCall.function.arguments),
                             ...args,
                         }, masterAgentActionSpan);
+                        if (code) {
+                            generatedCodeSteps.push(code);
+                        }
                     }
                     if (actions.isStuckInLoop()) {
                         throw new Error("Agent is not able to figure out next action when using hints");
                     }
                 }
                 else {
-                    await (0, browsing_1.executeTaskUsingBrowsingAgent)({
+                    generatedCodeSteps = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
                         trace: masterAgentActionSpan,
                         action: output.action,
                         logger,
@@ -343,7 +350,11 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
             await testGenAnnotatedSnapshotUpdatePromise;
         }
         masterAgentSpan?.end({
-            output: { action: output.action, reason: output.reason },
+            output: {
+                action: output.action,
+                reason: output.reason,
+                code: generatedCodeSteps,
+            },
         });
     }
     const { code, importPaths } = actions.generateCode();

package/dist/evals/fetch-pom-skills-agent.evals.js

@@ -24,7 +24,7 @@ const fetchSkillsAgentEvaluator = async ({ item, trace }) => {
             {
                 name: "equality",
                 value: output.some((o) => {
-                    return item.expectedOutput.some((e) => e.usageExample === o.usageExample);
+                    return item.expectedOutput.some((e) => e.methodName === o.methodName);
                 })
                     ? 1
                     : 0,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.37.3",
+  "version": "0.37.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"

package/dist/actions/reload-page.d.ts

@@ -1,4 +0,0 @@
-import { PlaywrightActionGenerator } from "../types";
-export declare const PLAYWRIGHT_RELOAD_ACTION_NAME = "page_reload";
-export declare const reloadActionGenerator: PlaywrightActionGenerator;
-//# sourceMappingURL=reload-page.d.ts.map

package/dist/actions/reload-page.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"reload-page.d.ts","sourceRoot":"","sources":["../../src/actions/reload-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,6BAA6B,gBAAgB,CAAC;AAE3D,eAAO,MAAM,qBAAqB,EAAE,yBAiCnC,CAAC"}

package/dist/actions/reload-page.js

@@ -1,41 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.reloadActionGenerator = exports.PLAYWRIGHT_RELOAD_ACTION_NAME = void 0;
-const utils_1 = require("../agent/browsing/utils");
-const constants_1 = require("./constants");
-const utils_2 = require("./utils");
-exports.PLAYWRIGHT_RELOAD_ACTION_NAME = "page_reload";
-const reloadActionGenerator = (page) => {
-    return {
-        execute: async () => {
-            await page.reload();
-            await page.waitForTimeout(3000);
-            await (0, utils_1.injectPwLocatorGenerator)(page);
-        },
-        template: () => {
-            const code = `await ${(0, utils_2.getPageVarName)()}.reload();`;
-            return {
-                code,
-            };
-        },
-        name: exports.PLAYWRIGHT_RELOAD_ACTION_NAME,
-        schema: {
-            type: "function",
-            function: {
-                name: exports.PLAYWRIGHT_RELOAD_ACTION_NAME,
-                description: "reload the page by calling this method. Call this method only when a page reload is requested in the task.",
-                parameters: {
-                    type: "object",
-                    properties: {
-                        reason: {
-                            type: "string",
-                            description: constants_1.DEFAULT_ACTION_REASON_PROMPT,
-                        },
-                    },
-                    required: ["reason"],
-                },
-            },
-        },
-    };
-};
-exports.reloadActionGenerator = reloadActionGenerator;