@empiricalrun/test-gen 0.47.1 → 0.47.3

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @empiricalrun/test-gen
 
+## 0.47.3
+
+### Patch Changes
+
+- 56ed4eb: chore: remove parseJson utility, let it throw exceptions
+
+## 0.47.2
+
+### Patch Changes
+
+- af97c0f: feat: cua agent can generate code
+- d7f1678: feat: support openai cua for overlay dismissal, bump openai to 4.87.3
+- 09e880a: feat: add more actions for cua with better types
+- Updated dependencies [d7f1678]
+  - @empiricalrun/llm@0.9.36
+
 ## 0.47.1
 
 ### Patch Changes

package/dist/actions/index.d.ts

@@ -9,7 +9,7 @@ export declare class PlaywrightActions {
     constructor(page: TestGenPage, stateVariables?: Record<string, any>);
     executeAction(name: string | undefined, args: ActionArgs, trace?: TraceClient): Promise<string | undefined>;
     getBrowsingActionSchemas(): ActionSchema[];
-    getMasterActionSchemas(): ActionSchema[];
+    getSkillsActionSchemas(): ActionSchema[];
     generateCode(): {
         code: string;
         importPaths: string[];

package/dist/actions/index.js

@@ -108,7 +108,7 @@ class PlaywrightActions {
         }))
             .map((a) => a.schema);
     }
-    getMasterActionSchemas() {
+    getSkillsActionSchemas() {
         return [skill_1.skillActionGenerator]
             .map((a) => a(this.page, {
             stateVariables: this.stateVariables,

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CA8D1C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAMlD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CA8D1C"}

package/dist/agent/browsing/index.js

@@ -6,7 +6,6 @@ const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a browser automation agent who is given a task to generate code for navigation and assertion. This task is your\ngoal and you must achieve it.\n\nYou will be provided with already executed actions and basis that you need to pick the next step to achieve the task.\nRemember that the goal must be achieved.\n\nYou will be provided with the web page snapshot in the form of Document Object Model. Based on the goal and available\ntool calls you need to pick the appropriate tool call.\n\nInstructions:\n- Take actions one at a time. Do not try to take multiple actions\n- You can respond with multiple assertions in one shot\n- Do not repeat the same actions again otherwise your response will be marked INVALID\n- Avoid repeating errors which we got while executing the last action\n- Stick to the task provided to you and mark the task done once the task is complete\n- Do not execute any action which is not mentioned in the task\n- Do not repeat actions which are already executed more than twice otherwise your response will be marked INVALID\n- Always refer to \"Executed actions\" before deciding your next action for completion of the task.\n- End the task done if all actions required for task are executed\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\nCurrent page snapshot:\n{{pageSnapshot}}\n{{/section}}";
 const reporter_1 = require("../../reporter");
 const html_1 = require("../../utils/html");
-const utils_1 = require("../utils");
 async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace, }) {
     let generatedCodeSteps;
     const tools = actions.getBrowsingActionSchemas();
@@ -48,9 +47,9 @@ async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace
     const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
     for (const i in toolCalls) {
         const toolCall = toolCalls[i];
-        const args = (0, utils_1.parseJson)(toolCall.function.arguments);
+        const args = JSON.parse(toolCall.function.arguments);
         try {
-            const code = await actions.executeAction(toolCall.function.name, (0, utils_1.parseJson)(toolCall.function.arguments), toolCallsSpan);
+            const code = await actions.executeAction(toolCall.function.name, JSON.parse(toolCall.function.arguments), toolCallsSpan);
             generatedCodeSteps = {
                 // Passing reason as action, in order to pass the correct action that took place to runtime planner
                 action: args.reason,

package/dist/agent/codegen/generate-code-apply-changes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate-code-apply-changes.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/generate-code-apply-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQrE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAOhD,OAAO,EAAsB,UAAU,EAAE,MAAM,SAAS,CAAC;AAqLzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,UAkD3D;AA2DD,wBAAsB,2BAA2B,CAAC,EAChD,IAAI,EACJ,KAAK,EACL,MAAM,EACN,gBAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gBAAgB,EAAE,MAAM,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuIxB"}
+{"version":3,"file":"generate-code-apply-changes.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/generate-code-apply-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQrE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAsB,UAAU,EAAE,MAAM,SAAS,CAAC;AAqLzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,UAkD3D;AA2DD,wBAAsB,2BAA2B,CAAC,EAChD,IAAI,EACJ,KAAK,EACL,MAAM,EACN,gBAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gBAAgB,EAAE,MAAM,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuIxB"}

package/dist/agent/codegen/generate-code-apply-changes.js

@@ -8,9 +8,8 @@ const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = require("path");
 const constants_1 = require("../../constants");
-const utils_1 = require("../utils");
 const types_1 = require("./types");
-const utils_2 = require("./utils");
+const utils_1 = require("./utils");
 function getCodeEditorToolCalls() {
     const strReplace = {
         name: "code-block-replace",
@@ -160,7 +159,7 @@ async function getPlanForCodeEditorAgent({ prompt, trace, }) {
     }
     if (completion.tool_calls[0].function.name === "change_plan") {
         const args = completion.tool_calls[0].function.arguments;
-        const plan = (0, utils_1.parseJson)(args).plan;
+        const plan = JSON.parse(args).plan;
         return plan;
     }
 }
@@ -342,7 +341,7 @@ async function generateCodeAndApplyChanges({ task, trace, logger, getRelevantFil
         }
         await Promise.all(createFileToolCalls.map((tc) => {
             return (async () => {
-                const args = (0, utils_1.parseJson)(tc.function.arguments);
+                const args = JSON.parse(tc.function.arguments);
                 updatedFiles.push({
                     filePath: args.filePath,
                     oldCode: "",
@@ -360,13 +359,13 @@ async function generateCodeAndApplyChanges({ task, trace, logger, getRelevantFil
         }
         // Filter out the tool calls which are for replacing code in existing files
         const fileChanges = strReplaceToolCalls
-            .map((toolCall) => (0, utils_1.parseJson)(toolCall.function.arguments))
+            .map((toolCall) => JSON.parse(toolCall.function.arguments))
             .filter((f) => f.filePath && fs_extra_1.default.existsSync(f.filePath));
         // We add all the suggested changes to the updatedFiles array
         // This is used to validate and format files later
         updatedFiles.push(...fileChanges);
         // applyChangesResponse contains the errors occurred while applying the changes
-        const updates = await (0, utils_2.applyFileChangesUsingStrReplace)({
+        const updates = await (0, utils_1.applyFileChangesUsingStrReplace)({
             trace: codeEditorSpan,
             fileChanges,
             logger,

package/dist/agent/codegen/run.d.ts

@@ -1,6 +1,8 @@
 import { TraceClient } from "@empiricalrun/llm";
 import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
+import OpenAI from "openai";
 import { CustomLogger } from "../../bin/logger";
+import { CreateTestCodeUpdate } from "./types";
 export declare function createTestWithCodeAgent({ testCase, file, repoFiles, trace, }: {
     testCase: TestCase;
     file: string;
@@ -8,9 +10,9 @@ export declare function createTestWithCodeAgent({ testCase, file, repoFiles, tra
     trace?: TraceClient;
     logger?: CustomLogger;
 }): Promise<{
-    prompt: import("openai/resources/index.mjs").ChatCompletionMessageParam[];
+    prompt: OpenAI.Chat.Completions.ChatCompletionMessageParam[];
     agentResponse: string;
-    fileChanges: import("./types").CreateTestCodeUpdate[];
+    fileChanges: CreateTestCodeUpdate[];
 }>;
 export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace?: TraceClient): Promise<TestCase[] | void>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAmBhD,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;;;;GAyDA;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CA8D5B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAYhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAQ/C,wBAAsB,uBAAuB,CAAC,EAC5C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC;IAC7D,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,oBAAoB,EAAE,CAAC;CACrC,CAAC,CAyDD;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CA8D5B"}

package/dist/agent/codegen/utils.d.ts

@@ -72,18 +72,6 @@ export declare function applyFileChangesUsingStrReplace({ trace, fileChanges, lo
     fileChanges: CodeUpdate[];
     logger?: CustomLogger;
 }): Promise<FileUpdateResponse[]>;
-export declare function searchAndReplaceCodeUsingStrReplace({ logger, fileChange, }: {
-    fileChange: {
-        filePath: string | undefined;
-        oldCode: string | undefined;
-        newCode: string | undefined;
-        reason: string | undefined;
-    };
-    logger?: CustomLogger;
-}): Promise<{
-    result: FileUpdateResponse;
-    updatedContent: string;
-}>;
 export declare function applyFileChangesForCreateTest({ trace, fileChanges, testgenUpdatesReporter, }: {
     trace?: TraceClient;
     fileChanges: CreateTestCodeUpdate[];

package/dist/agent/codegen/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAE/E;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,EAAE,CAiB9D;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,GACZ,oBAAoB,EAAE,CAgBxB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAqCF;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,SAAS,EACT,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC;IAClC,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA0BA;AAED,wBAAsB,+BAA+B,CAAC,EACpD,KAAK,EACL,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA6ChC;AAED,wBAAsB,mCAAmC,CAAC,EACxD,MAAM,EACN,UAAU,GACX,EAAE;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,kBAAkB,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CAuCD;AAED,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,WAAW,EACX,sBAAsB,GACvB,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD,iBAoDA;AAED,wBAAsB,oBAAoB,CAAC,EACzC,MAAM,EACN,UAAU,GACX,EAAE;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACnE,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CA6BD;AAED,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAkIxE"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAE/E;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,EAAE,CAiB9D;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,GACZ,oBAAoB,EAAE,CAgBxB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAqCF;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,SAAS,EACT,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC;IAClC,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA0BA;AAED,wBAAsB,+BAA+B,CAAC,EACpD,KAAK,EACL,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA6ChC;AAED,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,WAAW,EACX,sBAAsB,GACvB,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD,iBAoDA;AAED,wBAAsB,oBAAoB,CAAC,EACzC,MAAM,EACN,UAAU,GACX,EAAE;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IACV,MAAM,EAAE;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACnE,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CA8BD;AAED,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAkIxE"}

package/dist/agent/codegen/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.applyFileChanges = exports.searchAndReplaceCode = exports.applyFileChangesForCreateTest = exports.searchAndReplaceCodeUsingStrReplace = exports.applyFileChangesUsingStrReplace = exports.validateTypesAndFormatCode = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractCreateTestUpdates = exports.extractTestUpdates = void 0;
+exports.applyFileChanges = exports.searchAndReplaceCode = exports.applyFileChangesForCreateTest = exports.applyFileChangesUsingStrReplace = exports.validateTypesAndFormatCode = exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractCreateTestUpdates = exports.extractTestUpdates = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const ts_morph_1 = require("ts-morph");
@@ -198,44 +198,6 @@ async function applyFileChangesUsingStrReplace({ trace, fileChanges, logger, })
     return results;
 }
 exports.applyFileChangesUsingStrReplace = applyFileChangesUsingStrReplace;
-async function searchAndReplaceCodeUsingStrReplace({ logger, fileChange, }) {
-    let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
-    if (contents.includes(fileChange.oldCode)) {
-        // Check for multiple instances of old code block
-        // If there are multiple instances, then we cannot safely determine which instance to replace
-        const firstIndex = contents.indexOf(fileChange.oldCode);
-        const lastIndex = contents.lastIndexOf(fileChange.oldCode);
-        if (firstIndex !== lastIndex) {
-            return {
-                result: {
-                    error: true,
-                    errorMessage: `Multiple instances of the code block found in file "${fileChange.filePath}". Cannot safely determine which instance to replace.`,
-                    filePath: fileChange.filePath,
-                },
-                updatedContent: contents,
-            };
-        }
-        const updatedContent = contents.replace(fileChange.oldCode, `\n\n${fileChange.newCode}`);
-        return {
-            result: {
-                error: false,
-                errorMessage: "",
-                filePath: fileChange.filePath,
-            },
-            updatedContent,
-        };
-    }
-    logger?.error(`Unable to find the code to update in ${fileChange.filePath}`);
-    return {
-        result: {
-            error: true,
-            errorMessage: `The content of "old_code_block" corresponding to file path "${fileChange.filePath}" did not match the current content of the file "${fileChange.filePath}"`,
-            filePath: fileChange.filePath,
-        },
-        updatedContent: contents,
-    };
-}
-exports.searchAndReplaceCodeUsingStrReplace = searchAndReplaceCodeUsingStrReplace;
 async function applyFileChangesForCreateTest({ trace, fileChanges, testgenUpdatesReporter, }) {
     const repoEditFileChangesSpan = trace?.span({
         name: "create-test-file-changes",
@@ -298,6 +260,7 @@ async function searchAndReplaceCode({ logger, fileChange, }) {
     return {
         result: {
             error: true,
+            // TODO: old_code_block should be replaced with oldCode
             errorMessage: `The content of "old_code_block" corresponding to file path "${fileChange.filePath}" did not match the current content of the file "${fileChange.filePath}"`,
             filePath: fileChange.filePath,
         },

package/dist/agent/cua/computer.d.ts

@@ -0,0 +1,7 @@
+import { ResponseComputerToolCall } from "openai/resources/responses/responses.mjs";
+import type { Page } from "playwright";
+type ComputerAction = ResponseComputerToolCall.Click | ResponseComputerToolCall.DoubleClick | ResponseComputerToolCall.Drag | ResponseComputerToolCall.Keypress | ResponseComputerToolCall.Move | ResponseComputerToolCall.Screenshot | ResponseComputerToolCall.Scroll | ResponseComputerToolCall.Type | ResponseComputerToolCall.Wait;
+export declare function getScreenshot(page: Page): Promise<string>;
+export declare function handleModelAction(page: Page, action: ComputerAction): Promise<string>;
+export {};
+//# sourceMappingURL=computer.d.ts.map

package/dist/agent/cua/computer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,KAAK,cAAc,GACf,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAG7C;AAgCD,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC,MAAM,CAAC,CA2HjB"}

package/dist/agent/cua/computer.js

@@ -0,0 +1,151 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleModelAction = exports.getScreenshot = void 0;
+async function getScreenshot(page) {
+    const screenshotBytes = await page.screenshot();
+    return Buffer.from(screenshotBytes).toString("base64");
+}
+exports.getScreenshot = getScreenshot;
+const CUA_KEY_TO_PLAYWRIGHT_KEY = {
+    "/": "Divide",
+    "\\": "Backslash",
+    alt: "Alt",
+    arrowdown: "ArrowDown",
+    arrowleft: "ArrowLeft",
+    arrowright: "ArrowRight",
+    arrowup: "ArrowUp",
+    backspace: "Backspace",
+    capslock: "CapsLock",
+    // "cmd" and "ctrl" are both mapped to "ControlOrMeta" for platform
+    // agnostic behavior, as opposed to cmd: "Meta" and ctrl: "Control"
+    cmd: "ControlOrMeta",
+    ctrl: "ControlOrMeta",
+    delete: "Delete",
+    end: "End",
+    enter: "Enter",
+    esc: "Escape",
+    home: "Home",
+    insert: "Insert",
+    option: "Alt",
+    pagedown: "PageDown",
+    pageup: "PageUp",
+    shift: "Shift",
+    space: " ",
+    super: "Meta",
+    tab: "Tab",
+    win: "Meta",
+};
+async function handleModelAction(page, action) {
+    const actionType = action.type;
+    let actionCode = "";
+    try {
+        switch (actionType) {
+            case "click": {
+                const { x, y, button = "left" } = action;
+                console.log(`Action: click at (${x}, ${y}) with button '${button}'`);
+                let pwButton = undefined;
+                if (button === "left" || button === "right") {
+                    pwButton = button;
+                }
+                else if (button === "wheel") {
+                    pwButton = "middle";
+                }
+                if (pwButton) {
+                    const locator = await page.evaluate(([x, y]) => {
+                        const element = document.elementFromPoint(x, y);
+                        return window.playwright.generateLocator(element);
+                    }, [x, y]);
+                    actionCode = `await page.${locator}.click();\n`;
+                    await page.mouse.click(x, y, { button: pwButton });
+                }
+                if (button === "back" || button === "forward") {
+                    // Do page navigations, since there is no way to click on the back/forward buttons
+                    if (button === "back") {
+                        await page.goBack();
+                    }
+                    else if (button === "forward") {
+                        await page.goForward();
+                    }
+                }
+                break;
+            }
+            case "double_click": {
+                const { x, y } = action;
+                console.log(`Action: doubleclick at (${x}, ${y})`);
+                await page.mouse.dblclick(x, y, { button: "left" });
+                break;
+            }
+            case "move": {
+                const { x, y } = action;
+                console.log(`Action: move to (${x}, ${y})`);
+                await page.mouse.move(x, y);
+                break;
+            }
+            case "drag": {
+                const { path } = action;
+                console.log(`Action: drag along path ${path}`);
+                if (!path || path.length === 0) {
+                    break;
+                }
+                await page.mouse.move(path[0].x, path[0].y);
+                await page.mouse.down();
+                for (let i = 1; i < path.length; i++) {
+                    await page.mouse.move(path[i].x, path[i].y);
+                }
+                await page.mouse.up();
+                break;
+            }
+            case "scroll": {
+                const { x, y, scroll_x, scroll_y } = action;
+                console.log(`Action: scroll at (${x}, ${y}) with offsets (scroll_x=${scroll_x}, scroll_y=${scroll_y})`);
+                await page.mouse.move(x, y);
+                await page.evaluate(`window.scrollBy(${scroll_x}, ${scroll_y})`);
+                break;
+            }
+            case "keypress": {
+                const { keys } = action;
+                const mappedKeys = keys.map((k) => {
+                    return CUA_KEY_TO_PLAYWRIGHT_KEY[k.toLowerCase()] || k;
+                });
+                const mappedKey = mappedKeys.join("+"); // ["CTRL", "A"] becomes ControlOrMeta+A
+                console.log(`Action: keypress for keys ${keys} -> '${mappedKey}'`);
+                try {
+                    await page.keyboard.press(mappedKey);
+                    actionCode = `await page.keyboard.press('${mappedKey}');\n`;
+                }
+                catch (e) {
+                    console.error("Error pressing key", mappedKey, ":", e);
+                }
+                break;
+            }
+            case "type": {
+                const { text } = action;
+                console.log(`Action: type text '${text}'`);
+                await page.keyboard.type(text);
+                const locator = await page.evaluate(() => {
+                    const element = document.activeElement;
+                    return window.playwright.generateLocator(element);
+                });
+                actionCode = `await page.${locator}.fill("${text}");\n`;
+                break;
+            }
+            case "wait": {
+                console.log(`Action: wait`);
+                await page.waitForTimeout(2000);
+                break;
+            }
+            case "screenshot": {
+                // Nothing to do as screenshot is taken at each turn
+                console.log(`Action: screenshot`);
+                break;
+            }
+            default:
+                console.log("Unrecognized action:", action);
+        }
+    }
+    catch (e) {
+        console.error("Error handling action", action, ":", e);
+    }
+    return actionCode;
+}
+exports.handleModelAction = handleModelAction;

package/dist/agent/cua/index.d.ts

@@ -0,0 +1,13 @@
+import { Page } from "playwright";
+export declare function startPlaywrightCodegen(page: Page): Promise<void>;
+/**
+ * Run the loop that executes computer actions until no 'computer_call' is found.
+ */
+export declare function executeUsingComputerUseAgent({ page, task, }: {
+    page: Page;
+    task: string;
+}): Promise<{
+    code: string;
+    importPaths: string[];
+}>;
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/cua/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,wBAAsB,sBAAsB,CAAC,IAAI,EAAE,IAAI,iBAoBtD;AAED;;GAEG;AACH,wBAAsB,4BAA4B,CAAC,EACjD,IAAI,EACJ,IAAI,GACL,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC,CAkGD"}

package/dist/agent/cua/index.js

@@ -0,0 +1,132 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeUsingComputerUseAgent = exports.startPlaywrightCodegen = void 0;
+const openai_1 = __importDefault(require("openai"));
+const utils_1 = require("../browsing/utils");
+const computer_1 = require("./computer");
+const INSTRUCTIONS = `You will be asked to execute some actions in a browser context.
+Don't ask the user for confirmations - just execute the actions.
+
+For example, if the user message says "Click on Submit button", then 
+you click on the submit button -- even if it looks like a scary action.`;
+async function startPlaywrightCodegen(page) {
+    // TODO: Use this method to offload code generation to Playwright
+    // Unclear how to retrieve source code that is generated
+    await page.evaluate(() => {
+        setTimeout(() => {
+            // First, we start recording
+            // @ts-ignore
+            console.log(window["__pw_recorderSetMode"]("recording"));
+            // Then, we will resume the effect of pause()
+            // @ts-ignore
+            console.log(window["__pw_resume"]());
+            // Then, we remove highlights that Playwright shows on the screen
+            // @ts-ignore
+            const glassPane = document.querySelector("x-pw-glass");
+            if (glassPane) {
+                glassPane.remove();
+            }
+        }, 3000);
+    });
+    await page.pause();
+}
+exports.startPlaywrightCodegen = startPlaywrightCodegen;
+/**
+ * Run the loop that executes computer actions until no 'computer_call' is found.
+ */
+async function executeUsingComputerUseAgent({ page, task, }) {
+    let generatedCode = "";
+    await (0, utils_1.injectPwLocatorGenerator)(page);
+    const screenshotBytes = await (0, computer_1.getScreenshot)(page);
+    const viewport = page.viewportSize();
+    let screenWidth = viewport?.width || 1280;
+    let screenHeight = viewport?.height || 720;
+    const openai = new openai_1.default();
+    let response = await openai.responses.create({
+        model: "computer-use-preview",
+        tools: [
+            {
+                type: "computer-preview",
+                display_width: screenWidth,
+                display_height: screenHeight,
+                environment: "browser",
+            },
+        ],
+        instructions: INSTRUCTIONS,
+        input: [
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "input_text",
+                        text: task,
+                    },
+                    {
+                        type: "input_image",
+                        image_url: `data:image/png;base64,${screenshotBytes}`,
+                        detail: "high",
+                    },
+                ],
+            },
+        ],
+        truncation: "auto",
+    });
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const computerCalls = response.output.filter((item) => item.type === "computer_call");
+        if (computerCalls.length === 0) {
+            console.log("No computer call found. Output from model:");
+            /**
+             * TODO: Sometimes the mdoel will ask for a user confirmation - handle this flow
+             * item.type is "message", status is "completed", item.content.type is "output_text"
+             */
+            response.output.forEach((item) => {
+                console.log(JSON.stringify(item, null, 2));
+            });
+            break; // Exit when no computer calls are issued.
+        }
+        // We expect at most one computer call per response.
+        const computerCall = computerCalls[0];
+        const lastCallId = computerCall.call_id;
+        const action = computerCall.action;
+        // Execute the action (function defined in step 3)
+        const actionCode = await (0, computer_1.handleModelAction)(page, action);
+        generatedCode += actionCode;
+        await new Promise((resolve) => setTimeout(resolve, 1000)); // Allow time for changes to take effect.
+        // Take a screenshot after the action (function defined in step 4)
+        const screenshotBytes = await (0, computer_1.getScreenshot)(page);
+        // Send the screenshot back as a computer_call_output
+        response = await openai.responses.create({
+            model: "computer-use-preview",
+            previous_response_id: response.id,
+            tools: [
+                {
+                    type: "computer-preview",
+                    display_width: screenWidth,
+                    display_height: screenHeight,
+                    environment: "browser",
+                },
+            ],
+            input: [
+                {
+                    call_id: lastCallId,
+                    type: "computer_call_output",
+                    output: {
+                        type: "computer_screenshot",
+                        image_url: `data:image/png;base64,${screenshotBytes}`,
+                    },
+                },
+            ],
+            truncation: "auto",
+        });
+    }
+    return {
+        code: generatedCode,
+        // TODO: Does not support skills, so import paths are empty
+        importPaths: [],
+    };
+}
+exports.executeUsingComputerUseAgent = executeUsingComputerUseAgent;

package/dist/agent/diagnosis-agent/strict-mode-violation.d.ts

@@ -1,9 +1,10 @@
 import { TestErrorDiagnosisDetails } from "@empiricalrun/shared-types";
+import OpenAI from "openai";
 export declare function fixStrictModeViolationPrompt({ screenshotsData, diagnosis, }: {
     screenshotsData: {
         success: string[];
         failure: string[];
     };
     diagnosis: TestErrorDiagnosisDetails;
-}): import("openai/resources/index.mjs").ChatCompletionMessageParam[];
+}): OpenAI.Chat.Completions.ChatCompletionMessageParam[];
 //# sourceMappingURL=strict-mode-violation.d.ts.map

package/dist/agent/diagnosis-agent/strict-mode-violation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"strict-mode-violation.d.ts","sourceRoot":"","sources":["../../../src/agent/diagnosis-agent/strict-mode-violation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAoBvE,wBAAgB,4BAA4B,CAAC,EAC3C,eAAe,EACf,SAAS,GACV,EAAE;IACD,eAAe,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC1D,SAAS,EAAE,yBAAyB,CAAC;CACtC,qEAiBA"}
+{"version":3,"file":"strict-mode-violation.d.ts","sourceRoot":"","sources":["../../../src/agent/diagnosis-agent/strict-mode-violation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,MAAM,MAAM,QAAQ,CAAC;AAoB5B,wBAAgB,4BAA4B,CAAC,EAC3C,eAAe,EACf,SAAS,GACV,EAAE;IACD,eAAe,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC1D,SAAS,EAAE,yBAAyB,CAAC;CACtC,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAiBvD"}

package/dist/agent/master/browser-tests/index.spec.js

@@ -49,7 +49,7 @@ click on maverick inside ford dropdown`,
     (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page.+Ford.+.click/))).toBeTruthy();
     (0, fixtures_1.expect)(lines.find((l) => l.match(/^await page.+Maverick.+.click/))).toBeTruthy();
 });
-(0, fixtures_1.test)("agent can click icons accurately", async ({ page, server }) => {
+(0, fixtures_1.test)("master agent can click icons accurately", async ({ page, server }) => {
     await page.goto(`${server.baseURL}/icons-navbar.html`);
     await (0, fixtures_1.expect)(page.getByText("select an icon")).toBeVisible();
     const response = await (0, run_1.createTestUsingMasterAgent)({
@@ -72,6 +72,20 @@ click on maverick inside ford dropdown`,
     // expect(icons.length).toBe(4); // 1 for each unique icon
     fs_1.default.unlinkSync(iconsRegistryFile);
 });
+(0, fixtures_1.test)("cua agent can click icons accurately", async ({ page, server }) => {
+    await page.goto(`${server.baseURL}/icons-navbar.html`);
+    await (0, fixtures_1.expect)(page.getByText("select an icon")).toBeVisible();
+    const response = await (0, run_1.executeUsingComputerUseAgent)({
+        task: `click on the gear icon`,
+        page,
+    });
+    // Validate code generated and action performed
+    await (0, fixtures_1.expect)(page.getByText("you clicked Gear")).toBeVisible();
+    console.log(response);
+    (0, fixtures_1.expect)(response.importPaths.length).toBe(0);
+    (0, fixtures_1.expect)(response.code).toContain("page.locator");
+    (0, fixtures_1.expect)(response.code).toContain("click()");
+});
 (0, fixtures_1.test)("annotate and enrich annotations correctly", async ({ page, server }) => {
     await (0, utils_1.injectPwLocatorGenerator)(page);
     await page.goto(`${server.baseURL}/iframe-elements.html`);

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA2DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAgBF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAoFD"}
+{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA2DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAgBF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAoFD"}

package/dist/agent/master/element-annotation.js

@@ -4,7 +4,6 @@ exports.getAnnotationKeys = exports.getElementAnnotation = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements\npresent in the image.\n\nYou will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation\nis done by drawing a red box around the element and a small yellow box on it which contains unique element id.\n\nYou are given a Annotations which contains list of unique element id and description of the element separated by \":\".\n\nYou are also given the description of the element on which the action needs to be taken. The description includes\ninformation about how the element looks, it's position etc.\n\nYour task is to provide the annotation of the element on which the action needs to be performed based on the element\ndescription.\n\nFollow steps to fulfil your task:\n- Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each\nelement.\n- For describing each element Id\n-- iterate over each element Id in annotation list\n-- check if the description is already present for the element Id in the Annotation provided to you. If present skip\ndescribing it and use it as is.\n-- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or\nicon enclosed in the element.\n- Respond with the mapped element Ids as \"enriched_annotations\"\n- Based on the description provided to you and the enriched annotations, first identify the element Id whose description\nmatches the task provided\n\nNote:\n- Ensure providing the description of all the elements in the list.\n- Don't update the description if its already present in the given annotations\n- Replace all the \"NA\" with description of the element. Its position, how does it look like etc.\n- There should be no \"NA\" present in any of the element description\n{{/section}}\n\n{{#section \"user\"}}\nElement description:\n{{elementDescription}}\n\nAnnotations:\n{{annotations}}\n\n{{image annotatedScreenshot}}\n{{/section}}";
-const utils_1 = require("../utils");
 const icon_descriptor_1 = require("./icon-descriptor");
 const annotationToolAction = {
     name: "element_annotation",
@@ -94,7 +93,7 @@ async function getElementAnnotation({ elementDescription, annotations, annotated
     const toolCall = completion?.tool_calls?.[0];
     annotationsSpan?.end({ output: toolCall });
     if (toolCall) {
-        const args = (0, utils_1.parseJson)(toolCall.function.arguments);
+        const args = JSON.parse(toolCall.function.arguments);
         return args.element_annotation;
     }
     return;

package/dist/agent/master/execute-browser-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,GAAG,EAAE,GAAG,CAAC;CACV;;;gBAIW,MAAM;gBACN,MAAM;;;GAkIjB"}
+{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAWlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,GAAG,EAAE,GAAG,CAAC;CACV;;;gBAIW,MAAM;gBACN,MAAM;;;GAkIjB"}

package/dist/agent/master/execute-browser-action.js

@@ -3,12 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeBrowserAction = void 0;
 const reporter_1 = require("../../reporter");
 const browsing_1 = require("../browsing");
-const utils_1 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
 const element_annotation_1 = require("./element-annotation");
 const with_hints_1 = require("./with-hints");
 async function executeBrowserAction({ page, nextAction, flags, actions, llm, trace, }) {
-    const args = (0, utils_1.parseJson)(nextAction.toolCallArgs);
+    const args = JSON.parse(nextAction.toolCallArgs);
     let generatedCodeSteps = [];
     let output = {
         action: args.action,

package/dist/agent/master/execute-skill-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-skill-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-skill-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,wBAAsB,kBAAkB,CAAC,EACvC,OAAO,EACP,UAAU,EACV,KAAK,GACN,EAAE;IACD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;CAChC,+BAeA"}
+{"version":3,"file":"execute-skill-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-skill-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAMlD,wBAAsB,kBAAkB,CAAC,EACvC,OAAO,EACP,UAAU,EACV,KAAK,GACN,EAAE;IACD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;CAChC,+BAeA"}

package/dist/agent/master/execute-skill-action.js

@@ -5,9 +5,8 @@ const skill_1 = require("../../actions/skill");
 const errors_1 = require("../../errors");
 const human_in_the_loop_1 = require("../../human-in-the-loop");
 const env_1 = require("../../utils/env");
-const utils_1 = require("../utils");
 async function executeSkillAction({ actions, nextAction, trace, }) {
-    const args = (0, utils_1.parseJson)(nextAction.toolCallArgs);
+    const args = JSON.parse(nextAction.toolCallArgs);
     if (!(0, env_1.isRunningOnCloud)()) {
         const { skillDetails } = (0, skill_1.extractSkillFromArgs)(args);
         const feedback = await human_in_the_loop_1.humanLoop.getFeedback({

package/dist/agent/master/next-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAWhD,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,cAAc,EACd,IAAI,EACJ,eAAe,EACf,aAAa,EACb,KAAK,EACL,GAAG,EACH,OAAO,EACP,OAAO,EACP,aAAa,EACb,MAAM,GACP,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACxC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CACP;IACE,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,GACD,SAAS,CACZ,CA8FA"}
+{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAUhD,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,cAAc,EACd,IAAI,EACJ,eAAe,EACf,aAAa,EACb,KAAK,EACL,GAAG,EACH,OAAO,EACP,OAAO,EACP,aAAa,EACb,MAAM,GACP,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACxC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CACP;IACE,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,GACD,SAAS,CACZ,CA8FA"}

package/dist/agent/master/next-action.js

@@ -5,7 +5,6 @@ const llm_1 = require("@empiricalrun/llm");
 const skill_1 = require("../../actions/skill");
 const constants_1 = require("../../constants");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are a web automation tool which is given a task to complete. You need to execute the\ntask provided to you with the help of web page screenshot, a browser automation tool or skills\nwhich are learnt while writing previous tests. \n  \nBrowser automation tool is a tool which uses Playwright and browser to execute action using\nnext_action tool call.\nSkill usage is a tool which helps to execute previously known pieces of code to achieve a task.\n\nYou will be provided with a screenshot of the webpage which you will use to extract the action\nthat needs to be taken.\n\nYou will be provided with previously executed actions by the browser automation tool and based\non the current screenshot and previously executed actions, you need to predict the next action\nto be taken.\n\nYou will also be provided with failed next action predicted by you, so that you can avoid\nsuggesting the same action again - which failed.\n\nThe next action should be as atomic as possible. e.g: scroll, click on an element, fill an input\nelement, assert, extract text from an element are valid next action as they are atomic in nature.\n\nYou also need to provide the action type using the list below, action type which is not present in\nthe list is invalid: {{validActionTypes}}\n\nYou will also be provided with skill usage tool which you can use to execute action. These skills\nare compound functions which helps you to complete your action.\n\nYou need to respond with either:\n- Next action to be taken by a browser automation tool \n- Use previously learnt skills in the form of tool call.\n  \nYou need to make a decision whether the given skill can be reused if \"YES\" respond with the\nskill else respond with the next action.\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\n-----\n\nPrevious executed actions:\n{{executedActions}}\n\n-----\n\nPrevious failed actions:\n{{failedActions}}\n\n-----\n\nYou are also provided with a page screenshot for you to decide the next action.\n\nCurrent page URL: {{pageUrl}}\n\nFollow the instructions before responding:\n- Divide the task into sub tasks\n- Using previously executed actions, identify tasks are complete and which tasks needs to be executed next.\n- You will be provided a skill usage action, if the testStep matches the next action then respond with the skill usage.\n- If responding with next action, ensure next action to be detailed and explicit about what action needs to be done. Provide all the information which can be extracted from the screenshot as a part of next action.\n- Mark task as complete only when executed actions provided to you indicates that the task is done.\n- Refer to the text and references available in the screenshot to create the next action.\n- Do not take any extra actions which are not required for the execution of the task\n- If there are no further actions required based on the task, then respond with task as done.\n- Do not recommend actions which are not available in the screenshot\n\nScreenshots:\n{{images pageScreenshots}}\n\n{{/section}}\n";
-const utils_1 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
 const scroller_1 = require("./scroller");
 async function getNextAction({ page, pageScreenshot, task, executedActions, failedActions, trace, llm, options, actions, disableSkills, logger, }) {
@@ -33,7 +32,7 @@ async function getNextAction({ page, pageScreenshot, task, executedActions, fail
     });
     const actionSchemas = disableSkills || skill_1.testCaseSkills.getAvailableSkills().length === 0
         ? []
-        : actions.getMasterActionSchemas();
+        : actions.getSkillsActionSchemas();
     const tools = [...(0, action_tool_calls_1.getActionToolCalls)(), ...actionSchemas];
     llm =
         llm ||
@@ -57,7 +56,7 @@ async function getNextAction({ page, pageScreenshot, task, executedActions, fail
     const toolCall = completion?.tool_calls?.[0];
     nextActionSpan?.end({ output: toolCall });
     if (toolCall) {
-        const toolCallArgs = (0, utils_1.parseJson)(toolCall.function.arguments);
+        const toolCallArgs = JSON.parse(toolCall.function.arguments);
         const actionType = toolCall.function.name;
         // If the action type is scroll, we need to scroll the page and get the reference to the frame in which the element is visible
         // else we return the next action

package/dist/agent/master/planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../../../src/agent/master/planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAKvC,wBAAsB,4BAA4B,CAAC,EACjD,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GAwIA"}
+{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../../../src/agent/master/planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAIvC,wBAAsB,4BAA4B,CAAC,EACjD,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GAwIA"}

package/dist/agent/master/planner.js

@@ -4,7 +4,6 @@ exports.runtimePlannerWithScreenshot = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
 const constants_1 = require("../../constants");
-const utils_1 = require("../utils");
 async function runtimePlannerWithScreenshot({ trace, task, conversation, pages, page, currentPage, }) {
     const buffer = await page.screenshot({
     //This is done to improve element annotation accuracy, anyways it doesn't annotate elements which are out of viewport
@@ -120,7 +119,7 @@ async function runtimePlannerWithScreenshot({ trace, task, conversation, pages,
     });
     const toolCallResp = (response?.tool_calls || [])[0];
     if (toolCallResp) {
-        const toolCall = (0, utils_1.parseJson)(toolCallResp.function.arguments);
+        const toolCall = JSON.parse(toolCallResp.function.arguments);
         const output = {
             pageName: toolCall.pageName,
             isDone: toolCall.isDone,

package/dist/agent/master/run.d.ts

@@ -1,6 +1,7 @@
 import type { TestCase, TestGenConfigOptions } from "@empiricalrun/shared-types";
 import { Page } from "playwright";
 import { ScopeVars } from "../../types";
+export { executeUsingComputerUseAgent } from "../cua";
 export declare function createTestUsingMasterAgent({ task, page, testCase, specPath, options, scopeVars, }: {
     task: string;
     page: Page;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAgCxC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwRA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAAE,4BAA4B,EAAE,MAAM,QAAQ,CAAC;AAuBtD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwRA"}

package/dist/agent/master/run.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createTestUsingMasterAgent = void 0;
+exports.createTestUsingMasterAgent = exports.executeUsingComputerUseAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const actions_1 = require("../../actions");
 const skill_1 = require("../../actions/skill");
@@ -15,11 +15,12 @@ const utils_2 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
 const run_1 = require("../planner/run");
 const run_time_planner_1 = require("../planner/run-time-planner");
-const utils_3 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
 const execute_browser_action_1 = require("./execute-browser-action");
 const execute_skill_action_1 = require("./execute-skill-action");
 const next_action_1 = require("./next-action");
+var cua_1 = require("../cua");
+Object.defineProperty(exports, "executeUsingComputerUseAgent", { enumerable: true, get: function () { return cua_1.executeUsingComputerUseAgent; } });
 const MAX_ERROR_COUNT = 2;
 function getPageVariables(stateVariables) {
     const keys = Object.keys(stateVariables);
@@ -162,7 +163,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                 await testgenUpdatesReporter.sendMessage("Agent is not able to figure out next action since element is not visible on screen.");
                 break;
             }
-            const args = (0, utils_3.parseJson)(nextAction.toolCallArgs);
+            const args = JSON.parse(nextAction.toolCallArgs);
             const masterAgentActionSpan = masterAgentSpan?.span({
                 name: "master-agent-execute-action",
             });

package/dist/agent/master/scroller.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAG7C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAchD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AA2ZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}
+{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAG7C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAahD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AA2ZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}

package/dist/agent/master/scroller.js

@@ -5,7 +5,6 @@ const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
 const constants_1 = require("../../constants");
 const reporter_1 = require("../../reporter");
-const utils_1 = require("../utils");
 const action_tool_calls_1 = require("./action-tool-calls");
 const element_annotation_1 = require("./element-annotation");
 let usedAnnotations = [];
@@ -165,7 +164,7 @@ Follow the instructions before responding:
     const toolCall = completion?.tool_calls?.[0];
     scrollSpan?.end({ output: toolCall });
     if (toolCall) {
-        const args = (0, utils_1.parseJson)(toolCall.function.arguments);
+        const args = JSON.parse(toolCall.function.arguments);
         isVisible = args.is_visible || false;
     }
     else {
@@ -303,7 +302,7 @@ ${annotationKeysString}`,
     const toolCall = completion?.tool_calls?.[0];
     annotationsSpan?.end({ output: toolCall });
     if (toolCall) {
-        const args = (0, utils_1.parseJson)(toolCall.function.arguments);
+        const args = JSON.parse(toolCall.function.arguments);
         const isAnnotationPresentInKeys = annotationKeys.some((annotation) => annotation.elementID === args.element_annotation);
         if (args.element_annotation !== "NA" && isAnnotationPresentInKeys) {
             usedAnnotations.push(args.element_annotation);

package/dist/agent/planner/run-time-planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;CAC1B;;;;GA+FA"}
+{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;CAC1B;;;;GA+FA"}

package/dist/agent/planner/run-time-planner.js

@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.runtimePlanner = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const promptTemplate_0 = "{{#section \"system\"}}\nYou are given a list of successfully executed actions that are done towards completing a task (which\nis also provided to you). Your goal is to analyse the list and determine if the task is completed.\n\nIf the task is not fully completed, identify which specific actions are missing\nand suggest next steps to complete the task. Assume that the conversation provided\nis entirely truthful and no additional actions were performed beyond those listed.\n\nThese actions were executed by AI agents using Playwright on a browser. These agents\nalready have access to browser tabs to execute actions. If there is a pending action,\none of the agents will execute it in the browser. However, they need your help to\nchoose which browser tab (= page) to use for the next action.\n\nTo fulfil your goal, follow these steps:\n- Divide the task into individual actions.\n- Compare each task action against the actions listed in the successfully executed actions list.\n- Identify which actions have been executed and which have not.\n- If all actions are executed, respond with the task as done.\n- If any actions are missing, respond with the task as not done, listing all actions\n  and specifying which are complete and which are missing.\n- If provided with list of pages, based on the next pending action and previously executed\n  action, identify the page on which next action needs to be taken\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\n----\n\nSuccessfully executed actions:\n{{successfulActions}}\n\n----\n\nList of pages with their current URLs:\n{{pagesSummary}}\n\n\n{{/section}}\n";
-const utils_1 = require("../utils");
 async function runtimePlanner({ trace, task, successfulActions, pages, currentPage, }) {
     const runTimePlannerSpan = trace?.span({
         name: "runtime-planner",
@@ -76,7 +75,7 @@ async function runtimePlanner({ trace, task, successfulActions, pages, currentPa
     });
     const toolCallResp = (response?.tool_calls || [])[0];
     if (toolCallResp) {
-        const toolCall = (0, utils_1.parseJson)(toolCallResp.function.arguments);
+        const toolCall = JSON.parse(toolCallResp.function.arguments);
         const output = {
             pageName: toolCall.pageName,
             isDone: toolCall.isDone,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.47.1",
+  "version": "0.47.3",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -64,7 +64,7 @@
     "mime": "^4.0.4",
     "minimatch": "^10.0.1",
     "nanoid": "^5.0.7",
-    "openai": "4.67.0",
+    "openai": "4.87.3",
     "picocolors": "^1.0.1",
     "prettier": "^3.2.5",
     "remove-markdown": "^0.5.5",
@@ -73,9 +73,9 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/reporter": "^0.23.1",
+    "@empiricalrun/llm": "^0.9.36",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/llm": "^0.9.35"
+    "@empiricalrun/reporter": "^0.23.1"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",
@@ -91,7 +91,7 @@
     "js-levenshtein": "^1.1.6",
     "playwright": "1.47.1",
     "ts-patch": "^3.3.0",
-    "@empiricalrun/shared-types": "0.0.5"
+    "@empiricalrun/shared-types": "0.0.6"
   },
   "scripts": {
     "dev": "tspc --build --watch",

package/dist/agent/utils.d.ts

@@ -1,2 +0,0 @@
-export declare function parseJson(args: string): any;
-//# sourceMappingURL=utils.d.ts.map

package/dist/agent/utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/agent/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,OAMrC"}

package/dist/agent/utils.js

@@ -1,12 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseJson = void 0;
-function parseJson(args) {
-    try {
-        return JSON.parse(args);
-    }
-    catch (e) {
-        console.error(`Failed to parse JSON with args ${args}`, e);
-    }
-}
-exports.parseJson = parseJson;