@empiricalrun/test-gen 0.38.11 → 0.38.15

package/CHANGELOG.md

@@ -1,5 +1,34 @@
 # @empiricalrun/test-gen
 
+## 0.38.15
+
+### Patch Changes
+
+- 91ded8f: fix: incorrect annotations
+
+## 0.38.14
+
+### Patch Changes
+
+- bd5c945: fix: run update scenario prompts on claude
+- 31f8805: fix: runtime planner calling out actions as done
+- Updated dependencies [bd5c945]
+  - @empiricalrun/llm@0.9.26
+
+## 0.38.13
+
+### Patch Changes
+
+- babf00a: fix: updated check for instance element
+
+## 0.38.12
+
+### Patch Changes
+
+- 3739ba6: fix: userContext not getting imported in case of edit test
+- 368a859: fix: skill wasn't used in case task didn't contain next action details
+- 3d0185e: fix: reduce payload for fix ts error call
+
 ## 0.38.11
 
 ### Patch Changes

package/browser-injected-scripts/annotate-elements.js

@@ -103,8 +103,8 @@ function annotateClickableElements(options = {}) {
   }
 
   // Check if an element is clickable
-  function isElementClickable(element) {
-    // if (!(element instanceof Element)) return false;
+  function isElementClickable(element, windowToAnnotate) {
+    if (!(element instanceof windowToAnnotate.Element)) return false;
 
     const tagName = element.tagName.toLowerCase();
     let isClickable = false;
@@ -380,7 +380,7 @@ function annotateClickableElements(options = {}) {
   
     // Filter for clickable elements
     const clickableElements = Array.from(windowToAnnotate.document.querySelectorAll("*")).filter((el) => {
-      const isClickable = isElementClickable(el);
+      const isClickable = isElementClickable(el, windowToAnnotate);
       const isClickNotBlocked = isElementClickNotBlocked(el, windowToAnnotate);
       return isClickable && isClickNotBlocked;
     });  

package/dist/actions/skill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,KAAK,KAAK,GAAG;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,eAAO,MAAM,oBAAoB,EAAE,yBA8GlC,CAAC"}
+{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../../src/actions/skill.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC,KAAK,KAAK,GAAG;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,cAAM,cAAc;IACN,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,KAAK,EAAE;IAEnC,kBAAkB;IAIlB,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;CAG7B;AAED,eAAO,MAAM,cAAc,gBAAyB,CAAC;AAErD,eAAO,MAAM,oBAAoB,EAAE,yBAmHlC,CAAC"}

package/dist/actions/skill.js

@@ -27,7 +27,7 @@ const skillActionGenerator = (page, options) => {
     const availableSkills = exports.testCaseSkills.getAvailableSkills();
     return {
         execute: async ({ args, trace }) => {
-            const skill = args.skill;
+            const { skill, action } = args;
             const [skillDetails] = availableSkills.filter((skillDetails) => skillDetails.testStep === skill);
             if (!skillDetails) {
                 throw new Error(`No skill found for skill: ${skill}`);
@@ -39,7 +39,7 @@ const skillActionGenerator = (page, options) => {
             const skillFile = await promises_1.default.readFile(skillFilePath, "utf-8");
             const imports = Object.keys(module);
             const code = await (0, use_skill_1.generateSkillUsageCode)({
-                task: skillDetails.testStep,
+                task: action,
                 sampleUsageMethod: skillDetails.usageExample,
                 scopeVariablesMapStr: JSON.stringify(options.stateVariables || {}, null, 2),
                 pageVariableName: (0, utils_1.getPageVarName)(),
@@ -97,7 +97,11 @@ const skillActionGenerator = (page, options) => {
                     properties: {
                         reason: {
                             type: "string",
-                            description: "explain how this action will help to complete the task. the reason should align with the task provided",
+                            description: "Explain how this action will help to complete the task. The reason should align with the task provided",
+                        },
+                        action: {
+                            type: "string",
+                            description: "Explain the action for which this skill needs to be used. Mention all details about the action so that it could be used by the skill",
                         },
                         skill: {
                             type: "string",
@@ -107,7 +111,7 @@ const skillActionGenerator = (page, options) => {
                                 .join(", ")}`,
                         },
                     },
-                    required: ["reason", "skill"],
+                    required: ["reason", "action", "skill"],
                 },
             },
         },

package/dist/agent/browsing/utils.js

@@ -79,12 +79,12 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
     });
     await fs_extra_1.default.writeFile(createTestFilePath, (0, web_1.addNewImport)(await fs_extra_1.default.readFile(createTestFilePath, "utf-8"), ["createTest"], "@empiricalrun/test-gen"));
     updateFileSpan?.end();
-    const { codePrompt, pomPrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
+    const { pomPrompt, nonSpecFilePrompt } = await (0, context_1.contextForGeneration)(createTestFilePath);
     await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
         trace,
         file: createTestFilePath,
-        testCode: codePrompt,
         pomCode: pomPrompt,
+        nonSpecFileCode: nonSpecFilePrompt,
         testCase: testCase,
         options: genConfig.options,
     });
@@ -196,7 +196,7 @@ async function injectPwLocatorGenerator(page) {
                         });
                     }
                     catch (e) {
-                        console.warn("Error injecting script in iframe:", e);
+                        console.warn("Error injecting script in iframe.");
                     }
                 };
                 const iframes = document.getElementsByTagName("iframe");
@@ -235,7 +235,7 @@ async function injectPwLocatorGenerator(page) {
                     });
                 }
                 catch (e) {
-                    console.warn("Error injecting script in iframe:", e);
+                    console.warn("Error injecting script in iframe:");
                 }
             };
             const iframes = document.getElementsByTagName("iframe");
@@ -254,7 +254,7 @@ async function injectPwLocatorGenerator(page) {
         });
     }
     catch (e) {
-        console.warn("Error injecting script in iframe:", e);
+        console.warn("Error injecting script in iframe:");
     }
 }
 exports.injectPwLocatorGenerator = injectPwLocatorGenerator;

package/dist/agent/codegen/fix-ts-errors.d.ts

@@ -1,12 +1,12 @@
 import { TraceClient } from "@empiricalrun/llm";
 import { CustomLogger } from "../../bin/logger";
 import { TestCase, TestGenConfigOptions } from "../../types";
-export declare function validateAndFixTypescriptErrors({ trace, logger, file, testCode, pomCode, testCase, options, }: {
+export declare function validateAndFixTypescriptErrors({ trace, logger, file, pomCode, nonSpecFileCode, testCase, options, }: {
     trace?: TraceClient;
     logger?: CustomLogger;
     file: string;
-    testCode: string;
     pomCode: string;
+    nonSpecFileCode: string;
     testCase: TestCase;
     options?: TestGenConfigOptions;
 }): Promise<void>;

package/dist/agent/codegen/fix-ts-errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAQhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,iBAoEA"}
+{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAQhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,OAAO,EACP,eAAe,EACf,QAAQ,EACR,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,iBAwEA"}

package/dist/agent/codegen/fix-ts-errors.js

@@ -9,7 +9,7 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 const logger_1 = require("../../bin/logger");
 const web_1 = require("../../bin/utils/platform/web");
 const constants_1 = require("../../constants");
-async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.CustomLogger(), file, testCode, pomCode, testCase, options, }) {
+async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.CustomLogger(), file, pomCode, nonSpecFileCode, testCase, options, }) {
     const validateTypesSpan = trace?.span({ name: "detect-type-errors-in-file" });
     logger.log("Validating types...");
     let errors = (0, web_1.validateTypescript)(file);
@@ -35,13 +35,13 @@ async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.Cus
         errors.forEach((e) => console.warn(e));
         const promptSpan = trace?.span({ name: "fix-type-errors-prompt" });
         const instruction = await (0, llm_1.getPrompt)("fix-file-errors-ts", {
-            testFiles: testCode || "",
             pageFiles: pomCode || "",
+            fixtureFiles: nonSpecFileCode || "",
             scenarioFile: file,
             errors: errors,
             fileContent: fileContent,
-            scenaioName: testCase.name,
-        });
+            scenarioName: testCase.name,
+        }, 10);
         promptSpan?.end({ output: { instruction } });
         const llm = new llm_1.LLM({
             trace,

package/dist/agent/codegen/run.js

@@ -48,7 +48,7 @@ async function generateTest(testCase, file, options, trace) {
         fs_extra_1.default.createFileSync(file);
     }
     const context = await (0, context_1.contextForGeneration)(file);
-    const { codePrompt, pomPrompt, testFileContent } = context;
+    const { codePrompt, pomPrompt, nonSpecFilePrompt, testFileContent } = context;
     const { testBlock } = (0, web_1.getTypescriptTestBlock)({
         scenarioName: testCase?.name,
         content: testFileContent,
@@ -96,8 +96,8 @@ async function generateTest(testCase, file, options, trace) {
         trace,
         logger,
         file,
-        testCode: codePrompt,
         pomCode: pomPrompt,
+        nonSpecFileCode: nonSpecFilePrompt,
         testCase: testCase,
         options,
     });

package/dist/agent/codegen/update-flow.d.ts

@@ -11,7 +11,12 @@ export declare function getUpdateTestCodeCompletion({ testCase, testFileContent,
     testFileContent: string;
     trace?: TraceClient;
     options?: TestGenConfigOptions;
-}): Promise<string>;
+}): Promise<{
+    filePath: string | undefined;
+    oldCode: string | undefined;
+    newCode: string | undefined;
+    reason: string | undefined;
+}[]>;
 export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean, validate?: boolean, trace?: TraceClient): Promise<UpdatedTestCase[]>;
 export declare function getAppendCreateTestBlockCompletion({ testFiles, pageFiles, testCase, testFilePath, options, trace, }: {
     trace?: TraceClient;

package/dist/agent/codegen/update-flow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAsB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC,MAAM,CAAC,CA6ClB;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA6D5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBA4DA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAyC7B"}
+{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAoB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CACT;IACE,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CACJ,CA+CA;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA4D5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBA4DA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAyC7B"}

package/dist/agent/codegen/update-flow.js

@@ -16,7 +16,7 @@ const constants_1 = require("../../constants");
 const session_1 = require("../../session");
 const fix_ts_errors_1 = require("./fix-ts-errors");
 const utils_1 = require("./utils");
-async function applyFileChanges({ validateTypes = true, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, codePrompt, }) {
+async function applyFileChanges({ validateTypes = true, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }) {
     await Promise.allSettled(fileChanges.map(async (fileChange) => {
         if (!fileChange.filePath) {
             return;
@@ -92,8 +92,8 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
                 trace,
                 logger,
                 file: fileChange.filePath,
-                testCode: codePrompt,
                 pomCode: pomPrompt,
+                nonSpecFileCode: nonSpecFilePrompt,
                 testCase: testCase,
                 options: testGenOptions,
             });
@@ -128,28 +128,30 @@ async function getUpdateTestCodeCompletion({ testCase, testFileContent, testFile
         scenarioFile: testFilePath,
         currentScenarioCodeBlock,
     });
-    promptSpan?.end({ output: { instruction } });
     const llm = new llm_1.LLM({
         trace,
-        provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-        defaultModel: options?.model || constants_1.DEFAULT_MODEL,
-        providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
+        provider: "anthropic",
+        defaultModel: "claude-3-5-sonnet-latest",
+        providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
     });
     const firstShotMessage = await llm.createChatCompletion({
         messages: instruction,
         modelParameters: {
             ...constants_1.DEFAULT_MODEL_PARAMETERS,
             ...options?.modelParameters,
+            temperature: 0,
         },
     });
     let response = firstShotMessage?.content || "";
-    return response;
+    const fileChanges = (0, utils_1.extractTestUpdates)(response);
+    promptSpan?.end({ output: fileChanges });
+    return fileChanges;
 }
 exports.getUpdateTestCodeCompletion = getUpdateTestCodeCompletion;
 async function updateTest(testCase, file, options, logging = true, validate = true, trace) {
     const logger = new logger_1.CustomLogger({ useReporter: logging });
     const context = await (0, context_1.contextForGeneration)(file);
-    const { codePrompt, pomPrompt, testFileContent } = context;
+    const { codePrompt, pomPrompt, nonSpecFilePrompt, testFileContent } = context;
     const generatedTestCases = [];
     logger.logEmptyLine();
     const session = (0, session_1.getSessionDetails)();
@@ -176,12 +178,11 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
         name: "update-test",
         input: request,
     });
-    const response = await getUpdateTestCodeCompletion({
+    const fileChanges = await getUpdateTestCodeCompletion({
         ...request,
         trace: updateTestSpan,
     });
     logger.success("Test generated successfully!");
-    const fileChanges = (0, utils_1.extractTestUpdates)(response);
     await applyFileChanges({
         validateTypes: validate,
         trace: updateTestSpan,
@@ -190,7 +191,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
         logger,
         testGenOptions: options,
         pomPrompt: pomPrompt,
-        codePrompt: codePrompt,
+        nonSpecFilePrompt: nonSpecFilePrompt,
     });
     if (trace) {
         logger.log(`Trace: ${trace?.getTraceUrl()}`);
@@ -199,7 +200,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
         ...testCase,
         updatedFiles: fileChanges.map((f) => f.filePath),
     });
-    updateTestSpan?.end({ output: { response } });
+    updateTestSpan?.end({ output: { fileChanges } });
     await (0, llm_1.flushAllTraces)();
     return generatedTestCases;
 }
@@ -273,7 +274,7 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
             content: scenarioFileContent,
         },
     ]);
-    const { codePrompt, pomPrompt } = context;
+    const { pomPrompt, nonSpecFilePrompt } = context;
     const generatedTestCases = [];
     const appendCreateTestResp = await getAppendCreateTestBlockCompletion({
         testCase,
@@ -291,7 +292,7 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
         logger,
         testGenOptions: options,
         pomPrompt: pomPrompt,
-        codePrompt: codePrompt,
+        nonSpecFilePrompt: nonSpecFilePrompt,
         validateTypes,
     });
     if (trace) {

package/dist/agent/master/run.d.ts

@@ -16,7 +16,10 @@ export declare function getNextAction({ task, executedActions, failedActions, pa
     actions: PlaywrightActions;
     disableSkills: boolean;
     useHints: boolean;
-    annotations?: string[];
+    annotations?: {
+        elementID: string;
+        text: string;
+    }[];
 }): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;
 export declare function createTestUsingMasterAgent({ task, page, testCase, options, scopeVars, }: {
     task: string;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAclD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAoBrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FA2FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAsTA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAclD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAoBrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACrD,2FA6FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAiUA"}

package/dist/agent/master/run.js

@@ -52,8 +52,10 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
         failedActions: failedActions.map((a) => a).join("\n"),
         executedActions: executedActions.map((a) => a).join("\n"),
         pageUrl,
-        annotations,
-    }, 24);
+        annotations: annotations
+            ?.map((a) => `${a.elementID}:${a.text}`)
+            .join("\n"),
+    }, 27);
     // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
     const userMessage = promptMessages.filter((m) => m.role === "user")[0];
     const systemMessage = promptMessages.filter((m) => m.role === "system")[0];
@@ -175,7 +177,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         const plannerResp = await (0, run_time_planner_1.runtimePlanner)({
             trace: masterAgentSpan,
             task,
-            conversation: ["Successfully executed actions", ...masterAgentActions],
+            conversation: [...masterAgentActions],
             pages: getPageVariables(actions.getStateVariables()),
             currentPage: (0, utils_1.getPageVarName)(),
         });
@@ -210,8 +212,18 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                 // @ts-ignore
                 // eslint-disable-next-line no-undef
                 window.annotationInstance = annotateClickableElements();
+                const annotations = Object.entries(
                 // @ts-ignore
-                return Object.keys(window.annotationInstance.annotations);
+                window.annotationInstance.annotations).map(([key, value]) => ({
+                    elementID: key, // Assign the key to elementID
+                    text: 
+                    //@ts-ignore
+                    value.node.text?.trim() ||
+                        //@ts-ignore
+                        value.node.textContent?.trim() ||
+                        "<This is an icon or image. Check the screenshot>",
+                }));
+                return annotations;
             });
             await page.waitForTimeout(2000);
             const annonationBuffer = await page.screenshot({

package/dist/agent/master/with-hints.d.ts

@@ -13,7 +13,10 @@ export declare const triggerHintsFlow: ({ outputFromGetNextAction, generatedAnno
         action: string;
         elementAnnotation?: string;
     };
-    generatedAnnotations: Record<string, any>;
+    generatedAnnotations: {
+        elementID: string;
+        text: string;
+    }[];
     page: TestGenPage;
     llm: LLM;
     trace?: TraceClient | undefined;

package/dist/agent/master/with-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,WAAW;SACZ,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAuGA,CAAC"}
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;UACrD,WAAW;SACZ,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAwGA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -37,7 +37,7 @@ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations,
     try {
         const hasElementAnnotation = outputFromGetNextAction?.elementAnnotation?.length &&
             outputFromGetNextAction?.elementAnnotation?.trim()?.length &&
-            generatedAnnotations?.includes(outputFromGetNextAction?.elementAnnotation);
+            generatedAnnotations.some((annotation) => annotation.elementID === outputFromGetNextAction?.elementAnnotation);
         trace?.event({
             name: "has-element-annotation",
             output: {

package/dist/agent/planner/run-time-planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GA6GA"}
+{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GAiHA"}

package/dist/agent/planner/run-time-planner.js

@@ -22,14 +22,14 @@ async function runtimePlanner({ trace, task, conversation, pages, currentPage, }
         {
             role: "system",
             content: `
-Given a conversation that lists only the actions that were successfully executed and a task comprising multiple actions, your goal is to analyse the conversation and determine if the entire task is completed.
-These conversations are between AI agents using Playwright to execute actions on browser. These agents already have access to browser tabs to execute steps. The successfully executed steps on browser post browser has opened, is provided to you as conversation.
+Given a successfully executed actions that lists only the actions that were successfully executed and a task comprising multiple actions, your goal is to analyse the list and determine if the entire task is completed.
+These actions are executed by AI agents using Playwright on a browser. These agents already have access to browser tabs to execute actions. The successfully executed actions on browser post browser has opened, is provided to you as successfully executed actions.
 
 If the task is not fully completed, identify which specific actions are missing and suggest next steps to complete the task. Assume that the conversation provided is entirely truthful and no additional actions were performed beyond those listed.
 
 To fulfil your goal, follow these steps:
 - Divide the task into individual actions.
-- Compare each task action against the actions listed in the conversation.
+- Compare each task action against the actions listed in the successfully executed actions list.
 - Identify which actions have been executed and which have not.
 - If all actions are executed, respond with the task as done.
 - If any actions are missing, respond with the task as not done, listing all actions and specifying which are complete and which are missing.
@@ -41,9 +41,13 @@ To fulfil your goal, follow these steps:
             content: `
 Task: ${task}
 
-Conversation:
+----
+
+Following are successfully executed actions:
 ${conversation.join("\n")}
 
+----
+
 Current page:
 ${currentPage}
         `,

package/dist/bin/utils/context.d.ts

@@ -2,6 +2,7 @@ export declare function createGitIgnoreFileFilter(): Promise<(pathname: string)
 export declare function contextForGeneration(file?: string): Promise<{
     codePrompt: string | undefined;
     pomPrompt: string | undefined;
+    nonSpecFilePrompt: string | undefined;
     testFileContent: string;
 }>;
 export declare function fetchAppKnowledge(): Promise<string>;

package/dist/bin/utils/context.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/context.ts"],"names":[],"mappings":"AAKA,wBAAsB,yBAAyB,2CAS9C;AAED,wBAAsB,oBAAoB,CAAC,IAAI,CAAC,EAAE,MAAM;;;;GAOvD;AAED,wBAAsB,iBAAiB,oBAYtC"}
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/context.ts"],"names":[],"mappings":"AAQA,wBAAsB,yBAAyB,2CAS9C;AAED,wBAAsB,oBAAoB,CAAC,IAAI,CAAC,EAAE,MAAM;;;;;GAQvD;AAED,wBAAsB,iBAAiB,oBAYtC"}

package/dist/bin/utils/context.js

@@ -23,6 +23,7 @@ async function contextForGeneration(file) {
     return {
         codePrompt: await (0, fs_1.generatePromptFromDirectory)("./tests", filter),
         pomPrompt: await (0, fs_1.generatePromptFromDirectory)("./pages", filter),
+        nonSpecFilePrompt: await (0, fs_1.generatePromptFromNonSpecFiles)("./tests", filter),
         testFileContent: file ? await fs_extra_1.default.readFile(file, "utf-8") : "",
     };
 }

package/dist/bin/utils/fs/index.d.ts

@@ -2,4 +2,5 @@ import { FileContent } from "../../../types";
 export declare function readFilesInDirectory(dir: string | undefined, filterFunc: (file: string) => boolean): Promise<FileContent[]>;
 export declare function convertFileContentsToString(files?: FileContent[]): string;
 export declare function generatePromptFromDirectory(dir: string | undefined, filterFunc: (file: string) => boolean): Promise<string | undefined>;
+export declare function generatePromptFromNonSpecFiles(dir: string | undefined, filterFunc: (file: string) => boolean): Promise<string | undefined>;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/fs/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/fs/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,wBAAsB,oBAAoB,CACxC,GAAG,oBAAa,EAChB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,0BAgBtC;AAED,wBAAgB,2BAA2B,CAAC,KAAK,GAAE,WAAW,EAAO,UAQpE;AAED,wBAAsB,2BAA2B,CAC/C,GAAG,oBAAK,EACR,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,+BAStC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/fs/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,wBAAsB,oBAAoB,CACxC,GAAG,oBAAa,EAChB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,0BAgBtC;AAED,wBAAgB,2BAA2B,CAAC,KAAK,GAAE,WAAW,EAAO,UAQpE;AAED,wBAAsB,2BAA2B,CAC/C,GAAG,oBAAK,EACR,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,+BAStC;AAED,wBAAsB,8BAA8B,CAClD,GAAG,oBAAa,EAChB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,+BAgBtC"}

package/dist/bin/utils/fs/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generatePromptFromDirectory = exports.convertFileContentsToString = exports.readFilesInDirectory = void 0;
+exports.generatePromptFromNonSpecFiles = exports.generatePromptFromDirectory = exports.convertFileContentsToString = exports.readFilesInDirectory = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 async function readFilesInDirectory(dir = "", filterFunc) {
@@ -44,3 +44,20 @@ async function generatePromptFromDirectory(dir = "", filterFunc) {
     }
 }
 exports.generatePromptFromDirectory = generatePromptFromDirectory;
+async function generatePromptFromNonSpecFiles(dir = "", filterFunc) {
+    try {
+        let nonSpecFiles = [];
+        const files = await readFilesInDirectory(dir, filterFunc);
+        for (const file of files) {
+            if (!file.filePath.endsWith(".spec.ts")) {
+                nonSpecFiles.push(file);
+            }
+        }
+        const prompt = convertFileContentsToString(nonSpecFiles);
+        return prompt;
+    }
+    catch (error) {
+        console.error("Error reading directory:", error);
+    }
+}
+exports.generatePromptFromNonSpecFiles = generatePromptFromNonSpecFiles;

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,IAAI,EAEJ,UAAU,EAEX,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,eAAO,MAAM,gCAAgC,eAC/B,UAAU,KACrB,MAgBF,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB,CA2CA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC;AAEF,eAAO,MAAM,iCAAiC,cACjC,MAAM,EAAE,gBACL,MAAM,sBAyBrB,CAAC;AAEF,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,QAAQ,EACR,MAAM,GACP,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,iBAsDA;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,iBA2BzB;AAED,wBAAgB,aAAa,CAAC,EAC5B,QAAQ,EACR,QAAQ,GACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;CACpB,WAYA;AAED,wBAAgB,mBAAmB,CAAC,EAClC,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,UAOA;AAED,wBAAgB,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA4B5E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,IAAI,EAEJ,UAAU,EAEX,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,eAAO,MAAM,gCAAgC,eAC/B,UAAU,KACrB,MAgBF,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB,CA2CA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC;AAEF,eAAO,MAAM,iCAAiC,cACjC,MAAM,EAAE,gBACL,MAAM,sBAyBrB,CAAC;AAEF,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,QAAQ,EACR,MAAM,GACP,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,iBA8CA;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,iBA2BzB;AAED,wBAAgB,aAAa,CAAC,EAC5B,QAAQ,EACR,QAAQ,GACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;CACpB,WAYA;AAED,wBAAgB,mBAAmB,CAAC,EAClC,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,UAOA;AAED,wBAAgB,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA4B5E"}

package/dist/bin/utils/platform/web/index.js

@@ -404,29 +404,25 @@ async function addUserContextFixture({ scenarioName, filePath, suites, }) {
         return;
     }
     const fileContent = await fs_extra_1.default.readFile(filePath, "utf-8");
-    const { testNode, testAlias } = getTypescriptTestBlock({
+    const { testNode } = getTypescriptTestBlock({
         scenarioName,
         content: fileContent,
         suites,
     });
     const currentBlock = testNode?.getText();
-    const testCallExpression = testNode?.getFirstDescendantByKind(ts_morph_1.SyntaxKind.CallExpression);
-    if (testCallExpression?.getExpression().getText() === testAlias) {
-        const arrowFunction = testCallExpression.getFirstDescendantByKind(ts_morph_1.SyntaxKind.ArrowFunction);
-        if (arrowFunction) {
-            const param = arrowFunction.getParameters()[0]; // Get the first parameter
-            const destructuringParam = param?.getFirstDescendantByKind(ts_morph_1.SyntaxKind.ObjectBindingPattern);
-            if (destructuringParam) {
-                const elements = destructuringParam.getElements();
-                const userContextExists = elements.some((e) => e.getText().includes("userContext"));
-                console.log("userContextExists", userContextExists);
-                if (!userContextExists) {
-                    const updatedImport = destructuringParam
-                        .getText()
-                        .replace("}", ", userContext }");
-                    destructuringParam.replaceWithText(updatedImport);
-                    await fs_extra_1.default.writeFile(filePath, fileContent.replace(currentBlock, testNode.getText()), "utf-8");
-                }
+    const arrowFunction = testNode?.getFirstDescendantByKind(ts_morph_1.SyntaxKind.ArrowFunction);
+    if (arrowFunction) {
+        const param = arrowFunction.getParameters()[0]; // Get the first parameter
+        const destructuringParam = param?.getFirstDescendantByKind(ts_morph_1.SyntaxKind.ObjectBindingPattern);
+        if (destructuringParam) {
+            const elements = destructuringParam.getElements();
+            const userContextExists = elements.some((e) => e.getText().includes("userContext"));
+            if (!userContextExists) {
+                const updatedImport = destructuringParam
+                    .getText()
+                    .replace("}", ", userContext }");
+                destructuringParam.replaceWithText(updatedImport);
+                await fs_extra_1.default.writeFile(filePath, fileContent.replace(currentBlock, testNode.getText()), "utf-8");
             }
         }
     }

package/dist/browser-injected-scripts/annotate-elements.js

@@ -103,8 +103,8 @@ function annotateClickableElements(options = {}) {
   }
 
   // Check if an element is clickable
-  function isElementClickable(element) {
-    // if (!(element instanceof Element)) return false;
+  function isElementClickable(element, windowToAnnotate) {
+    if (!(element instanceof windowToAnnotate.Element)) return false;
 
     const tagName = element.tagName.toLowerCase();
     let isClickable = false;
@@ -380,7 +380,7 @@ function annotateClickableElements(options = {}) {
   
     // Filter for clickable elements
     const clickableElements = Array.from(windowToAnnotate.document.querySelectorAll("*")).filter((el) => {
-      const isClickable = isElementClickable(el);
+      const isClickable = isElementClickable(el, windowToAnnotate);
       const isClickNotBlocked = isElementClickNotBlocked(el, windowToAnnotate);
       return isClickable && isClickNotBlocked;
     });  

package/dist/evals/update-scenario-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/update-scenario-agent.evals.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAUpC,QAAA,MAAM,+BAA+B,EAAE,UAiDtC,CAAC;AAEF,eAAe,+BAA+B,CAAC"}
+{"version":3,"file":"update-scenario-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/update-scenario-agent.evals.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAUpC,QAAA,MAAM,+BAA+B,EAAE,UAqDtC,CAAC;AAEF,eAAe,+BAA+B,CAAC"}

package/dist/evals/update-scenario-agent.evals.js

@@ -5,10 +5,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const js_levenshtein_1 = __importDefault(require("js-levenshtein"));
 const update_flow_1 = require("../agent/codegen/update-flow");
-const utils_1 = require("../agent/codegen/utils");
 const updateScenarioCodeAgentEvaluate = async ({ item, trace }) => {
     const { testCase, testFiles, pageFiles, testFilePath, testFileContent } = item.input;
-    const response = await (0, update_flow_1.getUpdateTestCodeCompletion)({
+    const fileChanges = await (0, update_flow_1.getUpdateTestCodeCompletion)({
         testCase,
         testFiles,
         pageFiles,
@@ -16,8 +15,7 @@ const updateScenarioCodeAgentEvaluate = async ({ item, trace }) => {
         testFileContent,
         trace,
     });
-    const fileChanges = (0, utils_1.extractTestUpdates)(response);
-    const expectedFileChanges = (0, utils_1.extractTestUpdates)(item.expectedOutput);
+    const expectedFileChanges = item.expectedOutput;
     const fileChangeCount = fileChanges.length;
     const expectedFileChangeCount = expectedFileChanges.length;
     const correctFilePathChanges = expectedFileChanges.every((ef) => fileChanges.some((f) => f.filePath === ef.filePath));
@@ -43,7 +41,7 @@ const updateScenarioCodeAgentEvaluate = async ({ item, trace }) => {
                 value: score,
             },
         ],
-        output: response,
+        output: fileChanges,
     };
 };
 exports.default = updateScenarioCodeAgentEvaluate;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.38.11",
+  "version": "0.38.15",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -55,10 +55,10 @@
     "remove-markdown": "^0.5.5",
     "sanitize-html": "^2.13.0",
     "slugify": "^1.6.6",
-    "ts-morph": "^23.0.0",
+    "ts-morph": "^24.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.25",
+    "@empiricalrun/llm": "^0.9.26",
     "@empiricalrun/r2-uploader": "^0.3.6",
     "@empiricalrun/reporter": "^0.21.3"
   },