@empiricalrun/test-gen 0.36.5 → 0.37.1

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # @empiricalrun/test-gen
 
+## 0.37.1
+
+### Patch Changes
+
+- 3526169: fix: updated prompt for master agent next step
+
+## 0.37.0
+
+### Minor Changes
+
+- 97e5659: feat: add append create test agent evals
+
+### Patch Changes
+
+- 2dfecf6: fix: infer agent before enrichment call
+- 5b4509b: updated annotation hints for vue
+
+## 0.36.6
+
+### Patch Changes
+
+- 63f1398: fix: triggerHintsFlow trace span nesting
+- Updated dependencies [42b4490]
+  - @empiricalrun/llm@0.9.24
+
 ## 0.36.5
 
 ### Patch Changes

package/browser-injected-scripts/annotate-elements.js

@@ -290,6 +290,9 @@ window.annotateClickableElements = function annotateClickableElements(
         isClickable = true;
       } else if (element.classList.contains("cursor-pointer")) {
         isClickable = true;
+      } else if (element.classList.contains("v-list-item--link")) {
+        // vue specific click handling
+        isClickable = true;
       } else if (element.style.cursor === "pointer") {
         isClickable = true;
       }

package/browser-injected-scripts/annotate-elements.spec.ts

@@ -18,7 +18,6 @@ test("should annotate all links on empirical landing page", async ({
     const { annotations } = window.annotateClickableElements();
 
     return Object.entries(annotations).map(([hint, config]) => ({
-      hint,
       innerText: config.node.innerText,
       tagName: config.node.tagName,
       href: config.node.href,
@@ -27,49 +26,41 @@ test("should annotate all links on empirical landing page", async ({
 
   test.expect(annotations).toEqual([
     {
-      hint: "A",
       innerText: "Empirical",
       tagName: "A",
       href: "https://assets-test.empirical.run/",
     },
     {
-      hint: "B",
       innerText: "Blog",
       tagName: "A",
       href: "https://assets-test.empirical.run/blog",
     },
     {
-      hint: "C",
       innerText: "Contact us",
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
     {
-      hint: "D",
       href: "https://dash.empirical.run/",
       innerText: "Login ↗\n(opens in a new tab)",
       tagName: "A",
     },
     {
-      hint: "E",
       innerText: "Get early access",
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
     {
-      hint: "F",
       innerText: "Playwright\n(opens in a new tab)",
       tagName: "A",
       href: "https://github.com/microsoft/playwright",
     },
     {
-      hint: "G",
       innerText: "Meet with us",
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
     {
-      hint: "H",
       innerText: "Privacy Policy",
       tagName: "A",
       href: "https://assets-test.empirical.run/privacy.html",

package/dist/actions/next-task.d.ts

@@ -17,6 +17,29 @@ export declare const NextTaskAction: {
                         type: string;
                         description: string;
                     };
+                    listOfAnnotations: {
+                        type: string;
+                        items: {
+                            type: string;
+                            properties: {
+                                elementID: {
+                                    type: string;
+                                };
+                                text: {
+                                    type: string;
+                                };
+                            };
+                            required: string[];
+                        };
+                        description: string;
+                    };
+                    unannotatedElements: {
+                        type: string;
+                        items: {
+                            type: string;
+                        };
+                        description: string;
+                    };
                     elementAnnotation: {
                         type: string;
                         description: string;

package/dist/actions/next-task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"next-task.d.ts","sourceRoot":"","sources":["../../src/actions/next-task.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,SAAS,cAAc,CAAC;AAErC,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+B1B,CAAC"}
+{"version":3,"file":"next-task.d.ts","sourceRoot":"","sources":["../../src/actions/next-task.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,SAAS,cAAc,CAAC;AAErC,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyD1B,CAAC"}

package/dist/actions/next-task.js

@@ -23,9 +23,32 @@ The next action should be as atomic as possible, precise and should contain enou
 E.g. each click, key press, input, assert should be a separate action.
 Each action should take the task to completion, if not the action is invalid.`,
                     },
+                    listOfAnnotations: {
+                        type: "array",
+                        items: {
+                            type: "object",
+                            properties: {
+                                elementID: {
+                                    type: "string",
+                                },
+                                text: {
+                                    type: "string",
+                                },
+                            },
+                            required: ["elementId", "text"],
+                        },
+                        description: "A list which contains mapping of all the unique element IDs within the yellow box on the screen to their corresponding texts/icons. If an element is an icon, return 'Icon: <description of icon>'.",
+                    },
+                    unannotatedElements: {
+                        type: "array",
+                        items: {
+                            type: "string",
+                        },
+                        description: "A list of text/icon of elements which are unannotated, clickable and are present on the screen. An element is annotated if there is a red box drawn around it. If an element is an icon, return 'Icon: <description of icon>'.",
+                    },
                     elementAnnotation: {
                         type: "string",
-                        description: `look at the annotated screenshot and identify the element annotation (the one in the yellow box) on which next task needs to be performed`,
+                        description: "If the element is present in listOfAnnotations, return the element ID; otherwise, return empty string ''.",
                     },
                 },
                 required: ["reason", "action", "elementAnnotation"],

package/dist/agent/codegen/update-flow.d.ts

@@ -13,6 +13,14 @@ export declare function getUpdateTestCodeCompletion({ testCase, testFileContent,
     options?: TestGenConfigOptions;
 }): Promise<string>;
 export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean, validate?: boolean, trace?: TraceClient): Promise<UpdatedTestCase[]>;
+export declare function getAppendCreateTestBlockCompletion({ testFiles, pageFiles, testCase, testFilePath, options, trace, }: {
+    trace?: TraceClient;
+    pageFiles?: string;
+    testFiles: string;
+    testCase: TestCase;
+    options?: TestGenConfigOptions;
+    testFilePath: string;
+}): Promise<string>;
 export declare function appendCreateTestBlock({ testCase, file, options, trace, validateTypes, }: {
     testCase: TestCase;
     file: string;

package/dist/agent/codegen/update-flow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAsB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC,MAAM,CAAC,CA6ClB;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA6D5B;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA+E7B"}
+{"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAsB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC,MAAM,CAAC,CA6ClB;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA6D5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBA4DA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAyC7B"}

package/dist/agent/codegen/update-flow.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.appendCreateTestBlock = exports.updateTest = exports.getUpdateTestCodeCompletion = void 0;
+exports.appendCreateTestBlock = exports.getAppendCreateTestBlockCompletion = exports.updateTest = exports.getUpdateTestCodeCompletion = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const crypto_1 = __importDefault(require("crypto"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
@@ -204,20 +204,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
     return generatedTestCases;
 }
 exports.updateTest = updateTest;
-async function appendCreateTestBlock({ testCase, file, options, trace, validateTypes = true, }) {
-    const logger = new logger_1.CustomLogger({ useReporter: false });
-    logger.log("Appending create test block");
-    const context = await (0, context_1.contextForGeneration)(file);
-    const scenarioFileContent = await fs_extra_1.default.readFile(file, "utf-8");
-    const testCodePrompt = (0, fs_1.convertFileContentsToString)([
-        {
-            filePath: file,
-            content: scenarioFileContent,
-        },
-    ]);
-    const { codePrompt, pomPrompt } = context;
-    const generatedTestCases = [];
-    // TODO: move this to a common place
+async function getAppendCreateTestBlockCompletion({ testFiles, pageFiles, testCase, testFilePath, options, trace, }) {
     const session = (0, session_1.getSessionDetails)();
     trace =
         trace ||
@@ -230,17 +217,26 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
                     options?.metadata.environment || "",
                 ].filter((s) => !!s),
             });
+    const appendCreateTestSpan = trace?.span({
+        name: "append-create-test-block",
+        input: {
+            testCase,
+            pageFiles,
+            testFiles,
+            testFilePath,
+        },
+    });
     const promptName = "append-create-test-block";
     const promptSpan = trace?.span({
         name: "append-create-test-block-prompt",
     });
     const instruction = await (0, llm_1.getPrompt)(promptName, {
-        testFiles: testCodePrompt,
-        pageFiles: pomPrompt,
+        testFiles: testFiles,
+        pageFiles: pageFiles,
         scenarioName: testCase.name,
         scenarioSteps: testCase.steps.join("\n"),
-        scenarioFile: file,
-    });
+        scenarioFile: testFilePath,
+    }, 25);
     promptSpan?.end({ output: { instruction } });
     const [userInstruction] = instruction.filter((s) => s.role === "user");
     const [systemInstruction] = instruction.filter((s) => s.role === "system");
@@ -262,7 +258,32 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
         },
     });
     let response = firstShotMessage?.content || "";
-    const fileChanges = (0, utils_1.extractTestUpdates)(response);
+    appendCreateTestSpan?.end({ output: response });
+    return response;
+}
+exports.getAppendCreateTestBlockCompletion = getAppendCreateTestBlockCompletion;
+async function appendCreateTestBlock({ testCase, file, options, trace, validateTypes = true, }) {
+    const logger = new logger_1.CustomLogger({ useReporter: false });
+    logger.log("Appending create test block");
+    const context = await (0, context_1.contextForGeneration)(file);
+    const scenarioFileContent = await fs_extra_1.default.readFile(file, "utf-8");
+    const testCodePrompt = (0, fs_1.convertFileContentsToString)([
+        {
+            filePath: file,
+            content: scenarioFileContent,
+        },
+    ]);
+    const { codePrompt, pomPrompt } = context;
+    const generatedTestCases = [];
+    const appendCreateTestResp = await getAppendCreateTestBlockCompletion({
+        testCase,
+        testFilePath: file,
+        options,
+        trace,
+        pageFiles: pomPrompt,
+        testFiles: testCodePrompt,
+    });
+    const fileChanges = (0, utils_1.extractAppendTestUpdates)(appendCreateTestResp);
     await applyFileChanges({
         trace,
         testCase,
@@ -280,7 +301,6 @@ async function appendCreateTestBlock({ testCase, file, options, trace, validateT
         ...testCase,
         updatedFiles: fileChanges.map((f) => f.filePath),
     });
-    trace?.update({ input: { testCase }, output: { response } });
     await (0, llm_1.flushAllTraces)();
     return generatedTestCases;
 }

package/dist/agent/codegen/utils.d.ts

@@ -18,7 +18,26 @@ export declare function extractTestUpdates(input: string): {
 }[];
 /**
  *
+ * method to extract append create test updates
+ * @export
+ * @param {string} input
+ * @return {*}  {({
+ *   intent: string | undefined,
+ *   updateLocation: string | undefined,
+ *   filePath: string | undefined;
+ *   oldCode: string | undefined;
+ *   newCode: string | undefined;
+ *   reason: string | undefined;
+ * }[])}
  */
+export declare function extractAppendTestUpdates(input: string): {
+    intent: string | undefined;
+    updateLocation: string | undefined;
+    filePath: string | undefined;
+    oldCode: string | undefined;
+    newCode: string | undefined;
+    reason: string | undefined;
+}[];
 export declare function extractTestStepsSuggestions(input: string): {
     testStep: string;
     filePath: string;

package/dist/agent/codegen/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;GAEG;AAEH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,EAAE,CAeF"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,EAAE,CAeF"}

package/dist/agent/codegen/utils.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractTestStepsSuggestions = exports.extractTestUpdates = void 0;
+exports.extractTestStepsSuggestions = exports.extractAppendTestUpdates = exports.extractTestUpdates = void 0;
 /**
  *
  * method to extract file path and code updates for the LLM response of update flow
@@ -32,7 +32,38 @@ function extractTestUpdates(input) {
 exports.extractTestUpdates = extractTestUpdates;
 /**
  *
+ * method to extract append create test updates
+ * @export
+ * @param {string} input
+ * @return {*}  {({
+ *   intent: string | undefined,
+ *   updateLocation: string | undefined,
+ *   filePath: string | undefined;
+ *   oldCode: string | undefined;
+ *   newCode: string | undefined;
+ *   reason: string | undefined;
+ * }[])}
  */
+function extractAppendTestUpdates(input) {
+    const result = [];
+    // TODO: use better structure for this. Do not kill me for this please.
+    const regex = /<reason_for_intent>(.*?)<\/reason_for_intent>[\s\S]*?<intent>(.*?)<\/intent>[\s\S]*?<location_of_update>([\s\S]*?)<\/location_of_update>[\s\S]*?<file_path>([\s\S]*?)<\/file_path>[\s\S]*?<old_code_block>([\s\S]*?)<\/old_code_block>[\s\S]*?<new_code_block>([\s\S]*?)<\/new_code_block>[\s\S]*?<change>([\s\S]*?)<\/change>/g;
+    let match;
+    while ((match = regex.exec(input)) !== null) {
+        const [, reasonForIntent, intent, updateLocation, filePath, oldCode, newCode, reason,] = match;
+        result.push({
+            reasonForIntent: reasonForIntent?.trim(),
+            intent: intent?.trim(),
+            updateLocation: updateLocation?.trim(),
+            filePath: filePath?.trim(),
+            oldCode: oldCode?.trim(),
+            newCode: newCode?.trim(),
+            reason: reason?.trim(),
+        });
+    }
+    return result;
+}
+exports.extractAppendTestUpdates = extractAppendTestUpdates;
 function extractTestStepsSuggestions(input) {
     const result = [];
     const regex = /<subtask>(.*?)<\/subtask>[\s\S]*?<file_import_path>([\s\S]*?)<\/file_import_path>[\s\S]*?<usage_example>([\s\S]*?)<\/usage_example>[\s\S]*?<reason>([\s\S]*?)<\/reason>/g;

package/dist/agent/master/run.js

@@ -40,7 +40,7 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
         failedActions: failedActions.map((a) => a).join("\n"),
         executedActions: executedActions.map((a) => a).join("\n"),
         pageUrl,
-    }, 18);
+    }, 20);
     // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
     const userMessage = promptMessages.filter((m) => m.role === "user")[0];
     const systemMessage = promptMessages.filter((m) => m.role === "system")[0];
@@ -246,7 +246,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                 let shouldTriggerHintsFlow;
                 let hintsExecutionCompletion;
                 if (useHints) {
-                    const triggerHintsFlowSpan = trace?.span({
+                    const triggerHintsFlowSpan = masterAgentActionSpan?.span({
                         name: "trigger-hints-flow",
                         input: {
                             outputFromGetNextAction: output,

package/dist/browser-injected-scripts/annotate-elements.js

@@ -290,6 +290,9 @@ window.annotateClickableElements = function annotateClickableElements(
         isClickable = true;
       } else if (element.classList.contains("cursor-pointer")) {
         isClickable = true;
+      } else if (element.classList.contains("v-list-item--link")) {
+        // vue specific click handling
+        isClickable = true;
       } else if (element.style.cursor === "pointer") {
         isClickable = true;
       }

package/dist/browser-injected-scripts/annotate-elements.spec.ts

@@ -18,7 +18,6 @@ test("should annotate all links on empirical landing page", async ({
     const { annotations } = window.annotateClickableElements();
 
     return Object.entries(annotations).map(([hint, config]) => ({
-      hint,
       innerText: config.node.innerText,
       tagName: config.node.tagName,
       href: config.node.href,
@@ -27,49 +26,41 @@ test("should annotate all links on empirical landing page", async ({
 
   test.expect(annotations).toEqual([
     {
-      hint: "A",
       innerText: "Empirical",
       tagName: "A",
       href: "https://assets-test.empirical.run/",
     },
     {
-      hint: "B",
       innerText: "Blog",
       tagName: "A",
       href: "https://assets-test.empirical.run/blog",
     },
     {
-      hint: "C",
       innerText: "Contact us",
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
     {
-      hint: "D",
       href: "https://dash.empirical.run/",
       innerText: "Login ↗\n(opens in a new tab)",
       tagName: "A",
     },
     {
-      hint: "E",
       innerText: "Get early access",
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
     {
-      hint: "F",
       innerText: "Playwright\n(opens in a new tab)",
       tagName: "A",
       href: "https://github.com/microsoft/playwright",
     },
     {
-      hint: "G",
       innerText: "Meet with us",
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
     {
-      hint: "H",
       innerText: "Privacy Policy",
       tagName: "A",
       href: "https://assets-test.empirical.run/privacy.html",

package/dist/evals/append-create-test-agent.evals.d.ts

@@ -0,0 +1,4 @@
+import { EvaluateFn } from "./type";
+declare const appendCreateTestAgentEvaluate: EvaluateFn;
+export default appendCreateTestAgentEvaluate;
+//# sourceMappingURL=append-create-test-agent.evals.d.ts.map

package/dist/evals/append-create-test-agent.evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"append-create-test-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/append-create-test-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,QAAA,MAAM,6BAA6B,EAAE,UAapC,CAAC;AAEF,eAAe,6BAA6B,CAAC"}

package/dist/evals/append-create-test-agent.evals.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const update_flow_1 = require("../agent/codegen/update-flow");
+const appendCreateTestAgentEvaluate = async ({ item, trace }) => {
+    const { testCase, testFiles, pageFiles, testFilePath } = item.input;
+    const response = await (0, update_flow_1.getAppendCreateTestBlockCompletion)({
+        testCase,
+        testFiles,
+        pageFiles,
+        testFilePath,
+        trace,
+    });
+    return {
+        scores: [],
+        output: response,
+    };
+};
+exports.default = appendCreateTestAgentEvaluate;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.36.5",
+  "version": "0.37.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -9,6 +9,16 @@
     "@empiricalrun/test-gen": "dist/bin/index.js"
   },
   "main": "dist/index.js",
+  "exports": {
+    "./agent/infer-agent": {
+      "types": "./dist/agent/infer-agent/index.d.ts",
+      "default": "./dist/agent/infer-agent/index.js"
+    },
+    "./types": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./dist/types/index.js"
+    }
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/empirical-run/empirical.git"
@@ -44,7 +54,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.23",
+    "@empiricalrun/llm": "^0.9.24",
     "@empiricalrun/r2-uploader": "^0.3.6",
     "@empiricalrun/reporter": "^0.21.3"
   },