@empiricalrun/test-gen 0.33.16 → 0.34.1

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.34.1
+
+### Patch Changes
+
+- Updated dependencies [25f3601]
+  - @empiricalrun/llm@0.9.16
+
+## 0.34.0
+
+### Minor Changes
+
+- b0c5c81: feat: expose hints workflow behind feature flag
+
 ## 0.33.16
 
 ### Patch Changes

package/dist/actions/assert.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"assert.d.ts","sourceRoot":"","sources":["../../src/actions/assert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBAsDjD,CAAC"}
+{"version":3,"file":"assert.d.ts","sourceRoot":"","sources":["../../src/actions/assert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBAuDjD,CAAC"}

package/dist/actions/assert.js

@@ -7,7 +7,7 @@ exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = "assert_text_visibility"
 const assertTextVisibilityActionGenerator = (page) => {
     return {
         execute: async ({ args }) => {
-            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(args.css_selector, args.xpath, page);
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(args.css_selector, args.xpath, page, args?.elementAnnotation);
             const exec = new Function("page", `return page.${locator}.isVisible({ timeout: 3000 })`);
             await exec(page);
             return {

package/dist/actions/click.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAsDlC,CAAC"}
+{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAuDlC,CAAC"}

package/dist/actions/click.js

@@ -8,7 +8,7 @@ const clickActionGenerator = (page) => {
     return {
         execute: async ({ args }) => {
             const selector = args.css_selector;
-            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(selector, args.xpath, page);
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(selector, args.xpath, page, args?.elementAnnotation);
             const exec = new Function("page", `return page.${locator}.click({ timeout: 3000 })`);
             await exec(page);
             await page.waitForTimeout(3000);

package/dist/actions/fill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAGhE,eAAO,MAAM,mBAAmB,EAAE,yBAiFjC,CAAC"}
+{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAGhE,eAAO,MAAM,mBAAmB,EAAE,yBAkFjC,CAAC"}

package/dist/actions/fill.js

@@ -11,7 +11,7 @@ const fillActionGenerator = (page, options) => {
     return {
         execute: async ({ args }) => {
             const css = args.css_selector;
-            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(css, args.xpath, page);
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(css, args.xpath, page, args?.elementAnnotation);
             const textToFill = options?.stateVariables[args.variable_name] || args.text;
             const exec = new Function("page", `return page.${locator}.fill("${textToFill}", { timeout: 3000 })`);
             await exec(page);
@@ -45,11 +45,11 @@ const fillActionGenerator = (page, options) => {
                         variable_name: {
                             type: "string",
                             enum: stateVariableNames,
-                            description: `variable name whose value needs to be filled in the input element. 
-                The variable name and values are: 
+                            description: `variable name whose value needs to be filled in the input element.
+                The variable name and values are:
 
                 ${JSON.stringify(stateVariableNames, null, 2)}
-                
+
                 - Choose the variable name based on the value above.
                 - Set variable_name as 'NA' if there is no variable to be used.`,
                         },

package/dist/actions/hover.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hover.d.ts","sourceRoot":"","sources":["../../src/actions/hover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAoDlC,CAAC"}
+{"version":3,"file":"hover.d.ts","sourceRoot":"","sources":["../../src/actions/hover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBAqDlC,CAAC"}

package/dist/actions/hover.js

@@ -8,7 +8,7 @@ const hoverActionGenerator = (page) => {
     return {
         execute: async ({ args }) => {
             const selector = args.css_selector;
-            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(selector, args.xpath, page);
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(selector, args.xpath, page, args?.elementAnnotation);
             const exec = new Function("page", `return page.${locator}.hover({ timeout: 3000 })`);
             await exec(page);
             await page.waitForTimeout(3000);

package/dist/actions/next-task.d.ts

@@ -17,6 +17,10 @@ export declare const NextTaskAction: {
                         type: string;
                         description: string;
                     };
+                    elementAnnotation: {
+                        type: string;
+                        description: string;
+                    };
                 };
                 required: string[];
             };

package/dist/actions/next-task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"next-task.d.ts","sourceRoot":"","sources":["../../src/actions/next-task.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,SAAS,cAAc,CAAC;AAErC,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;CA2B1B,CAAC"}
+{"version":3,"file":"next-task.d.ts","sourceRoot":"","sources":["../../src/actions/next-task.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,SAAS,cAAc,CAAC;AAErC,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+B1B,CAAC"}

package/dist/actions/next-task.js

@@ -19,12 +19,16 @@ exports.NextTaskAction = {
                     action: {
                         type: "string",
                         description: `explain the next action in natural language.
-The next action should be as atomic as possible, precise and should contain enough details about the action to be performed. 
-E.g. each click, key press, input, assert should be a separate action. 
+The next action should be as atomic as possible, precise and should contain enough details about the action to be performed.
+E.g. each click, key press, input, assert should be a separate action.
 Each action should take the task to completion, if not the action is invalid.`,
                     },
+                    elementAnnotation: {
+                        type: "string",
+                        description: `look at the annotated screenshot and identify the element annotation (the one in the yellow box) on which next task needs to be performed`,
+                    },
                 },
-                required: ["reason", "action"],
+                required: ["reason", "action", "elementAnnotation"],
             },
         },
     },

package/dist/actions/text-content.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"text-content.d.ts","sourceRoot":"","sources":["../../src/actions/text-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,iBAAiB,CAAC;AAE1D,eAAO,MAAM,0BAA0B,EAAE,yBA6DxC,CAAC"}
+{"version":3,"file":"text-content.d.ts","sourceRoot":"","sources":["../../src/actions/text-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,iBAAiB,CAAC;AAE1D,eAAO,MAAM,0BAA0B,EAAE,yBA8DxC,CAAC"}

package/dist/actions/text-content.js

@@ -8,7 +8,7 @@ const textContentActionGenerator = (page, options) => {
     return {
         execute: async (args) => {
             const css = args.css_selector;
-            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(css, args.xpath, page);
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(css, args.xpath, page, args?.elementAnnotation);
             const exec = new Function("page", `return page.${locator}.textContent()`);
             const value = (await exec(page));
             if (options) {

package/dist/actions/utils/index.d.ts

@@ -5,6 +5,6 @@ declare global {
         jQuery: any;
     }
 }
-export declare function getPlaywrightLocatorUsingCssSelector(cssSelector: string, xpath: string, page: Page): Promise<any>;
+export declare function getPlaywrightLocatorUsingCssSelector(cssSelector: string, xpath: string, page: Page, elementAnnotation?: string): Promise<any>;
 export declare function getPageVarName(): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/actions/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,IAAI,gBAmEX;AAED,wBAAgB,cAAc,WAG7B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,IAAI,EACV,iBAAiB,CAAC,EAAE,MAAM,gBAkF3B;AAED,wBAAgB,cAAc,WAG7B"}

package/dist/actions/utils/index.js

@@ -1,15 +1,23 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getPageVarName = exports.getPlaywrightLocatorUsingCssSelector = void 0;
-async function getPlaywrightLocatorUsingCssSelector(cssSelector, xpath, page) {
+async function getPlaywrightLocatorUsingCssSelector(cssSelector, xpath, page, elementAnnotation) {
     // TODO: analyse other solutions than just css. Also find other solutions to support :has-text -> prompting ?
     // jquery doesnt support :has-text. neither css. Only playwright locator supports this selector though.
     const sanitizedCssSelectorForJQuery = cssSelector
-        .replaceAll(":has-text", ":contains")
-        .replaceAll(":text", ":contains");
+        ?.replaceAll(":has-text", ":contains")
+        ?.replaceAll(":text", ":contains");
     return await page.evaluate((locator) => {
+        const elementAnnotation = locator?.elementAnnotation;
         let selectedElem;
-        if (locator.xpath) {
+        if (elementAnnotation &&
+            // @ts-ignore
+            window?.annotationInstance?.annotations?.[elementAnnotation]) {
+            selectedElem =
+                // @ts-ignore
+                window?.annotationInstance?.annotations?.[elementAnnotation]?.node;
+        }
+        else if (locator.xpath) {
             try {
                 const xpath = locator.xpath.replaceAll("text()", "normalize-space()");
                 const result = document.evaluate(xpath, document, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
@@ -59,7 +67,12 @@ async function getPlaywrightLocatorUsingCssSelector(cssSelector, xpath, page) {
             throw Error(`Unable to find element, css: ${locator.css}`);
         }
         return window.playwright.generateLocator(selectedElem);
-    }, { css: cssSelector, cssForJq: sanitizedCssSelectorForJQuery, xpath: xpath });
+    }, {
+        css: cssSelector,
+        cssForJq: sanitizedCssSelectorForJQuery,
+        xpath: xpath,
+        elementAnnotation,
+    });
 }
 exports.getPlaywrightLocatorUsingCssSelector = getPlaywrightLocatorUsingCssSelector;
 function getPageVarName() {

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAgBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAKzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAgBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAKzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAkBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -149,6 +149,7 @@ async function injectPwLocatorGenerator(page) {
     const scriptResps = await Promise.all([
         "https://assets-test.empirical.run/pw-selector.js",
         "https://code.jquery.com/jquery-3.7.1.min.js",
+        "https://assets-test.empirical.run/annotate-elements.js",
     ].map((url) => fetch(url)));
     const scripts = await Promise.all(scriptResps.map((s) => s.text()));
     page.on("load", async () => {

package/dist/agent/master/run.d.ts

@@ -3,7 +3,7 @@ import { Page } from "playwright";
 import { PlaywrightActions } from "../../actions";
 import { TestCase } from "../../types";
 import { BrowsingAgentOptions } from "../browsing";
-export declare function getNextAction({ task, executedActions, failedActions, page, trace, llm, options, pageScreenshot, actions, disableSkills, }: {
+export declare function getNextAction({ task, executedActions, failedActions, page, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints, }: {
     task: string;
     executedActions: string[];
     failedActions: any[];
@@ -12,8 +12,10 @@ export declare function getNextAction({ task, executedActions, failedActions, pa
     llm: LLM;
     options: BrowsingAgentOptions;
     pageScreenshot: string;
+    annotatedPageScreenshot?: string;
     actions: PlaywrightActions;
     disableSkills: boolean;
+    useHints: boolean;
 }): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;
 export declare function createTestUsingMasterAgent({ task, page, testCase, options, }: {
     task: string;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAOrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,aAAa,GACd,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;CACxB,2FAwDA;AAED,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GAkMA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAgEA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GA8QA"}

package/dist/agent/master/run.js

@@ -14,30 +14,39 @@ const browsing_1 = require("../browsing");
 const utils_1 = require("../browsing/utils");
 const skills_retriever_1 = require("../codegen/skills-retriever");
 const verification_1 = require("../verification");
+const with_hints_1 = require("./with-hints");
 const MAX_ERROR_COUNT = 2;
-async function getNextAction({ task, executedActions, failedActions, page, trace, llm, options, pageScreenshot, actions, disableSkills, }) {
+async function getNextAction({ task, executedActions, failedActions, page, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints = false, }) {
     const promptSpan = trace?.span({ name: "master-agent-prompt" });
     const promptMessages = await (0, llm_1.getPrompt)("test-gen", {
         task,
         failedActions: failedActions.map((a) => a).join("\n"),
         executedActions: executedActions.map((a) => a).join("\n"),
         pageUrl: page.url(),
-    }, 14);
+    }, useHints ? 16 : 14);
     // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
     const userMessage = promptMessages.filter((m) => m.role === "user")[0];
     const systemMessage = promptMessages.filter((m) => m.role === "system")[0];
-    userMessage.content = [
-        {
-            type: "text",
-            text: userMessage.content,
-        },
-        {
-            type: "image_url",
-            image_url: {
-                url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
-            },
-        },
-    ];
+    userMessage.content =
+        useHints && annotatedPageScreenshot
+            ? (0, with_hints_1.getUserMessageWithForHints)({
+                userMessage: userMessage,
+                options,
+                pageScreenshot,
+                annotatedPageScreenshot,
+            })
+            : [
+                {
+                    type: "text",
+                    text: userMessage.content,
+                },
+                {
+                    type: "image_url",
+                    image_url: {
+                        url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
+                    },
+                },
+            ];
     const messages = [
         systemMessage,
         userMessage,
@@ -64,7 +73,9 @@ async function getNextAction({ task, executedActions, failedActions, page, trace
     return toolCall;
 }
 exports.getNextAction = getNextAction;
+// let screenshotIndex = 0; // keeping this for local debugging
 async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
+    const useHints = options?.useHints || false;
     const logger = new logger_1.CustomLogger({ useReporter: false });
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
     // add timeout for the page to settle in
@@ -132,11 +143,46 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
                 break;
             }
         }
-        const buffer = await page.screenshot({ fullPage: true });
+        const buffer = await page.screenshot({
+            fullPage: true,
+            // path: `screenshots/screenshot-${screenshotIndex++}.png`, // enable this and screenshotIndex var for local debugging
+        });
         const testGenReporter = new reporter_1.TestGenUpdatesReporter();
         const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
         const pageScreenshot = buffer.toString("base64");
         let output;
+        let annotations;
+        let testGenAnnotatedSnapshotUpdatePromise;
+        let annotatedPageScreenshot;
+        if (useHints) {
+            await page.waitForTimeout(2000);
+            const annotationResult = await page.evaluate(() => {
+                // @ts-ignore
+                window.annotationInstance = window.annotateClickableElements();
+                // @ts-ignore
+                return window.annotationInstance;
+            });
+            annotations = annotationResult?.annotations || {};
+            await page.waitForTimeout(2000);
+            const annonationBuffer = await page.screenshot({
+                fullPage: true,
+                // path: `screenshots/screenshot-${screenshotIndex++}.png`,
+            });
+            await page.evaluate(() => {
+                console.log({
+                    // @ts-ignore
+                    disable: window?.annotationInstance?.disable,
+                });
+                // @ts-ignore
+                if (window?.annotationInstance && window?.annotationInstance?.destroy) {
+                    // @ts-ignore
+                    window?.annotationInstance?.destroy();
+                }
+            });
+            testGenAnnotatedSnapshotUpdatePromise =
+                testGenReporter.sendCurrentView(annonationBuffer);
+            annotatedPageScreenshot = annonationBuffer.toString("base64");
+        }
         if (await (0, session_1.shouldStopSession)()) {
             break;
         }
@@ -149,8 +195,10 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
             llm,
             options,
             pageScreenshot,
+            annotatedPageScreenshot,
             actions,
             disableSkills,
+            useHints,
         });
         if (await (0, session_1.shouldStopSession)()) {
             break;
@@ -165,12 +213,36 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
                     action: JSON.parse(toolCall.function.arguments).action ||
                         JSON.parse(toolCall.function.arguments).skill,
                     reason: JSON.parse(toolCall.function.arguments).reason,
+                    elementAnnotation: JSON.parse(toolCall.function.arguments)
+                        ?.elementAnnotation,
                 };
+                let shouldTriggerHintsFlow;
+                let hintsExecutionCompletion;
+                if (useHints) {
+                    const result = await (0, with_hints_1.triggerHintsFlow)({
+                        outputFromGetNextAction: output,
+                        generatedAnnotations: annotations,
+                        page,
+                        llm,
+                    });
+                    shouldTriggerHintsFlow = result.shouldTriggerHintsFlow;
+                    hintsExecutionCompletion = result.hintsExecutionCompletion;
+                }
                 void testGenReporter.sendMessage(output.reason);
                 logger.log(`Next Action: ${output.action}`);
                 if (toolCall.function.name === skill_1.SKILL_USAGE) {
                     await actions.executeAction(toolCall.function.name, args, masterAgentActionSpan);
                 }
+                else if (shouldTriggerHintsFlow && hintsExecutionCompletion) {
+                    const toolCalls = hintsExecutionCompletion?.tool_calls || [];
+                    for (const i in toolCalls) {
+                        const currentToolCall = toolCalls[i];
+                        await actions.executeAction(currentToolCall.function.name, {
+                            ...JSON.parse(currentToolCall.function.arguments),
+                            ...args,
+                        });
+                    }
+                }
                 else {
                     await (0, browsing_1.executeTaskUsingBrowsingAgent)({
                         trace: masterAgentActionSpan,
@@ -226,6 +298,9 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
         }
         trace.update({ input: { task }, output: { output } });
         await testGenSnapshotUpdatePromise;
+        if (testGenAnnotatedSnapshotUpdatePromise) {
+            await testGenAnnotatedSnapshotUpdatePromise;
+        }
         masterAgentSpan.end({
             output: { action: output.action, reason: output.reason },
         });

package/dist/agent/master/with-hints.d.ts

@@ -0,0 +1,23 @@
+import { LLM } from "@empiricalrun/llm";
+import OpenAI from "openai";
+import { Page } from "playwright";
+import { BrowsingAgentOptions } from "../browsing";
+export declare const getUserMessageWithForHints: ({ userMessage, options, pageScreenshot, annotatedPageScreenshot, }: {
+    userMessage: OpenAI.ChatCompletionUserMessageParam;
+    options: BrowsingAgentOptions;
+    pageScreenshot: string;
+    annotatedPageScreenshot: string;
+}) => string | OpenAI.ChatCompletionContentPart[];
+export declare const triggerHintsFlow: ({ outputFromGetNextAction, generatedAnnotations, page, llm, }: {
+    outputFromGetNextAction: {
+        action: string;
+        elementAnnotation?: string;
+    };
+    generatedAnnotations: Record<string, any>;
+    page: Page;
+    llm: LLM;
+}) => Promise<{
+    shouldTriggerHintsFlow: boolean;
+    hintsExecutionCompletion: OpenAI.ChatCompletionMessage | undefined;
+}>;
+//# sourceMappingURL=with-hints.d.ts.map

package/dist/agent/master/with-hints.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAExC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;aACzC,oBAAoB;oBACb,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAMF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,IAAI;SACL,GAAG;MACN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAgFA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.triggerHintsFlow = exports.getUserMessageWithForHints = void 0;
+const vision_1 = require("@empiricalrun/llm/vision");
+const actions_1 = require("../../actions");
+const constants_1 = require("../../constants");
+const getUserMessageWithForHints = ({ userMessage, options, pageScreenshot, annotatedPageScreenshot, }) => {
+    return [
+        {
+            type: "text",
+            text: userMessage.content,
+        },
+        {
+            type: "text",
+            text: "Screenshot in normal mode 👇",
+        },
+        {
+            type: "image_url",
+            image_url: {
+                url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
+            },
+        },
+        {
+            type: "text",
+            text: "Screenshot in annotated mode 👇",
+        },
+        {
+            type: "image_url",
+            image_url: {
+                url: (0, vision_1.imageFormatForProvider)(options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, annotatedPageScreenshot),
+            },
+        },
+    ];
+};
+exports.getUserMessageWithForHints = getUserMessageWithForHints;
+const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations, page, llm, }) => {
+    try {
+        const hasElementAnnotation = outputFromGetNextAction?.elementAnnotation?.length &&
+            outputFromGetNextAction?.elementAnnotation?.trim()?.length &&
+            outputFromGetNextAction?.elementAnnotation in
+                (generatedAnnotations || {});
+        if (!hasElementAnnotation) {
+            return {
+                shouldTriggerHintsFlow: false,
+                hintsExecutionCompletion: undefined,
+            };
+        }
+        const actions = new actions_1.PlaywrightActions(page);
+        const completion = await llm.createChatCompletion({
+            messages: [
+                {
+                    role: "system",
+                    content: `You are a browser automation agent who is given a task to generate code for navigation and assertion. This task is your goal and you must achieve it.
+
+  You need to pick the next step to achieve the task. Remember that the goal must be achieved.
+
+  Based on the goal and available tool calls you need to pick the appropriate tool call.
+
+
+  Instructions:
+  - Take actions one at a time. Do not try to take multiple actions
+  - You can respond with multiple assertions in one shot
+  - Stick to the task provided to you and mark the task done once the task is complete
+  - Do not execute any action which is not mentioned in the task`,
+                },
+                {
+                    role: "user",
+                    content: [
+                        {
+                            type: "text",
+                            text: `Task: ${outputFromGetNextAction?.action}`,
+                        },
+                    ],
+                },
+            ],
+            tools: actions
+                .getBrowsingActionSchemas()
+                .filter((action) => action.function.name !== "skill_usage"),
+            model: "gpt-4o-2024-08-06",
+            modelParameters: {
+                temperature: 0.5,
+                max_completion_tokens: 4000,
+                tool_choice: "required",
+            },
+        });
+        const canTriggerHintsFlow = completion?.tool_calls?.some((currentToolCall) => currentToolCall.function.name === "fill_input_element" ||
+            currentToolCall.function.name === "click_element");
+        if (!canTriggerHintsFlow) {
+            return {
+                shouldTriggerHintsFlow: false,
+                hintsExecutionCompletion: undefined,
+            };
+        }
+        return {
+            shouldTriggerHintsFlow: true,
+            // @ts-ignore
+            hintsExecutionCompletion: completion,
+        };
+    }
+    catch (e) {
+        console.error("Error in triggerHintsFlow", e);
+        return {
+            shouldTriggerHintsFlow: false,
+            hintsExecutionCompletion: undefined,
+        };
+    }
+};
+exports.triggerHintsFlow = triggerHintsFlow;

package/dist/types/index.d.ts

@@ -16,6 +16,7 @@ export type TestGenConfigOptions = {
     model: LLMModel;
     modelProvider: LLMProvider;
     modelParameters?: ModelParameters;
+    useHints?: boolean;
     metadata: {
         testSessionId: number;
         generationId: number;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/C,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACrC,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KACzB;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/C,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACrC,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KACzB;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.33.16",
+  "version": "0.34.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -44,7 +44,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.15",
+    "@empiricalrun/llm": "^0.9.16",
     "@empiricalrun/r2-uploader": "^0.3.6",
     "@empiricalrun/reporter": "^0.21.2"
   },