@empiricalrun/test-gen 0.38.1 → 0.38.4

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @empiricalrun/test-gen
 
+## 0.38.4
+
+### Patch Changes
+
+- be5370d: chore: send skill usage message on chat
+
+## 0.38.3
+
+### Patch Changes
+
+- ff76b50: fix: return first visible element matching with the CSS selector
+
+## 0.38.2
+
+### Patch Changes
+
+- 132c2b4: fix: generation assets failure shouldnt stop test generation
+- 213c357: fix: added available annotations in master agent prompt
+
 ## 0.38.1
 
 ### Patch Changes

package/dist/actions/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,IAAI,EACV,iBAAiB,CAAC,EAAE,MAAM,gBAkF3B;AAED,wBAAgB,cAAc,WAG7B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,MAAM;QACd,UAAU,EAAE,GAAG,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC;KACb;CACF;AAED,wBAAsB,oCAAoC,CACxD,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,IAAI,EACV,iBAAiB,CAAC,EAAE,MAAM,gBAoF3B;AAED,wBAAgB,cAAc,WAG7B"}

package/dist/actions/utils/index.js

@@ -52,10 +52,12 @@ async function getPlaywrightLocatorUsingCssSelector(cssSelector, xpath, page, el
         if (!selectedElem) {
             const elements = window.jQuery(locator.cssForJq);
             let elIdx = 0;
-            Array.from(elements).forEach((el, i) => {
+            Array.from(elements).some((el, i) => {
                 if (window.jQuery(el).is(":visible")) {
                     elIdx = i;
+                    return true;
                 }
+                return false;
             });
             //@ts-ignore
             selectedElem = elements[elIdx];

package/dist/agent/codegen/use-skill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"use-skill.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/use-skill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAShE,wBAAsB,sBAAsB,CAAC,EAC3C,IAAI,EACJ,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,mBA0CA"}
+{"version":3,"file":"use-skill.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/use-skill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAUhE,wBAAsB,sBAAsB,CAAC,EAC3C,IAAI,EACJ,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,mBA4CA"}

package/dist/agent/codegen/use-skill.js

@@ -2,8 +2,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateSkillUsageCode = void 0;
 const llm_1 = require("@empiricalrun/llm");
+const logger_1 = require("../../bin/logger");
 const constants_1 = require("../../constants");
 async function generateSkillUsageCode({ task, sampleUsageMethod, scopeVariablesMapStr, pageVariableName, trace, }) {
+    const logger = new logger_1.CustomLogger();
+    logger.log(`Generating code using skill usage example: ${sampleUsageMethod}`);
     const skillUsageSpan = trace?.span({
         name: "skill-usage",
         input: {

package/dist/agent/master/run.d.ts

@@ -3,7 +3,7 @@ import { Page } from "playwright";
 import { PlaywrightActions } from "../../actions";
 import { ScopeVars, TestCase } from "../../types";
 import { BrowsingAgentOptions } from "../browsing";
-export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints, }: {
+export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints, annotations, }: {
     task: string;
     executedActions: string[];
     failedActions: any[];
@@ -16,6 +16,7 @@ export declare function getNextAction({ task, executedActions, failedActions, pa
     actions: PlaywrightActions;
     disableSkills: boolean;
     useHints: boolean;
+    annotations?: string[];
 }): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;
 export declare function createTestUsingMasterAgent({ task, page, testCase, options, scopeVars, }: {
     task: string;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAwFA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA0TA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,EAChB,WAAW,GACZ,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB,2FA2FA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAuTA"}

package/dist/agent/master/run.js

@@ -16,7 +16,7 @@ const skills_retriever_1 = require("../codegen/skills-retriever");
 const verification_1 = require("../verification");
 const with_hints_1 = require("./with-hints");
 const MAX_ERROR_COUNT = 2;
-async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints = false, }) {
+async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, annotatedPageScreenshot, actions, disableSkills, useHints = false, annotations, }) {
     const nextActionSpan = trace?.span({
         name: "master-agent-next-action",
         input: {
@@ -30,6 +30,7 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
             disableSkills,
             useHints,
             skills: skill_1.testCaseSkills.getAvailableSkills(),
+            annotations,
         },
     });
     const promptSpan = nextActionSpan?.span({
@@ -40,7 +41,8 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
         failedActions: failedActions.map((a) => a).join("\n"),
         executedActions: executedActions.map((a) => a).join("\n"),
         pageUrl,
-    }, 20);
+        annotations,
+    }, 24);
     // assuming there is only one user message in the prompt. if there is a change in langfuse prompt format, this will need to be updated
     const userMessage = promptMessages.filter((m) => m.role === "user")[0];
     const systemMessage = promptMessages.filter((m) => m.role === "system")[0];
@@ -176,12 +178,11 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
         // path: `screenshots/screenshot-${screenshotIndex++}.png`, // enable this and screenshotIndex var for local debugging
         });
         const testGenReporter = new reporter_1.TestGenUpdatesReporter();
-        const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
+        await testGenReporter.sendCurrentView(buffer);
         const pageScreenshot = buffer.toString("base64");
         let output;
         let generatedCodeSteps = [];
         let annotations;
-        let testGenAnnotatedSnapshotUpdatePromise;
         let annotatedPageScreenshot;
         if (useHints) {
             await page.waitForTimeout(2000);
@@ -209,13 +210,13 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
                     window?.annotationInstance?.destroy();
                 }
             });
-            testGenAnnotatedSnapshotUpdatePromise =
-                testGenReporter.sendCurrentView(annonationBuffer);
+            await testGenReporter.sendCurrentView(annonationBuffer);
             annotatedPageScreenshot = annonationBuffer.toString("base64");
         }
         if (await (0, session_1.shouldStopSession)()) {
             break;
         }
+        const annotationKeys = annotations ? Object.keys(annotations) : [];
         const toolCall = await getNextAction({
             task,
             executedActions: masterAgentActions,
@@ -229,6 +230,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
             actions,
             disableSkills,
             useHints,
+            annotations: annotationKeys,
         });
         if (await (0, session_1.shouldStopSession)()) {
             break;
@@ -347,10 +349,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
             }
         }
         trace?.update({ input: { task }, output: { output } });
-        await testGenSnapshotUpdatePromise;
-        if (testGenAnnotatedSnapshotUpdatePromise) {
-            await testGenAnnotatedSnapshotUpdatePromise;
-        }
         masterAgentSpan?.end({
             output: {
                 action: output.action,

package/dist/evals/master-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA0C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
+{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA4C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/master-agent.evals.js

@@ -5,7 +5,7 @@ const actions_1 = require("../actions");
 const skill_1 = require("../actions/skill");
 const run_1 = require("../agent/master/run");
 const masterGetNextActionEvaluator = async ({ item, trace, }) => {
-    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, annotatedPageScreenshot, disableSkills, useHints, skills = [], } = item.input;
+    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, annotatedPageScreenshot, disableSkills, useHints, skills = [], annotations, } = item.input;
     const page = {};
     skill_1.testCaseSkills.updateSkills(skills);
     const actions = new actions_1.PlaywrightActions(page);
@@ -21,6 +21,7 @@ const masterGetNextActionEvaluator = async ({ item, trace, }) => {
         actions,
         disableSkills,
         useHints,
+        annotations,
     });
     return {
         scores: [

package/dist/reporter/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAa5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAgDK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3C,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1C,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAWxD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAa5E,KAAK,kBAAkB,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAKF,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAUlD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAGlE;AAED,qBAAa,sBAAsB;;IAE3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1C,eAAe,CAAC,EACpB,eAAe,EACf,QAAQ,GACT,EAAE;QACD,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAgDK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3C,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1C,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAWxD"}

package/dist/reporter/index.js

@@ -95,25 +95,30 @@ class TestGenUpdatesReporter {
             logger.log("Skipped uploading current view screenshot");
             return;
         }
-        // upload current screenshot to r2 and report it to reporter
-        if (!fs_extra_1.default.existsSync(path_1.default.join(process.cwd(), "gen-assets"))) {
-            await fs_extra_1.default.mkdir((process.cwd(), "gen-assets"));
+        try {
+            // upload current screenshot to r2 and report it to reporter
+            if (!fs_extra_1.default.existsSync(path_1.default.join(process.cwd(), "gen-assets"))) {
+                await fs_extra_1.default.mkdir((process.cwd(), "gen-assets"));
+            }
+            await fs_extra_1.default.writeFile(path_1.default.join(process.cwd(), "gen-assets", `current-view-${Date.now()}.png`), buffer);
+            const uploadDir = (0, uploader_1.getUploadPathForRun)(reporterConfig?.projectRepoName);
+            const files = await (0, r2_uploader_1.uploadDirectory)({
+                sourceDir: path_1.default.join(process.cwd(), "gen-assets"),
+                destinationDir: uploadDir,
+                uploadBucket: uploader_1.UPLOAD_BUCKET,
+            });
+            const filePath = Object.keys(files)[0];
+            const relativeFilePath = filePath.replace(path_1.default.join(process.cwd(), "gen-assets"), "");
+            const url = `${uploader_1.UPLOAD_DOMAIN}/${uploadDir}${relativeFilePath}`;
+            await getReporter()?.report(new reporter_1.ProcessLogMessageBuilder({
+                type: "current-snapshot",
+                message: JSON.stringify({ type: "current-view", url }),
+            }));
+            await fs_extra_1.default.rmdir((process.cwd(), "gen-assets"), { recursive: true });
+        }
+        catch (e) {
+            console.warn("Failed to upload current view screenshot", e);
         }
-        await fs_extra_1.default.writeFile(path_1.default.join(process.cwd(), "gen-assets", `current-view-${Date.now()}.png`), buffer);
-        const uploadDir = (0, uploader_1.getUploadPathForRun)(reporterConfig?.projectRepoName);
-        const files = await (0, r2_uploader_1.uploadDirectory)({
-            sourceDir: path_1.default.join(process.cwd(), "gen-assets"),
-            destinationDir: uploadDir,
-            uploadBucket: uploader_1.UPLOAD_BUCKET,
-        });
-        const filePath = Object.keys(files)[0];
-        const relativeFilePath = filePath.replace(path_1.default.join(process.cwd(), "gen-assets"), "");
-        const url = `${uploader_1.UPLOAD_DOMAIN}/${uploadDir}${relativeFilePath}`;
-        await getReporter()?.report(new reporter_1.ProcessLogMessageBuilder({
-            type: "current-snapshot",
-            message: JSON.stringify({ type: "current-view", url }),
-        }));
-        await fs_extra_1.default.rmdir((process.cwd(), "gen-assets"), { recursive: true });
     }
     async sendMessage(message) {
         const reporter = getReporter();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.38.1",
+  "version": "0.38.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"