@empiricalrun/test-gen 0.31.9 → 0.31.12

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @empiricalrun/test-gen
 
+## 0.31.12
+
+### Patch Changes
+
+- a8f2ea6: chore: add test for master agent
+- Updated dependencies [10f554d]
+  - @empiricalrun/reporter@0.20.2
+
+## 0.31.11
+
+### Patch Changes
+
+- Updated dependencies [9623b74]
+  - @empiricalrun/reporter@0.20.1
+
+## 0.31.10
+
+### Patch Changes
+
+- c065064: fix: throw error for invalid response from o1
+
 ## 0.31.9
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAOnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAmP9B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAOnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAqP9B"}

package/dist/agent/browsing/index.js

@@ -19,6 +19,8 @@ const utils_1 = require("./utils");
 async function browsingAgentUsingMasterAgent(task, page, options) {
     const logger = new logger_1.CustomLogger({ useReporter: false });
     const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
+    // add timeout for the page to settle in
+    await page.waitForTimeout(3000);
     const trace = llm_1.langfuseInstance.trace({
         name: "test-generator",
         id: crypto_1.default.randomUUID(),

package/dist/agent/browsing/o1-completion.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"o1-completion.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/o1-completion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAKpC,wBAAsB,eAAe,CAAC,EACpC,QAAQ,EACR,KAAK,EACL,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAoD7C"}
+{"version":3,"file":"o1-completion.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/o1-completion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAKpC,wBAAsB,eAAe,CAAC,EACpC,QAAQ,EACR,KAAK,EACL,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAyD7C"}

package/dist/agent/browsing/o1-completion.js

@@ -41,6 +41,10 @@ async function getO1Completion({ messages, tools, trace, }) {
         o1Span?.end({ output: response });
         const toolResponseStr = (0, remove_markdown_1.default)(response.content);
         const toolRespJSON = JSON.parse(toolResponseStr);
+        const parameters = toolRespJSON.function.parameters || toolRespJSON.function.arguments;
+        if (!parameters) {
+            throw new Error("No parameters found in tool response");
+        }
         const tool = {
             type: "function",
             function: {

package/dist/agent/master/run.d.ts

@@ -2,6 +2,15 @@ import { LLM, TraceClient } from "@empiricalrun/llm";
 import { Page } from "playwright";
 import { TestGenConfigOptions } from "../../types";
 type BrowsingAgentOptions = Partial<TestGenConfigOptions>;
+export declare function getNextAction({ task, executedActions, page, trace, llm, options, pageScreenshot, }: {
+    task: string;
+    executedActions: string[];
+    page: Page;
+    trace?: TraceClient;
+    llm: LLM;
+    options: BrowsingAgentOptions;
+    pageScreenshot: string;
+}): Promise<import("openai/resources/index.mjs").ChatCompletionMessageToolCall | undefined>;
 export declare function masterAgent(task: string, page: Page, executedActions: string[], trace: TraceClient, llm: LLM, options: BrowsingAgentOptions): Promise<any>;
 export {};
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAE1D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,eAAe,EAAE,MAAM,EAAE,EACzB,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,oBAAoB,gBA0E9B"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AASlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAE1D,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,GACf,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB,2FA+CA;AAED,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,eAAe,EAAE,MAAM,EAAE,EACzB,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,oBAAoB,gBAsC9B"}

package/dist/agent/master/run.js

@@ -1,19 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.masterAgent = void 0;
+exports.masterAgent = exports.getNextAction = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
 const done_1 = require("../../actions/done");
 const next_task_1 = require("../../actions/next-task");
 const constants_1 = require("../../constants");
 const reporter_1 = require("../../reporter");
-async function masterAgent(task, page, executedActions, trace, llm, options) {
-    trace.update({ input: { task } });
-    const promptSpan = trace.span({ name: "page-prompt" });
-    const buffer = await page.screenshot({ fullPage: true });
-    const testGenReporter = new reporter_1.TestGenUpdatesReporter();
-    const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
-    const pageScreenshot = buffer.toString("base64");
+async function getNextAction({ task, executedActions, page, trace, llm, options, pageScreenshot, }) {
+    const promptSpan = trace?.span({ name: "page-prompt" });
     const promptMessages = await (0, llm_1.getPrompt)("test-gen", {
         task,
         executedActions: executedActions.map((a) => a).join("\n"),
@@ -39,7 +34,7 @@ async function masterAgent(task, page, executedActions, trace, llm, options) {
         userMessage,
     ];
     const tools = [next_task_1.NextTaskAction.schema, (0, done_1.doneActionGenerator)(page).schema];
-    promptSpan.end({ output: { messages } });
+    promptSpan?.end({ output: { messages } });
     const completion = await llm.createChatCompletion({
         messages,
         modelParameters: {
@@ -52,8 +47,26 @@ async function masterAgent(task, page, executedActions, trace, llm, options) {
         // @ts-ignore
         tools,
     });
-    let output;
     const toolCall = completion?.tool_calls?.[0];
+    return toolCall;
+}
+exports.getNextAction = getNextAction;
+async function masterAgent(task, page, executedActions, trace, llm, options) {
+    trace.update({ input: { task } });
+    const buffer = await page.screenshot({ fullPage: true });
+    const testGenReporter = new reporter_1.TestGenUpdatesReporter();
+    const testGenSnapshotUpdatePromise = testGenReporter.sendCurrentView(buffer);
+    const pageScreenshot = buffer.toString("base64");
+    let output;
+    const toolCall = await getNextAction({
+        task,
+        executedActions,
+        page,
+        trace,
+        llm,
+        options,
+        pageScreenshot,
+    });
     if (toolCall) {
         if (toolCall.function.name === "task_done") {
             output = {

package/dist/bin/index.js

@@ -69,10 +69,12 @@ async function runAgent(testGenConfig) {
         testGenFailed = true;
         new logger_1.CustomLogger().error(`Failed to generate test for the scenario. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
     }
-    await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
-        projectRepoName: testGenConfig.options.metadata.projectRepoName,
-        testName: testGenConfig.testCase.name,
-    });
+    if (testGenConfig.options?.agent !== "code") {
+        await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
+            projectRepoName: testGenConfig.options.metadata.projectRepoName,
+            testName: testGenConfig.testCase.name,
+        });
+    }
     // TODO: move these reporters to a better lifecycle
     await (0, ci_1.reportOnCI)(testGenConfig.testCase);
     await (0, llm_1.flushAllTraces)();

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAGL,IAAI,EAIL,MAAM,UAAU,CAAC;AAGlB;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAuCA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAmBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAGL,IAAI,EAIL,MAAM,UAAU,CAAC;AAGlB;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAuCA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC"}

package/dist/bin/utils/platform/web/index.js

@@ -268,7 +268,8 @@ async function getPageVariableNameFromCreateTest(filePath) {
     const descendentTexts = descendants.map((d) => d.getText());
     const parameterSeparatorIdx = descendentTexts.findIndex((s) => s === ",");
     const pageVariableName = descendentTexts[parameterSeparatorIdx + 1];
-    return pageVariableName;
+    // handle cases like "page as Page"
+    return pageVariableName?.split(" ")[0];
 }
 exports.getPageVariableNameFromCreateTest = getPageVariableNameFromCreateTest;
 function replaceCreateTestWithNewCode(filePath, contents, generatedCode) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.31.9",
+  "version": "0.31.12",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -46,7 +46,7 @@
     "typescript": "^5.3.3",
     "@empiricalrun/llm": "^0.9.3",
     "@empiricalrun/r2-uploader": "^0.3.3",
-    "@empiricalrun/reporter": "^0.20.0"
+    "@empiricalrun/reporter": "^0.20.2"
   },
   "devDependencies": {
     "@types/detect-port": "^1.3.5",