@empiricalrun/test-gen 0.21.1 → 0.21.4

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @empiricalrun/test-gen
 
+## 0.21.4
+
+### Patch Changes
+
+- Updated dependencies [09edb5f]
+  - @empiricalrun/reporter@0.14.3
+
+## 0.21.3
+
+### Patch Changes
+
+- c92362f: fix: verification agent hallucination
+
+## 0.21.2
+
+### Patch Changes
+
+- c3b31ae: fix: logic of extracting test block
+
 ## 0.21.1
 
 ### Patch Changes

package/dist/actions/assert.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"assert.d.ts","sourceRoot":"","sources":["../../src/actions/assert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBA8CjD,CAAC"}
+{"version":3,"file":"assert.d.ts","sourceRoot":"","sources":["../../src/actions/assert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBA8CjD,CAAC"}

package/dist/actions/assert.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.assertTextVisibilityActionGenerator = exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = void 0;
+const constants_1 = require("./constants");
 const utils_1 = require("./utils");
 exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = "assert_text_visibility";
 const assertTextVisibilityActionGenerator = (page) => {
@@ -32,7 +33,7 @@ const assertTextVisibilityActionGenerator = (page) => {
                         },
                         reason: {
                             type: "string",
-                            description: "reason for calling this function",
+                            description: constants_1.DEFAULT_ACTION_REASON_PROMPT,
                         },
                     },
                     required: ["css_selector", "reason"],

package/dist/actions/click.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBA8ClC,CAAC"}
+{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,4BAA4B,kBAAkB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,yBA8ClC,CAAC"}

package/dist/actions/click.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.clickActionGenerator = exports.PLAYWRIGHT_CLICK_ACTION_NAME = void 0;
+const constants_1 = require("./constants");
 const utils_1 = require("./utils");
 exports.PLAYWRIGHT_CLICK_ACTION_NAME = "click_element";
 const clickActionGenerator = (page) => {
@@ -34,7 +35,7 @@ const clickActionGenerator = (page) => {
                         },
                         reason: {
                             type: "string",
-                            description: "reason for calling this function",
+                            description: constants_1.DEFAULT_ACTION_REASON_PROMPT,
                         },
                     },
                     required: ["css_selector", "reason"],

package/dist/actions/constants/index.d.ts

@@ -0,0 +1,2 @@
+export declare const DEFAULT_ACTION_REASON_PROMPT = "reason for calling this function. the reason should align with the task provided";
+//# sourceMappingURL=index.d.ts.map

package/dist/actions/constants/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/actions/constants/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,4BAA4B,qFAC2C,CAAC"}

package/dist/actions/constants/index.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_ACTION_REASON_PROMPT = void 0;
+exports.DEFAULT_ACTION_REASON_PROMPT = "reason for calling this function. the reason should align with the task provided";

package/dist/actions/done.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"done.d.ts","sourceRoot":"","sources":["../../src/actions/done.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAuBjC,CAAC"}
+{"version":3,"file":"done.d.ts","sourceRoot":"","sources":["../../src/actions/done.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAuBjC,CAAC"}

package/dist/actions/done.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.doneActionGenerator = exports.PLAYWRIGHT_DONE_ACTION_NAME = void 0;
+const constants_1 = require("./constants");
 exports.PLAYWRIGHT_DONE_ACTION_NAME = "task_done";
 const doneActionGenerator = () => {
     return {
@@ -17,7 +18,7 @@ const doneActionGenerator = () => {
                     properties: {
                         reason: {
                             type: "string",
-                            description: "reason for declaring the task is complete",
+                            description: constants_1.DEFAULT_ACTION_REASON_PROMPT,
                         },
                     },
                     required: [],

package/dist/actions/fill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAEhE,eAAO,MAAM,mBAAmB,EAAE,yBA8CjC,CAAC"}
+{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAEhE,eAAO,MAAM,mBAAmB,EAAE,yBA8CjC,CAAC"}

package/dist/actions/fill.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fillActionGenerator = exports.PLAYWRIGHT_FILL_ACTION_NAME = void 0;
+const constants_1 = require("./constants");
 const utils_1 = require("./utils");
 exports.PLAYWRIGHT_FILL_ACTION_NAME = "fill_input_element";
 const fillActionGenerator = (page) => {
@@ -33,11 +34,11 @@ const fillActionGenerator = (page) => {
                         },
                         css_selector: {
                             type: "string",
-                            description: "CSS selector to identify the element uniquely.When creating CSS selectors, ensure they are unique to the page and specific enough to select only one element",
+                            description: "CSS selector to identify the element uniquely.When creating CSS selectors, ensure they are unique to the page and specific enough to select only one element.",
                         },
                         reason: {
                             type: "string",
-                            description: "reason for calling this function",
+                            description: constants_1.DEFAULT_ACTION_REASON_PROMPT,
                         },
                     },
                     required: ["text", "css_selector", "reason"],

package/dist/actions/goto.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAqCjC,CAAC"}
+{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAqCjC,CAAC"}

package/dist/actions/goto.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.gotoActionGenerator = exports.PLAYWRIGHT_GOTO_ACTION_NAME = void 0;
 const utils_1 = require("../agent/browsing/utils");
+const constants_1 = require("./constants");
 exports.PLAYWRIGHT_GOTO_ACTION_NAME = "page_goto";
 const gotoActionGenerator = (page) => {
     return {
@@ -32,7 +33,7 @@ const gotoActionGenerator = (page) => {
                         },
                         reason: {
                             type: "string",
-                            description: "reason for calling this function",
+                            description: constants_1.DEFAULT_ACTION_REASON_PROMPT,
                         },
                     },
                     required: ["url", "reason"],

package/dist/actions/next-task.js

@@ -20,7 +20,7 @@ Each action should take the task to completion, if not the action is invalid.`,
                     },
                     reason: {
                         type: "string",
-                        description: "explain how this action will help to complete the task",
+                        description: "explain how this action will help to complete the task. the reason should align with the task provided",
                     },
                 },
                 required: ["action", "reason"],

package/dist/actions/reload-page.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reload-page.d.ts","sourceRoot":"","sources":["../../src/actions/reload-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAErD,eAAO,MAAM,6BAA6B,gBAAgB,CAAC;AAE3D,eAAO,MAAM,qBAAqB,EAAE,yBA+BnC,CAAC"}
+{"version":3,"file":"reload-page.d.ts","sourceRoot":"","sources":["../../src/actions/reload-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,6BAA6B,gBAAgB,CAAC;AAE3D,eAAO,MAAM,qBAAqB,EAAE,yBA+BnC,CAAC"}

package/dist/actions/reload-page.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.reloadActionGenerator = exports.PLAYWRIGHT_RELOAD_ACTION_NAME = void 0;
 const utils_1 = require("../agent/browsing/utils");
+const constants_1 = require("./constants");
 exports.PLAYWRIGHT_RELOAD_ACTION_NAME = "page_reload";
 const reloadActionGenerator = (page) => {
     return {
@@ -25,7 +26,7 @@ const reloadActionGenerator = (page) => {
                     properties: {
                         reason: {
                             type: "string",
-                            description: "reason for calling this function",
+                            description: constants_1.DEFAULT_ACTION_REASON_PROMPT,
                         },
                     },
                     required: ["reason"],

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAiK9B;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAqG9B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAmK9B;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBAqG9B"}

package/dist/agent/browsing/index.js

@@ -40,7 +40,7 @@ async function browsingAgentUsingMasterAgent(task, page, options) {
                 llm,
                 trace: masterAgentSpan,
                 task,
-                conversation: ["Successful actions", ...masterAgentActions],
+                conversation: ["Successfully executed actions", ...masterAgentActions],
             });
             isGivenTaskDone = verificationAgentResp.isDone;
             if (isGivenTaskDone) {
@@ -77,9 +77,11 @@ async function browsingAgentUsingMasterAgent(task, page, options) {
                     llm,
                     trace: browsingAgentSpan,
                     task: action,
-                    conversation: ["Successful actions", ...successfulActions],
+                    conversation: ["Successfully executed actions", ...successfulActions],
                 });
                 isTaskDone = verificationAgentResp.isDone;
+                logger.log(`isTaskDone: ${isTaskDone}`);
+                logger.log(`reason: ${verificationAgentResp.reason}`);
                 if (isTaskDone) {
                     browsingAgentSpan.event({ name: "task-done" });
                     browsingAgentSpan.end({

package/dist/agent/verification/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA4DA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA8DA"}

package/dist/agent/verification/index.js

@@ -10,7 +10,10 @@ async function verificationAgent({ llm, trace, task, conversation, }) {
         messages: [
             {
                 role: "system",
-                content: "Given a conversation and a task, your task is to analyse the conversation and tell if the task is completed. If not, you need to tell what is not completed and suggest next steps to complete the task.",
+                content: `Given a conversation and a task, your task is to analyse the conversation and tell if the task is completed. 
+If not, you need to tell what is not completed and suggest next steps to complete the task.
+You need to respond assuming the conversation provided to you is truthful.
+`,
             },
             {
                 role: "user",

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAQpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAiD7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAM3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAOhD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAYpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAiD7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAM3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAOhD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD"}

package/dist/bin/utils/platform/web/index.js

@@ -7,20 +7,19 @@ exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lin
 const eslint_1 = require("eslint");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const prettier_1 = __importDefault(require("prettier"));
+const ts_morph_1 = require("ts-morph");
 const typescript_1 = __importDefault(require("typescript"));
 function getTypescriptTestBlock(scenarioName, content) {
-    const regex = new RegExp(`test\\("${scenarioName}"[^]*?\\}\\);`, "g");
-    const result = regex.exec(content);
-    if (result && result[0]) {
-        return result[0];
-    }
-    else {
-        return undefined;
-    }
+    const project = new ts_morph_1.Project();
+    const sourceFile = project.createSourceFile("test.ts", content);
+    const testFunction = sourceFile.getFirstDescendant((node) => !!(node.isKind(ts_morph_1.SyntaxKind.CallExpression) &&
+        node.getExpression().getText() === "test" &&
+        node.getArguments()[0]?.getText().includes(scenarioName)));
+    return testFunction?.getText();
 }
 exports.getTypescriptTestBlock = getTypescriptTestBlock;
 function appendToTestBlock(testBlock, content) {
-    const updateTestBlock = testBlock.replace(/\}\);$/, `\n\n${content}\n\n });`);
+    const updateTestBlock = testBlock.replace(/\}\)$/, `\n\n${content}\n\n })`);
     return updateTestBlock;
 }
 exports.appendToTestBlock = appendToTestBlock;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.21.1",
+  "version": "0.21.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -38,10 +38,11 @@
     "prettier": "^3.2.5",
     "sanitize-html": "^2.13.0",
     "slugify": "^1.6.6",
+    "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "@empiricalrun/llm": "^0.4.4",
-    "@empiricalrun/reporter": "^0.14.2"
+    "@empiricalrun/reporter": "^0.14.3"
   },
   "devDependencies": {
     "@types/detect-port": "^1.3.5",