@empiricalrun/test-gen 0.46.9 → 0.46.11

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.46.11
+
+### Patch Changes
+
+- 0def0a2: feat: add inquirer prompts for missing CLI options
+
+## 0.46.10
+
+### Patch Changes
+
+- b3ca74f: fix: bubble up browsing agent execution feedback for runtime planner
+- b484292: fix: pages summary for runtime planner works without scoped vars
+
 ## 0.46.9
 
 ### Patch Changes

package/dist/actions/goto.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrE,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAAyB,CAAC,cAAc,CAyCzE,CAAC"}
+{"version":3,"file":"goto.d.ts","sourceRoot":"","sources":["../../src/actions/goto.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrE,eAAO,MAAM,2BAA2B,cAAc,CAAC;AAEvD,eAAO,MAAM,mBAAmB,EAAE,yBAAyB,CAAC,cAAc,CA0CzE,CAAC"}

package/dist/actions/goto.js

@@ -35,7 +35,7 @@ const gotoActionGenerator = (page) => {
                         },
                         url: {
                             type: "string",
-                            description: "URL to navigate page to",
+                            description: "URL to navigate page to. Must be prefixed with http or https (preferably https)",
                         },
                     },
                     required: ["reason", "url"],

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CA6D1C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CA8D1C"}

package/dist/agent/browsing/index.js

@@ -59,9 +59,10 @@ async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace
         }
         catch (e) {
             void testgenUpdatesReporter.sendMessage(e.message);
+            throw e;
         }
     }
-    toolCallsSpan?.end({ output: { toolCalls } });
+    toolCallsSpan?.end({ output: { toolCalls, generatedCodeSteps } });
     return generatedCodeSteps;
 }
 exports.executeTaskUsingBrowsingAgent = executeTaskUsingBrowsingAgent;

package/dist/agent/master/execute-browser-action.d.ts

@@ -15,5 +15,10 @@ export declare function executeBrowserAction({ page, nextAction, flags, actions,
     llm: LLM;
 }): Promise<{
     generatedCodeSteps: string[];
+    output: {
+        action: string;
+        reason: string;
+        elementAnnotation?: string | undefined;
+    };
 }>;
 //# sourceMappingURL=execute-browser-action.d.ts.map

package/dist/agent/master/execute-browser-action.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,GAAG,EAAE,GAAG,CAAC;CACV;;GAsIA"}
+{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,GAAG,EAAE,GAAG,CAAC;CACV;;;gBAIW,MAAM;gBACN,MAAM;;;GAkIjB"}

package/dist/agent/master/execute-browser-action.js

@@ -118,6 +118,7 @@ async function executeBrowserAction({ page, nextAction, flags, actions, llm, tra
     }
     return {
         generatedCodeSteps,
+        output,
     };
 }
 exports.executeBrowserAction = executeBrowserAction;

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAgCxC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAoRA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAelC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAgCxC,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GAwRA"}

package/dist/agent/master/run.js

@@ -115,7 +115,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
             task,
             successfulActions: [...masterAgentActions],
             pages: getPageVariables(actions.getStateVariables()),
-            currentPage: testGenPage.name,
+            currentPage: testGenPage,
         });
         isGivenTaskDone = plannerResp.isDone;
         if (isGivenTaskDone) {
@@ -191,7 +191,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                         break;
                     }
                     default: {
-                        const { generatedCodeSteps: codeFromExecuteAction } = await (0, execute_browser_action_1.executeBrowserAction)({
+                        const result = await (0, execute_browser_action_1.executeBrowserAction)({
                             page,
                             nextAction,
                             flags: {
@@ -201,7 +201,9 @@ async function createTestUsingMasterAgent({ task, page, testCase, specPath, opti
                             llm,
                             trace: masterAgentActionSpan,
                         });
+                        const { generatedCodeSteps: codeFromExecuteAction, output: outputFromExecuteAction, } = result;
                         generatedCodeSteps.push(...codeFromExecuteAction);
+                        output = outputFromExecuteAction;
                     }
                 }
                 // resetting error count as there is a successful action

package/dist/agent/planner/run-time-planner.d.ts

@@ -1,11 +1,12 @@
 import { TraceClient } from "@empiricalrun/llm";
 import type { Page } from "playwright";
+import { TestGenPage } from "../../types";
 export declare function runtimePlanner({ trace, task, successfulActions, pages, currentPage, }: {
     trace?: TraceClient;
     successfulActions: string[];
     task: string;
-    pages?: Record<string, Page>;
-    currentPage?: string;
+    pages: Record<string, Page>;
+    currentPage: TestGenPage;
 }): Promise<{
     pageName: string;
     isDone: boolean;

package/dist/agent/planner/run-time-planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAKvC,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;;;;GA4FA"}
+{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;CAC1B;;;;GA+FA"}

package/dist/agent/planner/run-time-planner.js

@@ -14,9 +14,12 @@ async function runtimePlanner({ trace, task, successfulActions, pages, currentPa
         },
     });
     const llm = new llm_1.LLM({ provider: "openai" });
+    // Add the current page to the page state variables
+    pages[currentPage.name] = currentPage.pwPageInstance;
     const pagesSummary = pages
         ? Object.entries(pages).map(([pageName, page]) => `${pageName}: Currently on ${page.url()}`)
         : [];
+    const pageNamesEnum = Object.keys(pages);
     const response = await llm.createChatCompletion({
         trace: runTimePlannerSpan,
         traceName: "runtime-planner-llm",
@@ -53,7 +56,7 @@ async function runtimePlanner({ trace, task, successfulActions, pages, currentPa
                             },
                             pageName: {
                                 type: "string",
-                                enum: pages ? Object.keys(pages) : [],
+                                enum: pageNamesEnum,
                                 description: "page name for the next action.",
                             },
                             isDone: {

package/dist/bin/index.js

@@ -160,13 +160,13 @@ async function runAgent(testGenConfig, testGenToken) {
         .option("--suites <suites>", "Comma separated list of describe blocks")
         .parse(process.argv);
     const options = program.opts();
-    (0, utils_2.validateCliOptions)(options);
-    const testGenConfig = options.token
-        ? (0, scenarios_1.loadTestConfigs)(options.token)
-        : (0, scenarios_1.buildTestConfigFromOptions)(options);
-    const testGenToken = options.token
-        ? options.token
-        : (0, scenarios_1.buildTokenFromOptions)(options);
+    const completedOptions = await (0, utils_2.validateAndCompleteCliOptions)(options);
+    const testGenConfig = completedOptions.token
+        ? (0, scenarios_1.loadTestConfigs)(completedOptions.token)
+        : (0, scenarios_1.buildTestConfigFromOptions)(completedOptions);
+    const testGenToken = completedOptions.token
+        ? completedOptions.token
+        : (0, scenarios_1.buildTokenFromOptions)(completedOptions);
     (0, reporter_1.setReporterConfig)({
         projectRepoName: testGenConfig.options?.metadata.projectRepoName,
         testSessionId: testGenConfig.options?.metadata.testSessionId,

package/dist/bin/utils/index.d.ts

@@ -5,5 +5,5 @@ export interface CliOptions {
     prompt?: string;
     suites?: string;
 }
-export declare function validateCliOptions(options: CliOptions): void;
+export declare function validateAndCompleteCliOptions(options: CliOptions): Promise<CliOptions>;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAS5D"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/utils/index.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AASD,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,UAAU,CAAC,CAiErB"}

package/dist/bin/utils/index.js

@@ -1,12 +1,64 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateCliOptions = void 0;
-function validateCliOptions(options) {
+exports.validateAndCompleteCliOptions = void 0;
+const inquirer_1 = __importDefault(require("inquirer"));
+async function validateAndCompleteCliOptions(options) {
     const hasToken = !!options.token;
-    const hasNameAndFile = !!options.name && !!options.file && !!options.prompt;
-    if (!hasToken && !hasNameAndFile) {
-        console.error("Invalid arguments. Provide either --token OR all of --name, --file, and --prompt");
-        process.exit(1);
+    if (hasToken) {
+        return options;
     }
+    const questions = [];
+    if (!options.name) {
+        questions.push({
+            type: "input",
+            name: "name",
+            message: "Enter the test name:",
+            validate: (input) => input.trim().length > 0 || "Test name is required",
+        });
+    }
+    if (!options.file) {
+        questions.push({
+            type: "input",
+            name: "file",
+            message: "Enter the test file path (inside tests dir):",
+            validate: (input) => input.trim().length > 0 || "Test file path is required",
+        });
+    }
+    if (!options.prompt) {
+        questions.push({
+            type: "editor",
+            name: "prompt",
+            message: "Enter the test prompt (opens in your editor):",
+            validate: (input) => input.trim().length > 0 || "Test prompt is required",
+        });
+    }
+    if (!options.suites) {
+        questions.push({
+            type: "input",
+            name: "suites",
+            message: "Enter comma-separated test suites (optional):",
+        });
+    }
+    if (questions.length > 0) {
+        // Ask each question individually to avoid type issues
+        const answers = {};
+        for (const question of questions) {
+            const answer = await inquirer_1.default.prompt({
+                type: question.type,
+                name: "value",
+                message: question.message,
+                validate: question.validate,
+            });
+            answers[question.name] = answer.value;
+        }
+        return {
+            ...options,
+            ...answers,
+        };
+    }
+    return options;
 }
-exports.validateCliOptions = validateCliOptions;
+exports.validateAndCompleteCliOptions = validateAndCompleteCliOptions;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.46.9",
+  "version": "0.46.11",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"