@empiricalrun/test-gen 0.43.2 → 0.43.3

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @empiricalrun/test-gen
 
+## 0.43.3
+
+### Patch Changes
+
+- 0b4cffa: feat: add human in the loop for planner
+- 694e202: fix: report gen assets should not throw in cli
+
 ## 0.43.2
 
 ### Patch Changes

package/dist/agent/planner/run.d.ts

@@ -3,5 +3,5 @@ export declare function planTask({ task, specPath, trace, }: {
     task: string;
     specPath: string;
     trace?: TraceClient;
-}): Promise<string>;
+}): Promise<any>;
 //# sourceMappingURL=run.d.ts.map

package/dist/agent/planner/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AA4BpE,wBAAsB,QAAQ,CAAC,EAC7B,IAAI,EACJ,QAAQ,EACR,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,mBAyCA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAapE,wBAAsB,QAAQ,CAAC,EAC7B,IAAI,EACJ,QAAQ,EACR,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,gBA0HA"}

package/dist/agent/planner/run.js

@@ -3,24 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.planTask = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const context_1 = require("../../bin/utils/context");
-const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert software engineer in test. You are given a task to provide a high level plan to create a test for a\ngiven scenario.\n\nYou will be provided with already added tests and page object object models which you can use to plan out how to write\nthe test.\n\nThe expected plan should be a list of bullet points and each bullet point is a step in the test.\nYou will be provided with app knowledge as well, which can help you groom the steps in the tests.\n{{/section}}\n\n{{#section \"user\"}}\nApp knowledge\n{{appKnowledge}}\n\n-----\n\nCurrent tests and page object model references\n\n{{fileContext}}\n\n------\n\nTask:\n\n{{task}}\n\n------\n\nFollow the steps to create a test plan:\n- create sub tasks\n- read the task step by step and create sub tasks from the given task\n- Ensure no new steps are added which are not mentioned in the task\n- enriched sub tasks\n- Read the app knowledge provided to you and enrich the verified sub tasks based on the provided information in app\nknowledge.\n- Fill in the missing information in the verified sub tasks based on the app knowledge.\n- based on the type of task, add sub tasks to the verified sub tasks based on the matching criteria\n- final plan:\n- Once all the subtasks are enriched, list all the sub tasks as bullet points\n- Each bullet point should be one of the following actions:\n- Open page, Click on, Fill in, Assert, hover on, press, extract textContent\n- Do not respond with points which do not start with above actions.\n\nFollow the steps before responding\n- The steps should only contain bullet points on list of steps for the test\n- Do not add any other assertion which is not mentioned in the task or app knowledge\n- Respond with <create_sub_tasks></create_sub_tasks>\n<enriched_sub_tasks></enriched_sub_tasks> and <final_plan></final_plan>\n- The final plan should not mention reference to the knowledge base used to generate it\n{{/section}}";
-function extractTestPlan(input) {
-    const result = {
-        createSubTasks: "",
-        verfiedSubTasks: "",
-        enrichedSubTasks: "",
-        finalPlan: "",
-    };
-    const regex = /<create_sub_tasks>([\s\S]*?)<\/create_sub_tasks>[\s\S]*?<enriched_sub_tasks>([\s\S]*?)<\/enriched_sub_tasks>[\s\S]*?<final_plan>([\s\S]*?)<\/final_plan>/g;
-    let match;
-    while ((match = regex.exec(input)) !== null) {
-        const [, createSubTasks, enrichedSubTasks, finalPlan] = match;
-        result.createSubTasks = createSubTasks?.trim() || "";
-        result.enrichedSubTasks = enrichedSubTasks?.trim() || "";
-        result.finalPlan = finalPlan?.trim() || "";
-    }
-    return result;
-}
+const human_in_the_loop_1 = require("../../human-in-the-loop");
+const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert software engineer in test and have capability to collaborate with user to generate a test plan.\nYou are given a task to provide a high level plan to create a test for a given scenario.\n\nYou will be provided with already added Playwright tests and page object object models which you can use generate the test plan.\nTest plan should be a list of bullet points and each bullet point representing each test step.\nYou will be provided with app knowledge which can help you groom the steps in the plan.\n\n{{#if humanLoopRequired}}\nDo NOT finish until you have a complete understanding of the plan and have all the minute details of how the test should look like.\n\nALWAYS VERIFY THE PLAN WITH USER BEFORE FINISHING. THE USER SHOULD BE ABLE TO REVIEW AND GIVE GO AHEAD TO THE PLAN.\n\nDO NOT FINISH UNTIL USER EXPLICITLY CONFIRMS THE PLAN IS GOOD TO GO.\n{{/if}}\n\n### Test Plan Creation Guidelines\n\n- **Create Subtasks:**\n  - Read the task step by step and create subtasks from the given task.\n  - Do not add any new steps that are not mentioned in the task.\n\n- **Enrich Subtasks:**\n  - Read the provided app knowledge.\n  - Enrich the subtasks based on the information from the app knowledge.\n\n- **Plan:**\n  - Once all subtasks are enriched, list them as bullet points.\n  - Each bullet point must start with one of the following actions:  \n    - Open page  \n    - Click on  \n    - Fill in  \n    - Assert  \n    - Hover on  \n    - Press  \n    - Extract textContent\n  - Do not include any bullet points that do not start with the above actions.\n\n- **Additional Notes:**\n  - The final output should only contain bullet points listing the test steps.\n  - Do not add any assertions that are not mentioned in the task or app knowledge.\n  - The plan should not reference the knowledge base used to generate it.\n\n{{/section}}\n\n{{#section \"user\"}}\n\nApp knowledge:\n{{appKnowledge}}\n\n-----\n\nCurrent tests and page object model references:\n{{fileContext}}\n\n------\n\nTask:\n{{task}}\n\n------\n{{/section}}";
 async function planTask({ task, specPath, trace, }) {
     let fileContext = "";
     try {
@@ -39,27 +23,106 @@ ${pomPrompt}
             task,
         },
     });
+    // if the code in running in ci worker or github actions, disable human loop
+    const humanLoopRequired = process.env.CI !== "true";
     const appKnowledge = await (0, context_1.fetchAppKnowledge)();
     const messages = (0, llm_1.compilePrompt)(promptTemplate_0, {
         appKnowledge,
         fileContext,
         task,
+        humanLoopRequired,
     });
     const llm = new llm_1.LLM({
         provider: "openai",
-        //TODO: change to o1
         defaultModel: "gpt-4o",
     });
-    const response = await llm.createChatCompletion({
-        trace: plannerSpan,
-        traceName: "planner-llm",
-        messages,
-        modelParameters: {
-            temperature: 0.5,
+    let maxRetries = 10;
+    // TODO: this should be environment driven
+    const hl = new human_in_the_loop_1.HumanLoop({ method: "ipc" });
+    const tools = [
+        {
+            type: "function",
+            function: {
+                name: "finish",
+                description: "Final output of the conversation",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        plan: {
+                            type: "string",
+                            description: "Final plan which is verified by the user",
+                        },
+                    },
+                    required: ["plan"],
+                    additionalProperties: false,
+                },
+            },
+        },
+    ];
+    if (humanLoopRequired) {
+        tools.push({
+            type: "function",
+            function: {
+                name: "send_user_message",
+                description: "Send user message",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        message: {
+                            type: "string",
+                            description: "Message to be sent to the user for any confirmation or clarification",
+                        },
+                    },
+                    required: ["message"],
+                    additionalProperties: false,
+                },
+                strict: true,
+            },
+        });
+    }
+    while (maxRetries--) {
+        const response = await llm.createChatCompletion({
+            trace: plannerSpan,
+            traceName: "planner-llm",
+            messages,
+            tools,
+            modelParameters: {
+                temperature: 0.5,
+                tool_choice: "required",
+            },
+        });
+        const toolCall = response?.tool_calls?.[0];
+        const respStr = toolCall?.function.arguments;
+        const resp = JSON.parse(respStr || "{}");
+        // TODO: handle this as a part of actions handler
+        if (toolCall?.function.name === "send_user_message") {
+            const feedback = await hl.getFeedback({
+                message: resp.message,
+            });
+            messages.push(response);
+            messages.push({
+                role: "tool",
+                tool_call_id: toolCall?.id,
+                content: feedback,
+            });
+        }
+        else {
+            const plan = resp.plan;
+            plannerSpan?.end({
+                output: {
+                    plan,
+                },
+            });
+            return plan;
+        }
+    }
+    // TODO: handle planner failure scenario
+    plannerSpan?.end({
+        output: {
+            plan: "",
+            error: "Failed to generate test plan",
         },
     });
-    const plan = extractTestPlan(response?.content || "");
-    plannerSpan?.end({ output: plan?.finalPlan });
-    return plan?.finalPlan;
+    return "";
 }
 exports.planTask = planTask;

package/dist/bin/index.js

@@ -154,7 +154,8 @@ async function runAgent(testGenConfig, testGenToken, span) {
             specPath,
             trace,
         });
-        new logger_1.CustomLogger({ useReporter: false }).log("Generated Plan:", plan);
+        new logger_1.CustomLogger({ useReporter: false }).log("Generated Plan:");
+        console.log(plan);
         await new reporter_1.TestGenUpdatesReporter().sendMessage(plan);
     }
     else if (agent === "code") {
@@ -221,7 +222,8 @@ async function runAgent(testGenConfig, testGenToken, span) {
             if (agentUsed &&
                 agentUsed !== "code" &&
                 agentUsed !== "plan" &&
-                testGenConfig.testCase.name) {
+                testGenConfig.testCase.name &&
+                testGenConfig.options) {
                 await new reporter_1.TestGenUpdatesReporter().reportGenAssets({
                     projectRepoName: testGenConfig.options.metadata.projectRepoName,
                     testName: testGenConfig.testCase.name,

package/dist/file/client.d.ts

@@ -6,6 +6,7 @@ declare class TestFileService {
         task: string;
         importPaths: string[];
     }): Promise<void>;
+    post(path: string, body: any): Promise<any>;
 }
 export default TestFileService;
 //# sourceMappingURL=client.d.ts.map

package/dist/file/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/file/client.ts"],"names":[],"mappings":"AAAA,cAAM,eAAe;IACnB,OAAO,EAAE,MAAM,CAAC;gBACJ,IAAI,EAAE,MAAM;IAIlB,UAAU,CAAC,EACf,aAAa,EACb,IAAI,EACJ,WAAW,GACZ,EAAE;QACD,aAAa,EAAE,MAAM,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;CAeF;AAED,eAAe,eAAe,CAAC"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/file/client.ts"],"names":[],"mappings":"AAAA,cAAM,eAAe;IACnB,OAAO,EAAE,MAAM,CAAC;gBACJ,IAAI,EAAE,MAAM;IAIlB,UAAU,CAAC,EACf,aAAa,EACb,IAAI,EACJ,WAAW,GACZ,EAAE;QACD,aAAa,EAAE,MAAM,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB;IAgBK,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG;CAgBnC;AAED,eAAe,eAAe,CAAC"}

package/dist/file/client.js

@@ -20,5 +20,21 @@ class TestFileService {
             console.log("Generated and updated test successfully");
         }
     }
+    async post(path, body) {
+        const resp = await fetch(`${this.baseUrl}${path}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(body),
+        });
+        if (!resp.ok) {
+            throw new Error(`API failed with status ${resp.statusText}`);
+        }
+        else {
+            const data = await resp.json();
+            return data;
+        }
+    }
 }
 exports.default = TestFileService;

package/dist/file/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAUA,qBAAa,WAAW;IACtB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;gBAEjB,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAKhE,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAwC1C;AAED,wBAAsB,gBAAgB,kBAAK"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAWA,qBAAa,WAAW;IACtB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;gBAEjB,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAKhE,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;CAyC1C;AAED,wBAAsB,gBAAgB,kBAAK"}

package/dist/file/server.js

@@ -8,6 +8,7 @@ const express_1 = __importDefault(require("express"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const web_1 = require("../bin/utils/platform/web");
+const ipc_1 = require("../human-in-the-loop/ipc");
 class FileService {
     port = 0;
     filePath = "";
@@ -22,6 +23,7 @@ class FileService {
     async startFileService() {
         const app = (0, express_1.default)();
         app.use(express_1.default.json());
+        (0, ipc_1.humanLoopRoute)(app);
         app.post("/test", async (req, res) => {
             const { generatedCode, importPaths } = req.body;
             try {

package/dist/human-in-the-loop/cli.d.ts

@@ -0,0 +1,2 @@
+export declare function getFeedbackUsingCli(message: string): Promise<string>;
+//# sourceMappingURL=cli.d.ts.map

package/dist/human-in-the-loop/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/human-in-the-loop/cli.ts"],"names":[],"mappings":"AAEA,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAe1E"}

package/dist/human-in-the-loop/cli.js

@@ -0,0 +1,24 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFeedbackUsingCli = void 0;
+const inquirer_1 = __importDefault(require("inquirer"));
+async function getFeedbackUsingCli(message) {
+    const answers = await inquirer_1.default.prompt([
+        {
+            type: "input",
+            name: "feedback",
+            message: message || "Does this plan look good?",
+            validate: (input) => {
+                if (input.trim().length === 0) {
+                    return "Feedback cannot be empty";
+                }
+                return true;
+            },
+        },
+    ]);
+    return answers.feedback;
+}
+exports.getFeedbackUsingCli = getFeedbackUsingCli;

package/dist/human-in-the-loop/index.d.ts

@@ -0,0 +1,10 @@
+export declare class HumanLoop {
+    private options;
+    constructor(options: {
+        method: "ipc" | "cli";
+    });
+    getFeedback({ message }: {
+        message: string;
+    }): Promise<string>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/human-in-the-loop/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/human-in-the-loop/index.ts"],"names":[],"mappings":"AAQA,qBAAa,SAAS;IAER,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE;QAAE,MAAM,EAAE,KAAK,GAAG,KAAK,CAAA;KAAE;IAChD,WAAW,CAAC,EAAE,OAAO,EAAE,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE;CAOnD"}

package/dist/human-in-the-loop/index.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HumanLoop = void 0;
+const cli_1 = require("./cli");
+const ipc_1 = require("./ipc");
+// TODO: how to make this more conversational
+// do we need a layer to understand the feedback ?
+// we need a layer to put the feedback in the context again
+// and then we need to generate final output which the user can review
+class HumanLoop {
+    options;
+    // TODO: have a better name for this
+    constructor(options) {
+        this.options = options;
+    }
+    async getFeedback({ message }) {
+        // this needs to have a provider based on environment
+        if (this.options.method === "ipc") {
+            return await (0, ipc_1.getFeedbackUsingIPC)(message);
+        }
+        return await (0, cli_1.getFeedbackUsingCli)(message);
+    }
+}
+exports.HumanLoop = HumanLoop;

package/dist/human-in-the-loop/ipc.d.ts

@@ -0,0 +1,4 @@
+import type { Express } from "express";
+export declare function humanLoopRoute(app: Express): void;
+export declare function getFeedbackUsingIPC(message: string): Promise<string>;
+//# sourceMappingURL=ipc.d.ts.map

package/dist/human-in-the-loop/ipc.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ipc.d.ts","sourceRoot":"","sources":["../../src/human-in-the-loop/ipc.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAKvC,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,QAwB1C;AAED,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAa1E"}

package/dist/human-in-the-loop/ipc.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFeedbackUsingIPC = exports.humanLoopRoute = void 0;
+const inquirer_1 = __importDefault(require("inquirer"));
+const client_1 = __importDefault(require("../file/client"));
+function humanLoopRoute(app) {
+    app.post("/converse", async (req, res) => {
+        const { message } = req.body;
+        try {
+            const answers = await inquirer_1.default.prompt([
+                {
+                    type: "input",
+                    name: "feedback",
+                    message: message || "Does this plan look good?",
+                    validate: (input) => {
+                        if (input.trim().length === 0) {
+                            return "Feedback cannot be empty";
+                        }
+                        return true;
+                    },
+                },
+            ]);
+            return res.send({
+                ...answers,
+            });
+        }
+        catch (e) {
+            console.error(e);
+        }
+    });
+}
+exports.humanLoopRoute = humanLoopRoute;
+async function getFeedbackUsingIPC(message) {
+    const port = process.env.APP_PORT || 3030;
+    const fileService = new client_1.default(Number(port));
+    try {
+        const resp = await fileService.post("/converse", { message });
+        return resp.feedback;
+    }
+    catch (e) {
+        console.error("[getFeedbackUsingIPC] Error while fetching feedback using IPC", e);
+        throw e;
+    }
+}
+exports.getFeedbackUsingIPC = getFeedbackUsingIPC;

package/dist/prompts/lib/ts-transformer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ts-transformer.d.ts","sourceRoot":"","sources":["../../../src/prompts/lib/ts-transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,YAAY,CAAC;AAEjC,iBAAS,2BAA2B,IAAI,EAAE,CAAC,kBAAkB,CAAC,EAAE,CAAC,UAAU,CAAC,CAkG3E;AAED,eAAe,2BAA2B,CAAC"}
+{"version":3,"file":"ts-transformer.d.ts","sourceRoot":"","sources":["../../../src/prompts/lib/ts-transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,YAAY,CAAC;AAEjC,iBAAS,2BAA2B,IAAI,EAAE,CAAC,kBAAkB,CAAC,EAAE,CAAC,UAAU,CAAC,CAsG3E;AAED,eAAe,2BAA2B,CAAC"}

package/dist/prompts/lib/ts-transformer.js

@@ -32,7 +32,9 @@ function createHandlebarsTransformer() {
         const importToVarMap = new Map();
         const visitor = (node) => {
             // Handle references to the imported identifiers
-            if (ts.isIdentifier(node) && importToVarMap.has(node.text)) {
+            if (ts.isIdentifier(node) &&
+                importToVarMap.has(node.text) &&
+                !ts.isPropertyAccessExpression(node.parent)) {
                 return ts.factory.createIdentifier(importToVarMap.get(node.text));
             }
             return ts.visitEachChild(node, visitor, context);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.43.2",
+  "version": "0.43.3",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -58,6 +58,7 @@
     "google-auth-library": "^9.10.0",
     "google-spreadsheet": "^4.1.2",
     "ignore": "^5.3.1",
+    "inquirer": "^12.4.2",
     "lodash.isequal": "^4.5.0",
     "md5": "^2.3.0",
     "mime": "^4.0.4",
@@ -72,8 +73,8 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/llm": "^0.9.35",
+    "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/reporter": "^0.23.1"
   },
   "devDependencies": {