@empiricalrun/test-gen 0.24.1 → 0.25.0

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @empiricalrun/test-gen
 
+## 0.25.0
+
+### Minor Changes
+
+- a3fe47f: feat: deprecate browser agent and remove dependency of test object in create test
+
+## 0.24.3
+
+### Patch Changes
+
+- Updated dependencies [be30850]
+  - @empiricalrun/llm@0.9.1
+
+## 0.24.2
+
+### Patch Changes
+
+- ce74ef8: fix: local mock github flow
+
 ## 0.24.1
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts

@@ -6,6 +6,5 @@ type BrowsingAgentOptions = Partial<TestGenConfigOptions> & {
     };
 };
 export declare function browsingAgentUsingMasterAgent(task: string, page: Page, options: BrowsingAgentOptions): Promise<string>;
-export declare function browsingAgent(task: string, page: Page, options: BrowsingAgentOptions): Promise<string>;
 export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBA+L9B;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBA0G9B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,KAAK,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IAC1D,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,oBAAoB,mBA+L9B"}

package/dist/agent/browsing/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.browsingAgent = exports.browsingAgentUsingMasterAgent = void 0;
+exports.browsingAgentUsingMasterAgent = void 0;
 const llm_1 = require("@empiricalrun/llm");
 const crypto_1 = __importDefault(require("crypto"));
 const actions_1 = require("../../actions");
@@ -185,102 +185,3 @@ async function browsingAgentUsingMasterAgent(task, page, options) {
     return code;
 }
 exports.browsingAgentUsingMasterAgent = browsingAgentUsingMasterAgent;
-async function browsingAgent(task, page, options) {
-    const logger = new logger_1.CustomLogger();
-    const session = (0, session_1.getSessionDetails)();
-    const trace = llm_1.langfuseInstance.trace({
-        name: "browsing-agent",
-        id: crypto_1.default.randomUUID(),
-        version: (0, session_1.getSessionDetails)().version,
-        metadata: {
-            generationId: (0, session_1.getSessionDetails)().generationId,
-            sessionId: (0, session_1.getSessionDetails)().sessionId,
-        },
-        release: session.version,
-        tags: [
-            options.metadata?.projectName,
-            options.metadata?.environment,
-        ].filter((s) => !!s),
-    });
-    const actions = new actions_1.PlaywrightActions(page);
-    const tools = actions.getActionSchemas();
-    let isTaskDone = false;
-    const executedActions = [];
-    await (0, utils_1.injectPwLocatorGenerator)(page);
-    trace.update({ input: { task } });
-    let lastActionExecTrace = "";
-    const llm = new llm_1.LLM({
-        trace,
-        provider: options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-        defaultModel: options.model || constants_1.DEFAULT_MODEL,
-        providerApiKey: constants_1.MODEL_API_KEYS[options.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
-    });
-    while (!isTaskDone) {
-        const pageContentSpan = trace.span({ name: "page-content" });
-        const pageContent = await page.content();
-        pageContentSpan.end({ output: { pageContent } });
-        const sanitizationSpan = trace.span({ name: "page-sanitization" });
-        const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent, options.htmlSanitize);
-        sanitizationSpan.end({ output: { pageSnapshot } });
-        const promptSpan = trace.span({ name: "page-prompt" });
-        // extract all successful actions
-        const successfulActions = executedActions
-            .filter((a) => !a.isError)
-            .map((a) => a.action);
-        const messages = await (0, utils_1.getPromptForNextAction)({
-            pageSnapshot,
-            previousActions: successfulActions,
-            task,
-            lastActionErrors: lastActionExecTrace ? [lastActionExecTrace] : [],
-        });
-        promptSpan.end({ output: { messages } });
-        const completion = await llm.createChatCompletion({
-            messages,
-            tools,
-            modelParameters: {
-                ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                ...options.modelParameters,
-                tool_choice: "required",
-            },
-        });
-        const toolCalls = completion?.tool_calls || [];
-        for (const i in toolCalls) {
-            const toolCall = toolCalls[i];
-            try {
-                await actions.executeAction(toolCall.function.name, JSON.parse(toolCall.function.arguments));
-                executedActions.push({
-                    isError: false,
-                    action: JSON.stringify(toolCall),
-                });
-                lastActionExecTrace = "";
-            }
-            catch (e) {
-                // TODO: implement feedback loop to llm
-                executedActions.push({
-                    isError: true,
-                    action: JSON.stringify(toolCall),
-                });
-                lastActionExecTrace = e.message;
-                logger.error(lastActionExecTrace, e);
-            }
-        }
-        isTaskDone = actions.isComplete();
-        // mark task as done if llm is stuck in loop
-        if (executedActions.length > 3) {
-            const lastThreeActions = executedActions.slice(-3);
-            const lastThreeActionsFailed = lastThreeActions.every((a) => a.isError);
-            if (lastThreeActionsFailed) {
-                // TODO: this should be sent to dashboard
-                logger.error("Agent is not able to figure out next action, marking task as done");
-                isTaskDone = true;
-            }
-        }
-    }
-    await page.close();
-    const code = actions.generateCode();
-    trace.update({ input: { task }, output: { code } });
-    logger.success("Successfully generated code for the given task");
-    logger.log(`Trace: ${trace.getTraceUrl()}`);
-    return code;
-}
-exports.browsingAgent = browsingAgent;

package/dist/agent/browsing/run.js

@@ -24,7 +24,7 @@ async function generateTestsUsingBrowsingAgent(testFilePath) {
     const project = await (0, utils_2.detectProjectName)(testFilePath, playwrightConfig);
     logger.log(`Detected playwright project name: ${project}`);
     //TODO: change this to per test
-    let command = `npx playwright test ${testFilePath} --retries 0 --project ${project}`;
+    let command = `npx playwright test ${testFilePath} --retries 0 --project ${project} --timeout 0`;
     if (!process.env.CI) {
         command = command.concat(` --headed`);
     }

package/dist/agent/browsing/utils.js

@@ -74,7 +74,7 @@ ${createTestGenBlock(steps)}
 function createTestGenBlock(steps) {
     return `
   await createTest(\`
-${steps}\`, page, test);
+${steps}\`, page);
   `;
 }
 async function injectPwLocatorGenerator(page) {

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CA+IrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAmJrB"}

package/dist/agent/codegen/run.js

@@ -21,7 +21,8 @@ async function generateTest(testCase, file, options) {
     }
     const context = await (0, context_1.contextForGeneration)(file);
     const { codePrompt, pomPrompt, testFileContent } = context;
-    const isUpdate = testFileContent.includes(`test("${testCase?.name}"`);
+    const { testBlock } = (0, web_1.getTypescriptTestBlock)(testCase?.name, testFileContent);
+    const isUpdate = !!testBlock;
     if (isUpdate) {
         return await (0, update_flow_1.updateTest)(testCase, file, options);
     }

package/dist/index.d.ts

@@ -1,3 +1,3 @@
 import { Page } from "playwright";
-export declare function createTest(task: string, page: Page, test: any): Promise<void>;
+export declare function createTest(task: string, page: Page): Promise<void>;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAalC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,iBA+BnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAalC,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,iBA0BxD"}

package/dist/index.js

@@ -14,7 +14,7 @@ process.on("beforeExit", async () => await (0, llm_1.flushAllTraces)());
 process.on("exit", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
 process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());
-async function createTest(task, page, test) {
+async function createTest(task, page) {
     const port = process.env.APP_PORT || 3030;
     const testConfigArg = process.env.TEST_GEN_TOKEN;
     const { testGenConfig } = await (0, utils_1.parseCliArgs)(testConfigArg);
@@ -30,11 +30,7 @@ async function createTest(task, page, test) {
         generationId: testGenConfig.options?.metadata.generationId,
     });
     const fileService = new client_1.default(Number(port));
-    test.setTimeout(900000);
-    const agent = testGenConfig.options?.agent === "browser"
-        ? browsing_1.browsingAgent
-        : browsing_1.browsingAgentUsingMasterAgent;
-    const code = await agent(task, page, {
+    const code = await (0, browsing_1.browsingAgentUsingMasterAgent)(task, page, {
         htmlSanitize: {
             disallowedStrings: ["v-data-table__td v-data-table-column--align-start"],
         },

package/dist/types/index.d.ts

@@ -6,7 +6,7 @@ export type FileContent = {
     content: string;
 };
 export type TestGenConfigOptions = {
-    agent: "code" | "browser" | "master";
+    agent: "code" | "master";
     model: LLMModel;
     modelProvider: LLMProvider;
     modelParameters?: ModelParameters;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;IACrC,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,QAAQ,CAAC;IACzB,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,MAAM,CAAC;CAC/E,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.24.1",
+  "version": "0.25.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.0",
+    "@empiricalrun/llm": "^0.9.1",
     "@empiricalrun/r2-uploader": "^0.1.1",
     "@empiricalrun/reporter": "^0.17.10"
   },