@empiricalrun/test-gen 0.42.16 → 0.42.18

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @empiricalrun/test-gen
 
+## 0.42.18
+
+### Patch Changes
+
+- 6f876ea: fix: remove code generation feedback loop from repo edit
+- 658451e: fix: tracing for vitests
+- Updated dependencies [658451e]
+  - @empiricalrun/llm@0.9.32
+
+## 0.42.17
+
+### Patch Changes
+
+- 65e821a: chore: remove dead code around selector hints
+- 286a1a5: fix: annotation test
+
 ## 0.42.16
 
 ### Patch Changes

package/dist/agent/codegen/fix-ts-errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAQhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,OAAO,EACP,eAAe,EACf,QAAQ,EACR,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,iBAwEA"}
+{"version":3,"file":"fix-ts-errors.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/fix-ts-errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAQhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,8BAA8B,CAAC,EACnD,KAAK,EACL,MAA2B,EAC3B,IAAI,EACJ,OAAO,EACP,eAAe,EACf,QAAQ,EACR,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,iBA4EA"}

package/dist/agent/codegen/fix-ts-errors.js

@@ -64,6 +64,10 @@ async function validateAndFixTypescriptErrors({ trace, logger = new logger_1.Cus
         await (0, web_1.lintErrors)(file);
         const validateTypesSpan = trace?.span({
             name: "detect-type-errors-in-file",
+            input: {
+                filePath: file,
+                content: response,
+            },
         });
         errors = (0, web_1.validateTypescript)(file);
         validateTypesSpan?.end({ output: { errors } });

package/dist/agent/codegen/generate-code-apply-changes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate-code-apply-changes.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/generate-code-apply-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AASrE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAOhD,OAAO,EAAsB,UAAU,EAAsB,MAAM,SAAS,CAAC;AAqL7E,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,UAkD3D;AAgED,wBAAsB,2BAA2B,CAAC,EAChD,IAAI,EACJ,KAAK,EACL,MAAM,EACN,gBAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gBAAgB,EAAE,MAAM,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAyLxB"}
+{"version":3,"file":"generate-code-apply-changes.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/generate-code-apply-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQrE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAOhD,OAAO,EAAsB,UAAU,EAAE,MAAM,SAAS,CAAC;AAqLzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,MAAM,UAkD3D;AA2DD,wBAAsB,2BAA2B,CAAC,EAChD,IAAI,EACJ,KAAK,EACL,MAAM,EACN,gBAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gBAAgB,EAAE,MAAM,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuIxB"}

package/dist/agent/codegen/generate-code-apply-changes.js

@@ -264,12 +264,11 @@ In order to execute the task, FOLLOW BELOW STEPS:
 }
 function deDupUpdatedFiles(updatedFiles) {
     return updatedFiles.filter((change, index, self) => index ===
-        self.findIndex((existing) => existing.filePath === change.filePath &&
-            existing.oldCode === change.oldCode &&
-            existing.newCode === change.newCode));
+        self.findIndex((existing) => existing.filePath === change.filePath));
 }
 async function generateCodeAndApplyChanges({ task, trace, logger, getRelevantFiles, }) {
-    let planRetries = 5;
+    // Reducing this from 5 to 3, if the required changes are getting missed will change it back.
+    let planRetries = 3;
     let updatedFiles = [];
     while (planRetries--) {
         const generateCodeAndApplyChangesSpan = trace?.span({
@@ -300,120 +299,81 @@ async function generateCodeAndApplyChanges({ task, trace, logger, getRelevantFil
             await (0, llm_1.flushAllTraces)();
             return deDupUpdatedFiles(updatedFiles);
         }
-        let strReplaceRetries = 3;
-        while (strReplaceRetries--) {
-            const promptForStrReplace = [
-                {
-                    role: "system",
-                    content: systemPromptBuilderForRepoEdit(files),
-                },
-                {
-                    role: "user",
-                    content: userPromptBuilderForStrReplace(strReplacePlan),
-                },
-            ];
-            const codeEditorSpan = generateCodeAndApplyChangesSpan?.span({
-                name: "code-editor-agent",
-                input: {
-                    prompt: promptForStrReplace,
-                },
-            });
-            const llm = new llm_1.LLM({
-                trace: codeEditorSpan,
-                provider: "anthropic",
-                defaultModel: "claude-3-5-sonnet-20240620",
-                providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
-            });
-            const completion = await llm.createChatCompletion({
-                messages: promptForStrReplace,
-                modelParameters: {
-                    ...constants_1.DEFAULT_MODEL_PARAMETERS,
-                    temperature: 0.1,
-                    tool_choice: "required",
-                },
-                trace: codeEditorSpan,
-                tools: getCodeEditorToolCalls(),
-            });
-            codeEditorSpan?.end({ output: { completion } });
-            if (!completion?.tool_calls || completion?.tool_calls?.length === 0) {
-                break;
-            }
-            let codeEditorToolCalls = completion.tool_calls;
-            // Filter out the tool calls which are for creating new files
-            const createFileToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.CREATE_FILE);
-            if (createFileToolCalls.length > 0) {
-                console.log(`create_file tool calls: `, createFileToolCalls);
-            }
-            await Promise.all(createFileToolCalls.map((tc) => {
-                return (async () => {
-                    const args = (0, utils_1.parseJson)(tc.function.arguments);
-                    updatedFiles.push({
-                        filePath: args.filePath,
-                        oldCode: "",
-                        newCode: args.code,
-                        reason: args.reason,
-                    });
-                    await fs_extra_1.default.mkdir((0, path_1.dirname)(args.filePath), { recursive: true });
-                    await fs_extra_1.default.writeFile(args.filePath, args.code, "utf-8");
-                    console.log(`Created file: ${args.filePath}`);
-                })();
-            }));
-            const strReplaceToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.STR_REPLACE);
-            if (strReplaceToolCalls.length > 0) {
-                console.log(`str_replace tool calls: `, strReplaceToolCalls);
-            }
-            // Filter out the tool calls which are for replacing code in existing files
-            const fileChanges = strReplaceToolCalls
-                .map((toolCall) => (0, utils_1.parseJson)(toolCall.function.arguments))
-                .filter((f) => f.filePath && fs_extra_1.default.existsSync(f.filePath));
-            updatedFiles.push(...fileChanges);
-            let failedCodeUpdates;
-            // applyChangesResponse contains the errors occurred while applying the changes
-            failedCodeUpdates = await (0, utils_2.applyFileChangesUsingStrReplace)({
-                trace: codeEditorSpan,
-                fileChanges,
-                logger,
-            });
-            // Filter out the responses having errors
-            failedCodeUpdates = failedCodeUpdates.filter((f) => f?.error);
-            // Filter out the tool calls which have errors
-            const toolCallsWithErrors = codeEditorToolCalls.filter((toolCall) => {
-                const args = (0, utils_1.parseJson)(toolCall.function.arguments);
-                return failedCodeUpdates.find((response) => response.filePath === args.filePath);
-            });
-            if (failedCodeUpdates.length === 0) {
-                break;
-            }
-            logger?.log(`Failed to apply changes, retrying...`, failedCodeUpdates);
-            const feedback = failedCodeUpdates
-                .map((updates) => `For file ${updates.filePath}: ${updates.errorMessage}`)
-                .join("\n");
-            promptForStrReplace.push({
-                role: "assistant",
-                tool_calls: toolCallsWithErrors,
-            });
-            toolCallsWithErrors.forEach((toolCall) => {
-                promptForStrReplace.push({
-                    role: "tool",
-                    tool_call_id: toolCall.id,
-                    content: `
-Errors while executing the changes provided in above tool call:
-${feedback}
-
-Please fix the errors and return the updated code.
-
-FOLLOW BELOW STEPS TO FIX THE ISSUES:
-- First read the error message and understand the issue.
-- Go through the new code block and current file code, to figure out the root cause of the issue.
-- Compile the steps that you need to follow to fix the issue.
-- Check the test names, to ensure that the changes are applied to the correct test.
-- Use separate 'str_replace' tool to make the changes for each update.
-- Return the updated code in the same format as provided in the tool call.
-
-NOTE: ONLY MAKE THE CHANGES TO FIX THE ISSUES MENTIONED IN THE ERROR MESSAGE AND NOTHING ELSE. NO EXTRA CODE REFACTORING IS REQUIRED.
-    `,
+        const promptForStrReplace = [
+            {
+                role: "system",
+                content: systemPromptBuilderForRepoEdit(files),
+            },
+            {
+                role: "user",
+                content: userPromptBuilderForStrReplace(strReplacePlan),
+            },
+        ];
+        const codeEditorSpan = generateCodeAndApplyChangesSpan?.span({
+            name: "code-editor-agent",
+            input: {
+                prompt: promptForStrReplace,
+            },
+        });
+        const llm = new llm_1.LLM({
+            trace: codeEditorSpan,
+            provider: "anthropic",
+            defaultModel: "claude-3-5-sonnet-20240620",
+            providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
+        });
+        const completion = await llm.createChatCompletion({
+            messages: promptForStrReplace,
+            modelParameters: {
+                ...constants_1.DEFAULT_MODEL_PARAMETERS,
+                temperature: 0.1,
+                tool_choice: "required",
+            },
+            trace: codeEditorSpan,
+            tools: getCodeEditorToolCalls(),
+        });
+        codeEditorSpan?.end({ output: { completion } });
+        if (!completion?.tool_calls || completion?.tool_calls?.length === 0) {
+            continue;
+        }
+        // Filter out the tool calls which are for creating new files
+        const createFileToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.CREATE_FILE);
+        if (createFileToolCalls.length > 0) {
+            console.log(`create_file tool calls: `, createFileToolCalls);
+        }
+        await Promise.all(createFileToolCalls.map((tc) => {
+            return (async () => {
+                const args = (0, utils_1.parseJson)(tc.function.arguments);
+                updatedFiles.push({
+                    filePath: args.filePath,
+                    oldCode: "",
+                    newCode: args.code,
+                    reason: args.reason,
                 });
-            });
+                await fs_extra_1.default.mkdir((0, path_1.dirname)(args.filePath), { recursive: true });
+                await fs_extra_1.default.writeFile(args.filePath, args.code, "utf-8");
+                console.log(`Created file: ${args.filePath}`);
+            })();
+        }));
+        const strReplaceToolCalls = completion.tool_calls.filter((tc) => tc.function.name === types_1.CodeEditorToolCall.STR_REPLACE);
+        if (strReplaceToolCalls.length > 0) {
+            console.log(`str_replace tool calls: `, strReplaceToolCalls);
+        }
+        // Filter out the tool calls which are for replacing code in existing files
+        const fileChanges = strReplaceToolCalls
+            .map((toolCall) => (0, utils_1.parseJson)(toolCall.function.arguments))
+            .filter((f) => f.filePath && fs_extra_1.default.existsSync(f.filePath));
+        // We add all the suggested changes to the updatedFiles array
+        // This is used to validate and format files later
+        updatedFiles.push(...fileChanges);
+        // applyChangesResponse contains the errors occurred while applying the changes
+        const updates = await (0, utils_2.applyFileChangesUsingStrReplace)({
+            trace: codeEditorSpan,
+            fileChanges,
+            logger,
+        });
+        const failedCodeUpdates = updates.filter((f) => f?.error);
+        if (failedCodeUpdates.length > 0) {
+            logger?.log(`Failed to apply changes, retrying...`, failedCodeUpdates);
         }
     }
     return deDupUpdatedFiles(updatedFiles);

package/dist/agent/master/with-hints.d.ts

@@ -1,13 +1,6 @@
 import { LLM, TraceClient } from "@empiricalrun/llm";
 import OpenAI from "openai";
 import { PlaywrightActions } from "../../actions";
-import { BrowsingAgentOptions } from "../browsing";
-export declare const getUserMessageWithForHints: ({ userMessage, options, pageScreenshot, annotatedPageScreenshot, }: {
-    userMessage: OpenAI.ChatCompletionUserMessageParam;
-    options?: BrowsingAgentOptions | undefined;
-    pageScreenshot: string;
-    annotatedPageScreenshot: string;
-}) => string | OpenAI.ChatCompletionContentPart[];
 export declare const triggerHintsFlow: ({ outputFromGetNextAction, generatedAnnotations, actions, llm, trace, }: {
     outputFromGetNextAction: {
         action: string;

package/dist/agent/master/with-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AASlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;aAChC,iBAAiB;SACrB,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAyGA,CAAC"}
+{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;aAChC,iBAAiB;SACrB,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAyGA,CAAC"}

package/dist/agent/master/with-hints.js

@@ -1,41 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.triggerHintsFlow = exports.getUserMessageWithForHints = void 0;
-const vision_1 = require("@empiricalrun/llm/vision");
+exports.triggerHintsFlow = void 0;
 const assert_1 = require("../../actions/assert");
 const click_1 = require("../../actions/click");
 const fill_1 = require("../../actions/fill");
 const hover_1 = require("../../actions/hover");
 const constants_1 = require("../../constants");
-const getUserMessageWithForHints = ({ userMessage, options, pageScreenshot, annotatedPageScreenshot, }) => {
-    return [
-        {
-            type: "text",
-            text: userMessage.content,
-        },
-        {
-            type: "text",
-            text: "Screenshot in normal mode 👇",
-        },
-        {
-            type: "image_url",
-            image_url: {
-                url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
-            },
-        },
-        {
-            type: "text",
-            text: "Screenshot in annotated mode 👇",
-        },
-        {
-            type: "image_url",
-            image_url: {
-                url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, annotatedPageScreenshot),
-            },
-        },
-    ];
-};
-exports.getUserMessageWithForHints = getUserMessageWithForHints;
 const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations, actions, llm, trace, }) => {
     try {
         const hasElementAnnotation = outputFromGetNextAction?.elementAnnotation?.length &&

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.42.16",
+  "version": "0.42.18",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -72,9 +72,9 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.31",
-    "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/reporter": "^0.23.1"
+    "@empiricalrun/llm": "^0.9.32",
+    "@empiricalrun/reporter": "^0.23.1",
+    "@empiricalrun/r2-uploader": "^0.3.8"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",