@empiricalrun/test-gen 0.75.0 → 0.77.0

package/dist/agent/code-review/index.js

@@ -0,0 +1,159 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CodeReviewAgent = exports.CodeReviewSeverity = exports.CodeReviewVerdict = void 0;
+const tools_1 = require("../../tools");
+const fetch_session_diff_1 = require("../../tools/fetch-session-diff");
+const base_1 = require("../base");
+const parser_1 = require("./parser");
+const types_1 = require("./types");
+Object.defineProperty(exports, "CodeReviewSeverity", { enumerable: true, get: function () { return types_1.CodeReviewSeverity; } });
+Object.defineProperty(exports, "CodeReviewVerdict", { enumerable: true, get: function () { return types_1.CodeReviewVerdict; } });
+class CodeReviewAgent extends base_1.BaseAgent {
+    getTools() {
+        const custom = [
+            fetch_session_diff_1.fetchSessionDiffTool,
+            ...(0, tools_1.textViewToolsForModel)(this.selectedModel),
+        ];
+        return {
+            custom,
+            builtInTextEditor: (0, tools_1.hasBuiltInTextEditor)(this.selectedModel),
+        };
+    }
+    getResult() {
+        const messages = this.messages || [];
+        const lastMessage = messages.length
+            ? messages[messages.length - 1]
+            : undefined;
+        const lastMessageTextPart = lastMessage
+            ? lastMessage.parts
+                .filter((p) => "text" in p)
+                .find((p) => "text" in p && !!p.text)
+            : undefined;
+        const textPart = lastMessageTextPart;
+        const text = textPart?.text.trim();
+        if (!text) {
+            return undefined;
+        }
+        return (0, parser_1.convertXmlToV2Format)(text);
+    }
+    async buildSystemPrompt(repoContext) {
+        return `
+You are an expert code reviewer that specializes in reviewing Playwright test code. You are
+provided with tools to fetch diff and pull-request metadata for a code review, where a test has been added, test modified,
+or some configuration has changed.
+
+# Your goals
+- Understand the purpose and scope of the code change. You can use available tools to gather context of the change.
+- Identify critical issues that must be fixed before the code can be safely merged.
+- Detect code smells, anti-patterns, and non-deterministic behaviors that reduce test reliability - see below.
+- Call out test data assumptions or lack of clean up.
+- Suggest improvements and best practices to enhance maintainability and readability.
+- Form a definite conclusion on whether the code can be merged or not.
+- Share your findings and conclusion in the structured format shared below
+
+# Output format
+- You are expected to return the following sections in your response: last_commit, describe_code_change, line_comments, verdict and version
+- The last commit comes from the session diff tool call along with pull request metadata -- reproduce the commit sha as it is, without any additions (ignore the last commit timestamp) or bullet points
+- describe_code_change: A brief summary of what the code change is doing. This should be 4-6 sentences in a bullet list, formatted in markdown where each bullet must begin with a hyphen followed by a space (- ). 
+  Do not use any other character for bullets.
+- line_comments: Individual comments for specific issues found in the code, attributed to one or more lines of problematic code. There can be multiple separate issues for each line, share them using separate comments of varying severity. 
+  Each comment should correspond to one issue and include the following tags:
+  - file: The relative path to the file from repository root
+  - line-start: Starting line number of the issue
+  - line-end: Ending line number of the issue (same as line-start for if issue is in a single line of code)
+  - severity: Either "merge-blocking" or "warning". It is possible to have multiple issues in the same set of lines of either nature, in that case add multiple line_comments for those lines. 
+    Here severity denotes the nature of the issue - any issue that is preventing the code from being safe to merge and should be considered high priority is "merge-blocking". Look for any of the specific
+    bits below or other red flags you might see in the code. Each comment should be 1-2 sentences. If no blocking issues are found, a review will have no "merge-blocking" line comments.
+    Alternatively, the severity should be "warning" for situations where best practices were not followed or contain minor issues or warnings that can be safely ignored ie 
+  - message: 1-2 sentences describing the specific issue and suggested fixes or improvements.
+- verdict: "Approved" if code can be merged to production ie there are no "merge-blocking" line-comments or "Rejected" if the issues cannot be safely ignored. "Approved" or
+  "Rejected" are the only two possible values for this field.
+
+Return these as XML tags with markdown inside them
+
+<last_commit>
+...
+</last_commit>
+
+<describe_code_change>
+...
+</describe_code_change>
+
+<line_comments>
+  <comment>
+    <file>..</file>
+    <line-start>..</line-start>
+    <line-end>..</line-end>
+    <severity>..</severity>
+    <message>..</message>
+  </comment>
+  
+  <comment>
+    <file>..</file>
+    <line-start>..</line-start>
+    <line-end>..</line-end>
+    <severity>..</severity>
+    <message>..</message>
+  </comment>
+</line_comments>
+
+<verdict>
+...
+</verdict>
+
+# Severity: Merge blocking
+
+## Functionality regression
+- If the change is modifying an existing test, we need to ensure the functionality of the original test
+  is maintained in the new version. No hacking our way to get a green test!
+
+## Exception handling
+  - Any form of try-catch or exception handling is a code smell in test code. If there's an
+  exception, the test should fail
+
+## Conditionals
+  - Any conditionals (if, switch, ternary) in test code is a code smell. Tests are expected to be
+  deterministic. If you see conditionals, check if there's a comment explaining why it's needed.
+  Critically review the comment -- if it's not convincing, call it out as a code smell.
+
+## Playwright common mistakes
+- Don't use waitForLoadState or networkidle - these are not required since Playwright auto-waits after navigations. networkidle
+  can cause failures because modern web apps often have background network activity, which never settles.
+- Use locators instead of selectors: waitForSelector, $, $$ are bad - use locators instead (e.g. locator.waitFor)
+- If the test relies on some Playwright APIs that do not auto-wait (e.g. isVisible(), count()), we need to ensure 
+  they are used AFTER some action that ensures the page has loaded. If nothing, at least it should have a waitForTimeout
+
+## Deprecated patterns
+- test.describe.serial(...) is not prefered: use test.describe.configure({ mode: "serial" }) if the tests need to be serial
+
+## Repo conventions
+- Tests are located in files in the tests/ directory (e.g. in tests/example.spec.ts)
+- Helper methods (that are imported in the tests) should be in pages/ directory (e.g. pages/common.ts)
+- Helper methods should be functional - not classes (conventional class-based page object models are NOT recommended - use functions!)
+
+# Severity: Warning
+
+## Call out test data assumptions
+- If new test data is created (e.g. creating a new entity in the app, doing some actions on it) - it should be cleaned up
+  at the end of the test. If not, call it out.
+- If the test data cannot be cleaned up, are we using some random names to ensure no conflicts in future test runs?
+- If the test assumes some data exists (e.g. a user with a specific email) - call it out. It might fail across other
+  environments.
+- No hard coded URLs - use relative URLs instead - that can work across environments.
+- Dependency on static data that can change across environments (e.g. number of rows in a table) should be avoided.
+
+## Remove debug artifacts
+- If there are console.logs or page.screenshot usage, call it out.
+
+## Extra waits
+- Wait for timeout for static values are bad, but sometimes needed. Some apps are flaky and need additional waiting.
+
+## Element locators
+- CSS selectors can be brittle - prefer user facing selectors like getByRole, getByText
+
+# Repo context
+${repoContext}
+`;
+    }
+}
+exports.CodeReviewAgent = CodeReviewAgent;

package/dist/agent/code-review/parser.d.ts

@@ -0,0 +1,5 @@
+import { type CodeReviewResultV2 } from "./types";
+export type { CodeReviewLineComment, CodeReviewResultV0, CodeReviewResultV1, CodeReviewResultV2, } from "./types";
+export { CodeReviewSeverity, CodeReviewVerdict } from "./types";
+export declare function convertXmlToV2Format(output: string): CodeReviewResultV2;
+//# sourceMappingURL=parser.d.ts.map

package/dist/agent/code-review/parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/agent/code-review/parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,kBAAkB,EAGxB,MAAM,SAAS,CAAC;AAEjB,YAAY,EACV,qBAAqB,EACrB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,SAAS,CAAC;AAEjB,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAchE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,CAmFvE"}

package/dist/agent/code-review/parser.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CodeReviewVerdict = exports.CodeReviewSeverity = void 0;
+exports.convertXmlToV2Format = convertXmlToV2Format;
+const types_1 = require("./types");
+var types_2 = require("./types");
+Object.defineProperty(exports, "CodeReviewSeverity", { enumerable: true, get: function () { return types_2.CodeReviewSeverity; } });
+Object.defineProperty(exports, "CodeReviewVerdict", { enumerable: true, get: function () { return types_2.CodeReviewVerdict; } });
+function inferVerdictFromCommentsV2(lineComments) {
+    const hasMergeBlockingIssues = lineComments.some((comment) => comment.severity === types_1.CodeReviewSeverity.MergeBlocking);
+    return hasMergeBlockingIssues
+        ? types_1.CodeReviewVerdict.Rejected
+        : types_1.CodeReviewVerdict.Approved;
+}
+function convertXmlToV2Format(output) {
+    const lastCommitMatch = output.match(/<last_commit>([\s\S]*?)<\/last_commit>/i);
+    const codeChangeMatch = output.match(/<describe_code_change>([\s\S]*?)<\/describe_code_change>/i);
+    const verdictMatch = output.match(/<verdict>([\s\S]*?)<\/verdict>/i);
+    const lineComments = [];
+    const lineCommentsMatch = output.match(/<line_comments>([\s\S]*?)<\/line_comments>/i);
+    if (lineCommentsMatch) {
+        const commentsContent = lineCommentsMatch[1];
+        const commentMatches = commentsContent.match(/<comment>([\s\S]*?)<\/comment>/gi);
+        if (commentMatches) {
+            for (const commentMatch of commentMatches) {
+                const fileMatch = commentMatch.match(/<file>([\s\S]*?)<\/file>/i);
+                const lineStartMatch = commentMatch.match(/<line-start>([\s\S]*?)<\/line-start>/i);
+                const lineEndMatch = commentMatch.match(/<line-end>([\s\S]*?)<\/line-end>/i);
+                const severityMatch = commentMatch.match(/<severity>([\s\S]*?)<\/severity>/i);
+                const messageMatch = commentMatch.match(/<message>([\s\S]*?)<\/message>/i);
+                if (fileMatch &&
+                    lineStartMatch &&
+                    lineEndMatch &&
+                    severityMatch &&
+                    messageMatch) {
+                    const severityText = severityMatch[1].trim();
+                    const severity = severityText === "merge-blocking"
+                        ? types_1.CodeReviewSeverity.MergeBlocking
+                        : severityText === "warning"
+                            ? types_1.CodeReviewSeverity.Warning
+                            : null;
+                    if (severity !== null) {
+                        lineComments.push({
+                            file: fileMatch[1].trim(),
+                            line_start: parseInt(lineStartMatch[1].trim(), 10),
+                            line_end: parseInt(lineEndMatch[1].trim(), 10),
+                            severity: severity,
+                            message: messageMatch[1].trim(),
+                        });
+                    }
+                }
+            }
+        }
+    }
+    const lastCommit = lastCommitMatch[1].trim();
+    const describeCodeChange = codeChangeMatch[1].trim();
+    const verdict = verdictMatch?.[1]?.trim();
+    const finalVerdict = verdict
+        ? verdict === types_1.CodeReviewVerdict.Approved
+            ? types_1.CodeReviewVerdict.Approved
+            : types_1.CodeReviewVerdict.Rejected
+        : inferVerdictFromCommentsV2(lineComments);
+    return {
+        version: "2.0",
+        last_commit: lastCommit,
+        describe_code_change: describeCodeChange,
+        line_comments: lineComments,
+        verdict: finalVerdict,
+    };
+}

package/dist/agent/code-review/types.d.ts

@@ -0,0 +1,36 @@
+export declare enum CodeReviewVerdict {
+    Approved = "Approved",
+    Rejected = "Rejected"
+}
+export declare enum CodeReviewSeverity {
+    MergeBlocking = "merge-blocking",
+    Warning = "warning"
+}
+export type CodeReviewResultV1 = {
+    version: "1.0";
+    last_commit: string | null;
+    code_review_comments: string | null;
+    describe_code_change: string | null;
+    merge_blocking_issues: string | null;
+    best_practices_and_warnings: string | null;
+    verdict: "Approved" | "Rejected" | null;
+};
+export type CodeReviewLineComment = {
+    file: string;
+    line_start: number;
+    line_end: number;
+    severity: CodeReviewSeverity;
+    message: string;
+};
+export type CodeReviewResultV2 = {
+    version: "2.0";
+    last_commit: string;
+    describe_code_change: string;
+    line_comments: CodeReviewLineComment[];
+    verdict: CodeReviewVerdict;
+};
+export type CodeReviewResultV0 = {
+    version: "0.1";
+    result: string;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/agent/code-review/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/agent/code-review/types.ts"],"names":[],"mappings":"AAAA,oBAAY,iBAAiB;IAC3B,QAAQ,aAAa;IACrB,QAAQ,aAAa;CACtB;AAED,oBAAY,kBAAkB;IAC5B,aAAa,mBAAmB;IAChC,OAAO,YAAY;CACpB;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,KAAK,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,OAAO,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,KAAK,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,qBAAqB,EAAE,CAAC;IACvC,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,KAAK,CAAC;IAEf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC"}

package/dist/agent/code-review/types.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CodeReviewSeverity = exports.CodeReviewVerdict = void 0;
+var CodeReviewVerdict;
+(function (CodeReviewVerdict) {
+    CodeReviewVerdict["Approved"] = "Approved";
+    CodeReviewVerdict["Rejected"] = "Rejected";
+})(CodeReviewVerdict || (exports.CodeReviewVerdict = CodeReviewVerdict = {}));
+var CodeReviewSeverity;
+(function (CodeReviewSeverity) {
+    CodeReviewSeverity["MergeBlocking"] = "merge-blocking";
+    CodeReviewSeverity["Warning"] = "warning";
+})(CodeReviewSeverity || (exports.CodeReviewSeverity = CodeReviewSeverity = {}));

package/dist/agent/cua/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAStE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAmClC,MAAM,MAAM,kBAAkB,GAAG,KAAK,CAClC;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,YAAY,CAAC;IACnB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CACJ,CAAC;AAEF,wBAAsB,+BAA+B,CAAC,EACpD,IAAI,EACJ,IAAI,EACJ,KAAK,EACL,8BAAsC,GACvC,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,8BAA8B,CAAC,EAAE,OAAO,CAAC;CAC1C,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd,CAAC,CAuND"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AAStE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAoDlC,MAAM,MAAM,kBAAkB,GAAG,KAAK,CAClC;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,YAAY,CAAC;IACnB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CACJ,CAAC;AAEF,wBAAsB,+BAA+B,CAAC,EACpD,IAAI,EACJ,IAAI,EACJ,KAAK,EACL,8BAAsC,GACvC,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,8BAA8B,CAAC,EAAE,OAAO,CAAC;CAC1C,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd,CAAC,CAuND"}

package/dist/agent/cua/index.js

@@ -28,6 +28,22 @@ function artifact(screenshot, name) {
         data: Buffer.from(screenshot, "base64"),
     };
 }
+function stateOfTheBrowser(page) {
+    const browserContext = page.context();
+    const pages = browserContext.pages();
+    return `
+## Browser window
+
+### Current page (what you are working on)
+Current page URL: ${page.url()}
+Current page title: ${page.title()}
+
+### All pages
+Number of open pages: ${pages.length}
+
+URLs and titles:
+${pages.map((p) => `  - ${p.url()} - ${p.title()}`).join("\n")}`;
+}
 async function createTestUsingComputerUseAgent({ page, task, trace, prefersElementFromPointCodegen = false, }) {
     const codegen = await getCodegenInstance(prefersElementFromPointCodegen);
     await codegen.initialize(page);
@@ -53,7 +69,7 @@ async function createTestUsingComputerUseAgent({ page, task, trace, prefersEleme
                 content: [
                     {
                         type: "input_text",
-                        text: `Task to execute: ${task}\n\nCurrent page URL: ${page.url()}`,
+                        text: `Task to execute: ${task}\n\n${stateOfTheBrowser(page)}`,
                     },
                     {
                         type: "input_image",
@@ -177,7 +193,7 @@ async function createTestUsingComputerUseAgent({ page, task, trace, prefersEleme
                     content: [
                         {
                             type: "input_text",
-                            text: `Action executed: ${executedActionSummary || "None"}\nCurrent page URL: ${page.url()}`,
+                            text: `Action executed: ${executedActionSummary || "None"}\n\n${stateOfTheBrowser(page)}`,
                         },
                     ],
                 },

package/dist/agent/cua/model.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/model.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAEL,QAAQ,EACR,iBAAiB,EAClB,MAAM,0CAA0C,CAAC;AA8BlD,wBAAsB,oBAAoB,CAAC,EACzC,KAAK,EACL,kBAAkB,EAClB,WAAW,EACX,YAAY,EACZ,YAAY,GACb,EAAE;IACD,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAuBpB;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;;;EAQrE"}
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/model.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAEL,QAAQ,EACR,iBAAiB,EAClB,MAAM,0CAA0C,CAAC;AAiClD,wBAAsB,oBAAoB,CAAC,EACzC,KAAK,EACL,kBAAkB,EAClB,WAAW,EACX,YAAY,EACZ,YAAY,GACb,EAAE;IACD,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAuBpB;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;;;EAQrE"}

package/dist/agent/cua/model.js

@@ -9,7 +9,10 @@ For example, if the user message says "Click on Submit button", then
 you click on the submit button -- even if it looks like a scary action.
 
 If you have been asked to retrieve text or verify something on the UI, then communicate
-that in your responses so that the user can see your thinking process in its entirety.`;
+that in your responses so that the user can see your thinking process in its entirety.
+
+Your work is limited to the current browser page (tab) that you are provided with. You will
+have to conclude your actions before the user can ask you to do actions on different pages (tabs).`;
 const pageGotoTool = {
     type: "function",
     name: "page_goto",

package/dist/agent/cua/pw-codegen/pw-pause/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/agent/cua/pw-codegen/pw-pause/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAKjD,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AAE9E,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,MAAM,oBAwCvD;AAED,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,IAAI,CAAmB;IAC/B,OAAO,CAAC,MAAM,CAA4C;IAC1D,OAAO,CAAC,iBAAiB,CAAqB;;YAMhC,QAAQ;IAUhB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBrC,sBAAsB,CAAC,IAAI,EAAE,IAAI;IAqBjC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;CAU9C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/agent/cua/pw-codegen/pw-pause/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAMjD,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AAE9E,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,MAAM,oBAwCvD;AAED,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,IAAI,CAAmB;IAC/B,OAAO,CAAC,MAAM,CAA4C;IAC1D,OAAO,CAAC,iBAAiB,CAAqB;;YAMhC,QAAQ;IAUhB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBrC,sBAAsB,CAAC,IAAI,EAAE,IAAI;IAqBjC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;CAU9C"}

package/dist/agent/index.d.ts

@@ -0,0 +1,10 @@
+import type { AgentModeEnum } from "@empiricalrun/shared-types";
+import { type AgentParams, BaseAgent } from "./base";
+import { ChatAgent } from "./chat";
+import { CodeReviewAgent } from "./code-review";
+import { TriageAgent } from "./triage";
+import { VideoAnalysisAgent } from "./video-analysis";
+export declare const MODE_TO_AGENT_MAP: Record<AgentModeEnum, (params: AgentParams) => BaseAgent>;
+export { BaseAgent, ChatAgent, CodeReviewAgent, TriageAgent, VideoAnalysisAgent, };
+export type { AgentParams };
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAE,KAAK,WAAW,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,eAAO,MAAM,iBAAiB,EAAE,MAAM,CACpC,aAAa,EACb,CAAC,MAAM,EAAE,WAAW,KAAK,SAAS,CAMnC,CAAC;AAEF,OAAO,EACL,SAAS,EACT,SAAS,EACT,eAAe,EACf,WAAW,EACX,kBAAkB,GACnB,CAAC;AACF,YAAY,EAAE,WAAW,EAAE,CAAC"}

package/dist/agent/index.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VideoAnalysisAgent = exports.TriageAgent = exports.CodeReviewAgent = exports.ChatAgent = exports.BaseAgent = exports.MODE_TO_AGENT_MAP = void 0;
+const base_1 = require("./base");
+Object.defineProperty(exports, "BaseAgent", { enumerable: true, get: function () { return base_1.BaseAgent; } });
+const chat_1 = require("./chat");
+Object.defineProperty(exports, "ChatAgent", { enumerable: true, get: function () { return chat_1.ChatAgent; } });
+const code_review_1 = require("./code-review");
+Object.defineProperty(exports, "CodeReviewAgent", { enumerable: true, get: function () { return code_review_1.CodeReviewAgent; } });
+const triage_1 = require("./triage");
+Object.defineProperty(exports, "TriageAgent", { enumerable: true, get: function () { return triage_1.TriageAgent; } });
+const video_analysis_1 = require("./video-analysis");
+Object.defineProperty(exports, "VideoAnalysisAgent", { enumerable: true, get: function () { return video_analysis_1.VideoAnalysisAgent; } });
+exports.MODE_TO_AGENT_MAP = {
+    triage: (params) => new triage_1.TriageAgent(params),
+    chat: (params) => new chat_1.ChatAgent(params),
+    video: (params) => new video_analysis_1.VideoAnalysisAgent(params),
+    "code-review": (params) => new code_review_1.CodeReviewAgent(params),
+};

package/dist/agent/triage/index.d.ts

@@ -0,0 +1,7 @@
+import type { ToolsForLLM } from "@empiricalrun/shared-types";
+import { BaseAgent } from "../base";
+export declare class TriageAgent extends BaseAgent {
+    protected getTools(): ToolsForLLM;
+    protected buildSystemPrompt(repoContext?: string): Promise<string>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/triage/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/triage/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAoB9D,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC,qBAAa,WAAY,SAAQ,SAAS;IACxC,SAAS,CAAC,QAAQ,IAAI,WAAW;cA2BjB,iBAAiB,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAoEzE"}

package/dist/agent/triage/index.js

@@ -0,0 +1,103 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TriageAgent = void 0;
+const tools_1 = require("../../tools");
+const base_1 = require("../base");
+const test_case_def_1 = require("../chat/prompt/test-case-def");
+class TriageAgent extends base_1.BaseAgent {
+    getTools() {
+        const tools = [
+            // Common tools
+            tools_1.runTestTool,
+            tools_1.grepTool,
+            tools_1.fetchDiagnosisReportTool,
+            tools_1.listEnvironmentsTool,
+            tools_1.downloadBuildTool,
+            tools_1.fetchFileTool,
+            tools_1.traceDotZipTool,
+            // Triage specific tools
+            tools_1.listIssuesTool,
+            tools_1.createIssueTool,
+            tools_1.updateIssueTool,
+            tools_1.viewFailedTestRunReportTool,
+            tools_1.analyseVideo,
+            tools_1.fetchLastSuccessfulTestRunTool,
+            tools_1.sendTriageSummaryTool,
+            // Model-specific tools
+            ...(0, tools_1.textEditorToolsForModel)(this.selectedModel),
+        ];
+        return {
+            custom: tools,
+            builtInTextEditor: (0, tools_1.hasBuiltInTextEditor)(this.selectedModel),
+        };
+    }
+    async buildSystemPrompt(repoContext) {
+        if (!repoContext) {
+            throw new Error(`Triage agent needs repo context`);
+        }
+        return `
+You are a helpful assistant that help with analysis of Playwright test reports. Your goal is to help the user analyse a test report and identify the root cause of the test failures, and log the unique failuers as issues so that the user can keep a track and fix them.
+
+You are working on a test code repository that contains Playwright tests and other related files. Your working directory has been checked out on a git branch.
+
+# Your capabilities
+
+When provided with a test report URL, you can use these capabilities to triage the test failures in the report:
+
+## Fetch and view the test report
+
+- Use viewFailedTestRunReportTool tool to get more information about all tests that failed in the run
+
+## Analyze each test case
+
+You are provided with multiple tools to help you understand each failing test case better. Understanding each test case allows you to identify the root cause and create more accurate issues. These tools can also be called in parallel.
+
+- Each test case generates artifacts: images, videos, playwright trace zip file. With your tools, you can fetch image, analyze the video frames and trace.zip to find out failing network requests and console logs
+  - Each video represents one browser tab of the test case (so multiple videos implies the test had multiple tabs or browser windows)
+- Read the error stack and test file to understand what the test is doing
+- Fetch the last successful run of the test case to understand the earlier flow. This report will contain image and video URLs that can also be analyzed with your available tools.
+- If you think the issue is explained by a timing or intermittent issue, you can also re-run the test case
+
+## Listing, updating and creating issues
+
+- Test failures will become issues that can be assigned to developers to fix the app or update the test. Similar test failures should be grouped into one issue to avoid duplicates.
+- Before you create a new issue, you MUST list existing issues that have been created for this repo, to avoid creating duplicate issues.
+- If you find duplicates, use the update issue tool to update the existing issue with new information from the test report
+- When you are creating a new issue, use the description and title to clearly call out the error reason (share error stack, error message, relevant lines of code, etc.) so that a follow-up triaging session can match the issue against a new failure and avoid duplicate issues.
+- What makes a good issue: accurate classification between app or test issue, accurate grouping, and a good auto-fix prompt - see more about this below
+
+## Classify tests as app or test issues
+- An app issue is an issue in the application that is being tested. This often shows up as a network failure, or error message in the console log, or an error toast in the UI. Use the last successful run artifacts to compare the app state between the successful and failed run.
+- A test issue is an issue in the test code. If the application has changed the UI, a selector in the test may no longer work. Or if the application has changed the flow, the test may need to be updated to reflect the new flow.
+
+## Grouping test failures
+- Before you create issues, group the failures together so that we create useful issues
+- What makes a good group: failures that have the same root cause - because of similar error stacks - and can be fixed with the same change to the app or test
+  - Both "type of failure" and "proposed fix" are important to determine if two failures belong to the same group
+  - Example: if two tests fail with strict mode violations, but for 2 different selectors, they are different groups because the proposed fixes are different
+- What does not make a good group: the location of the test or the name of the test. Two tests that are located in the same file or have similar names should ONLY be grouped together if the root cause of failures is same
+
+## Crafting a good auto-fix prompt for test issues
+- When you create issues with type "test", you are expected to share a test_issue_prompt which is your proposed change to the test to adapt to the new app state.
+- This prompt is handed over to another agent to update the test code, and your prompt is the ONLY context that the agent has to update the test.
+- Therefore, your prompt must contain:
+  - Which test cases to be updated - with test and describe block names, file name
+  - What failed in the test - error message, error stack, relevant lines of code, or bits to locate the failure
+  - Your suggested change to the test
+  - Emphasis to re-run the test after making the change, to ensure that the change works
+
+## Conclusion
+- After you are done with triaging and creating issues, summarize the work done with a list of created issues for the user to review. Don't be too verbose - a bullet list of issues created or updated, with a small description is enough.
+- It is important to show proof that you have gone through all of the failures in the test run report, so use numbers to call out 1. total failures, and 2. failures associated with each issue.
+
+${test_case_def_1.testCasesDefinitionPrompt}
+
+# Repo context
+${repoContext}
+
+# Reference
+Today's date is ${new Date().toDateString()}
+`;
+    }
+}
+exports.TriageAgent = TriageAgent;

package/dist/agent/video-analysis/executor/index.d.ts

@@ -0,0 +1,5 @@
+import { BaseToolExecutor, BaseToolExecutorProps } from "../../../tools/executor/base";
+export declare class VideoAnalysisToolExecutor extends BaseToolExecutor {
+    constructor(params: Omit<BaseToolExecutorProps, "tools">);
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/video-analysis/executor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/agent/video-analysis/executor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,qBAAqB,EACtB,MAAM,8BAA8B,CAAC;AAEtC,qBAAa,yBAA0B,SAAQ,gBAAgB;gBACjD,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,OAAO,CAAC;CAGzD"}

package/dist/agent/video-analysis/executor/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VideoAnalysisToolExecutor = void 0;
+const base_1 = require("../../../tools/executor/base");
+class VideoAnalysisToolExecutor extends base_1.BaseToolExecutor {
+    constructor(params) {
+        super({ ...params, tools: [] });
+    }
+}
+exports.VideoAnalysisToolExecutor = VideoAnalysisToolExecutor;

package/dist/agent/video-analysis/index.d.ts

@@ -0,0 +1,7 @@
+import type { ToolsForLLM } from "@empiricalrun/shared-types";
+import { BaseAgent } from "../base";
+export declare class VideoAnalysisAgent extends BaseAgent {
+    protected getTools(): ToolsForLLM;
+    protected buildSystemPrompt(): Promise<string>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/video-analysis/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/video-analysis/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC,qBAAa,kBAAmB,SAAQ,SAAS;IAC/C,SAAS,CAAC,QAAQ,IAAI,WAAW;cAOjB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;CA+CrD"}

package/dist/agent/video-analysis/index.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VideoAnalysisAgent = void 0;
+const base_1 = require("../base");
+class VideoAnalysisAgent extends base_1.BaseAgent {
+    getTools() {
+        return {
+            custom: [],
+            builtInTextEditor: false,
+        };
+    }
+    async buildSystemPrompt() {
+        return `
+You are a video analysis agent specialized in analyzing screen recordings and user interface interactions.
+
+You will receive individual video frames with their Frame IDs as user input for detailed visual analysis.
+
+When analyzing the provided frames:
+1. Analyze each frame for UI elements, user actions, and state changes
+2. Provide specific observations about what's happening in each frame
+3. The Summary should be in the given XML format
+
+Your analysis should be:
+- Detailed and specific about UI elements and interactions
+- Sequential, following the flow of actions in the video
+
+CRITICAL: You MUST use the EXACT frame IDs that are provided with each frame. Each frame will be labeled with text like "Frame ID: frame_000000" - use this exact ID in your <key_frame> tags.
+
+Note: The Last frame from the attachments should always be included in the <key_frame> tag
+
+# Output format
+<summary>
+    <section>
+      <key_frame>frame_id</key_frame>
+      <description>text description of the frame</description>
+    </section>
+    <section>
+      <key_frame>frame_id</key_frame>
+      <description>text description of the frame</description>
+    </section>
+    .
+    .
+    .
+    <section>
+      <key_frame>frame_id</key_frame>
+      <description>text description of the frame</description>
+    </section>
+</summary>
+
+## Example
+If the attachments include "frame_000000.png", "frame_000078.png", and "frame_000156.png", then:
+- To reference the first frame, use: <key_frame>frame_000000</key_frame>
+- To reference the second frame, use: <key_frame>frame_000078</key_frame>
+- To reference the third frame, use: <key_frame>frame_000156</key_frame>
+
+WRONG: <key_frame>frame_000001</key_frame> (unless there's actually a file named frame_000001.png in the attachments)
+  `;
+    }
+}
+exports.VideoAnalysisAgent = VideoAnalysisAgent;

package/dist/artifacts/index.d.ts

@@ -46,7 +46,7 @@ export declare class UploadArtifactsQueue {
     private artifactResults;
     private uploadPromise;
     constructor(baseRepoPath: string, toolCallId: string);
-    addTask(artifacts: ArtifactInput[]): Promise<void>;
+    addTask(artifacts: ArtifactInput[]): Promise<Artifact[]>;
     waitForCompletion(): Promise<Artifact[]>;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/artifacts/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,QAAQ,EACR,aAAa,EAId,MAAM,4BAA4B,CAAC;AAwBpC,wBAAgB,2BAA2B,uBAM1C;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,EAAE,EACvB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAsErB;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,aAAa,CAA8B;gBAEvC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAKvC,OAAO,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAelD,iBAAiB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;CAMtD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,QAAQ,EACR,aAAa,EAId,MAAM,4BAA4B,CAAC;AAwBpC,wBAAgB,2BAA2B,uBAM1C;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,EAAE,EACvB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAsErB;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,aAAa,CAAoC;gBAE7C,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAK7C,OAAO,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAiBlD,iBAAiB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;CAMtD"}