@empiricalrun/test-gen 0.75.0 → 0.77.0

package/dist/video-core/xml-parser.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processAnalysisAndLoadFrames = processAnalysisAndLoadFrames;
+const utils_1 = require("./utils");
+function extractKeyFramesFromAnalysis(analysisData) {
+    const keyFrames = [];
+    if (analysisData && Array.isArray(analysisData)) {
+        analysisData.forEach((section) => {
+            if (section.key_frame && typeof section.key_frame === "string") {
+                keyFrames.push(section.key_frame);
+            }
+        });
+    }
+    return keyFrames;
+}
+async function processAnalysisAndLoadFrames({ rawAnalysis, artifactsPath, }) {
+    const parsedXml = (0, utils_1.parseXmlSummaryToJson)(rawAnalysis);
+    const keyFrames = extractKeyFramesFromAnalysis(parsedXml);
+    let keyFrameImagesMap = new Map();
+    if (keyFrames.length > 0) {
+        try {
+            keyFrameImagesMap = await (0, utils_1.loadKeyFrameImages)(keyFrames, artifactsPath);
+        }
+        catch (error) { }
+    }
+    return { parsedXml, keyFrameImagesMap };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.75.0",
+  "version": "0.77.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -62,7 +62,6 @@
     "date-fns": "^3.6.0",
     "detect-port": "^1.6.1",
     "dotenv": "^16.4.5",
-    "eslint": "^9.30.0",
     "express": "^4.19.2",
     "ignore": "^5.3.1",
     "inquirer": "^12.4.2",
@@ -85,7 +84,7 @@
     "yauzl": "^3.1.3",
     "zod": "^3.23.8",
     "@empiricalrun/ast-parser": "^0.0.10",
-    "@empiricalrun/llm": "^0.22.0",
+    "@empiricalrun/llm": "^0.23.0",
     "@empiricalrun/r2-uploader": "^0.3.11",
     "@empiricalrun/test-run": "^0.11.1"
   },
@@ -106,13 +105,13 @@
     "playwright": "1.53.2",
     "serve-handler": "^6.1.6",
     "ts-patch": "^3.3.0",
-    "@empiricalrun/shared-types": "0.10.1"
+    "@empiricalrun/shared-types": "0.12.0"
   },
   "scripts": {
     "dev": "tspc --build --watch",
-    "build": "tspc --build && cp -r src/browser-injected-scripts dist",
+    "build": "tspc --build && cp -r src/browser-injected-scripts dist && cp src/video-core/analysis-viewer.html dist/video-core/",
     "clean": "tspc --build --clean",
-    "lint": "eslint .",
+    "lint": "biome check --unsafe",
     "test": "vitest run",
     "test-browser": "pnpm build && RUNNING_BROWSER_TESTS_FOR_TEST_GEN=1 npx playwright test",
     "test:watch": "vitest",

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/index.ts","./src/logger.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/skill.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/base/index.ts","./src/agent/browsing/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/filesystem-cache.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/chat/prompt/index.ts","./src/agent/chat/prompt/pw-utils-docs.ts","./src/agent/chat/prompt/repo.ts","./src/agent/chat/utils/tool-calls.ts","./src/agent/code-review/prompt.ts","./src/agent/codegen/create-test-block.ts","./src/agent/codegen/fix-ts-errors.ts","./src/agent/codegen/lexical-scoped-vars.ts","./src/agent/codegen/skills-retriever.ts","./src/agent/codegen/test-update-feedback.ts","./src/agent/codegen/types.ts","./src/agent/codegen/update-flow.ts","./src/agent/codegen/use-skill.ts","./src/agent/codegen/utils.ts","./src/agent/cua/computer.ts","./src/agent/cua/index.ts","./src/agent/cua/model.ts","./src/agent/cua/pw-codegen/element-from-point.ts","./src/agent/cua/pw-codegen/types.ts","./src/agent/cua/pw-codegen/pw-pause/for-recorder.ts","./src/agent/cua/pw-codegen/pw-pause/index.ts","./src/agent/cua/pw-codegen/pw-pause/ipc.ts","./src/agent/cua/pw-codegen/pw-pause/patch.ts","./src/agent/cua/pw-codegen/pw-pause/types.ts","./src/agent/cua/pw-codegen/pw-pause/utils.ts","./src/agent/diagnosis-agent/index.ts","./src/agent/diagnosis-agent/strict-mode-violation.ts","./src/agent/master/action-tool-calls.ts","./src/agent/master/element-annotation.ts","./src/agent/master/execute-browser-action.ts","./src/agent/master/execute-skill-action.ts","./src/agent/master/next-action.ts","./src/agent/master/planner.ts","./src/agent/master/run.ts","./src/agent/master/scroller.ts","./src/agent/master/with-hints.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/master/browser-tests/skills.spec.ts","./src/agent/master/icon-descriptor/index.ts","./src/agent/master/icon-descriptor/normalize-svg.ts","./src/agent/planner/run-time-planner.ts","./src/agent/planner/run.ts","./src/artifacts/index.ts","./src/artifacts/utils.ts","./src/auth/cli-auth.ts","./src/auth/index.ts","./src/auth/token-store.ts","./src/bin/environments.ts","./src/bin/index.ts","./src/bin/setup.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/browser-injected-scripts/annotate-elements.spec.ts","./src/constants/index.ts","./src/dashboard/client.ts","./src/dashboard/index.ts","./src/dashboard/totp.ts","./src/dashboard/types.ts","./src/errors/index.ts","./src/evals/append-create-test-agent.evals.ts","./src/evals/fetch-pom-skills-agent.evals.ts","./src/evals/master-agent.evals.ts","./src/evals/type.ts","./src/evals/update-scenario-agent.evals.ts","./src/file/client.ts","./src/file/server.ts","./src/file-info/index.ts","./src/file-info/adapters/file-system/index.ts","./src/file-info/adapters/file-system/reader.ts","./src/file-info/adapters/github/index.ts","./src/file-info/adapters/github/reader.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/recorder/env-variables.ts","./src/recorder/index.ts","./src/recorder/request.ts","./src/recorder/temp-files.ts","./src/recorder/upload.ts","./src/recorder/validation.ts","./src/test-build/index.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/index.ts","./src/tools/list-environments.ts","./src/tools/run-test.ts","./src/tools/test-gen-browser.ts","./src/tools/commit-and-create-pr/index.ts","./src/tools/definitions/commit-and-create-pr.ts","./src/tools/definitions/delete-file.ts","./src/tools/definitions/download-build.ts","./src/tools/definitions/fetch-video-analysis.ts","./src/tools/definitions/grep.ts","./src/tools/definitions/list-tests-and-projects.ts","./src/tools/definitions/merge-conflicts.ts","./src/tools/definitions/rename-file.ts","./src/tools/definitions/run-test.ts","./src/tools/definitions/str_replace_editor.ts","./src/tools/definitions/test-gen-browser.ts","./src/tools/definitions/upgrade-packages.ts","./src/tools/definitions/utils.ts","./src/tools/delete-file/index.ts","./src/tools/download-build/index.ts","./src/tools/executor/index.ts","./src/tools/executor/utils/checkpoint.ts","./src/tools/executor/utils/git.ts","./src/tools/executor/utils/index.ts","./src/tools/executor/utils/pr-description.ts","./src/tools/fetch-file/index.ts","./src/tools/fetch-last-successful-test-run/index.ts","./src/tools/fetch-video-analysis/index.ts","./src/tools/fetch-video-analysis/local-ffmpeg-client.ts","./src/tools/fetch-video-analysis/open-ai.ts","./src/tools/fetch-video-analysis/utils.ts","./src/tools/fetch-video-analysis/video-analysis.ts","./src/tools/file-operations/create.ts","./src/tools/file-operations/index.ts","./src/tools/file-operations/insert.ts","./src/tools/file-operations/replace.ts","./src/tools/file-operations/shared/git-helper.ts","./src/tools/file-operations/shared/helpers.ts","./src/tools/file-operations/view/index.ts","./src/tools/grep/index.ts","./src/tools/grep/types.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/issues/create-issue.ts","./src/tools/issues/index.ts","./src/tools/issues/list-issues.ts","./src/tools/issues/metadata-schema.ts","./src/tools/issues/update-issue.ts","./src/tools/issues/utils.ts","./src/tools/list-tests-and-projects/index.ts","./src/tools/merge-conflicts/index.ts","./src/tools/rename-file/index.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/test-run-fetcher/types.ts","./src/tools/trace-dot-zip/index.ts","./src/tools/trace-dot-zip/types.ts","./src/tools/trace-dot-zip/utils/console-trace.ts","./src/tools/trace-dot-zip/utils/extract-zip.ts","./src/tools/trace-dot-zip/utils/network-trace.ts","./src/tools/triage-summary/index.ts","./src/tools/triage-summary/types.ts","./src/tools/triage-summary/utils.ts","./src/tools/upgrade-packages/index.ts","./src/tools/upgrade-packages/utils.ts","./src/tools/utils/queue.ts","./src/tools/view-failed-test-run-report/index.ts","./src/trace-utils/index.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/utils/SQSClient.ts","./src/utils/dedup-image-fs.ts","./src/utils/dedup-image.ts","./src/utils/env.ts","./src/utils/exec.ts","./src/utils/file.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/json.ts","./src/utils/model.ts","./src/utils/playwright-report-parser.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts"],"version":"5.8.3"}
+{"root":["./src/index.ts","./src/logger.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/skill.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/cli.ts","./src/agent/index.ts","./src/agent/base/index.ts","./src/agent/browsing/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/filesystem-cache.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/chat/prompt/pw-utils-docs.ts","./src/agent/chat/prompt/repo.ts","./src/agent/chat/prompt/test-case-def.ts","./src/agent/code-review/index.ts","./src/agent/code-review/parser.ts","./src/agent/code-review/types.ts","./src/agent/code-review/executor/index.ts","./src/agent/codegen/create-test-block.ts","./src/agent/codegen/fix-ts-errors.ts","./src/agent/codegen/lexical-scoped-vars.ts","./src/agent/codegen/skills-retriever.ts","./src/agent/codegen/test-update-feedback.ts","./src/agent/codegen/types.ts","./src/agent/codegen/update-flow.ts","./src/agent/codegen/use-skill.ts","./src/agent/codegen/utils.ts","./src/agent/cua/computer.ts","./src/agent/cua/index.ts","./src/agent/cua/model.ts","./src/agent/cua/pw-codegen/element-from-point.ts","./src/agent/cua/pw-codegen/types.ts","./src/agent/cua/pw-codegen/pw-pause/for-recorder.ts","./src/agent/cua/pw-codegen/pw-pause/index.ts","./src/agent/cua/pw-codegen/pw-pause/ipc.ts","./src/agent/cua/pw-codegen/pw-pause/patch.ts","./src/agent/cua/pw-codegen/pw-pause/types.ts","./src/agent/cua/pw-codegen/pw-pause/utils.ts","./src/agent/master/action-tool-calls.ts","./src/agent/master/element-annotation.ts","./src/agent/master/execute-browser-action.ts","./src/agent/master/execute-skill-action.ts","./src/agent/master/next-action.ts","./src/agent/master/planner.ts","./src/agent/master/run.ts","./src/agent/master/scroller.ts","./src/agent/master/with-hints.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/master/browser-tests/skills.spec.ts","./src/agent/master/icon-descriptor/index.ts","./src/agent/master/icon-descriptor/normalize-svg.ts","./src/agent/planner/run-time-planner.ts","./src/agent/planner/run.ts","./src/agent/triage/index.ts","./src/agent/video-analysis/index.ts","./src/agent/video-analysis/executor/index.ts","./src/artifacts/index.ts","./src/artifacts/utils.ts","./src/auth/cli-auth.ts","./src/auth/index.ts","./src/auth/token-store.ts","./src/bin/environments.ts","./src/bin/index.ts","./src/bin/setup.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/browser-injected-scripts/annotate-elements.spec.ts","./src/constants/index.ts","./src/dashboard/client.ts","./src/dashboard/index.ts","./src/dashboard/totp.ts","./src/dashboard/types.ts","./src/errors/index.ts","./src/evals/append-create-test-agent.evals.ts","./src/evals/fetch-pom-skills-agent.evals.ts","./src/evals/master-agent.evals.ts","./src/evals/type.ts","./src/evals/update-scenario-agent.evals.ts","./src/file/client.ts","./src/file/server.ts","./src/file-info/index.ts","./src/file-info/adapters/file-system/index.ts","./src/file-info/adapters/file-system/reader.ts","./src/file-info/adapters/github/index.ts","./src/file-info/adapters/github/reader.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/recorder/env-variables.ts","./src/recorder/index.ts","./src/recorder/request.ts","./src/recorder/temp-files.ts","./src/recorder/upload.ts","./src/recorder/validation.ts","./src/test-build/index.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/index.ts","./src/tools/list-environments.ts","./src/tools/run-test.ts","./src/tools/test-gen-browser.ts","./src/tools/analyse-video/index.ts","./src/tools/create-pull-request/index.ts","./src/tools/create-pull-request/utils.ts","./src/tools/definitions/analyse-video.ts","./src/tools/definitions/delete-file.ts","./src/tools/definitions/download-build.ts","./src/tools/definitions/grep.ts","./src/tools/definitions/list-tests-and-projects.ts","./src/tools/definitions/merge-conflicts.ts","./src/tools/definitions/rename-file.ts","./src/tools/definitions/review-pull-request.ts","./src/tools/definitions/run-test.ts","./src/tools/definitions/str_replace_editor.ts","./src/tools/definitions/test-gen-browser.ts","./src/tools/definitions/upgrade-packages.ts","./src/tools/definitions/utils.ts","./src/tools/delete-file/index.ts","./src/tools/download-build/index.ts","./src/tools/executor/base.ts","./src/tools/executor/index.ts","./src/tools/executor/utils/checkpoint.ts","./src/tools/executor/utils/git.ts","./src/tools/executor/utils/index.ts","./src/tools/executor/utils/pr-description.ts","./src/tools/fetch-file/index.ts","./src/tools/fetch-last-successful-test-run/index.ts","./src/tools/fetch-session-diff/index.ts","./src/tools/file-operations/create.ts","./src/tools/file-operations/index.ts","./src/tools/file-operations/insert.ts","./src/tools/file-operations/replace.ts","./src/tools/file-operations/shared/helpers.ts","./src/tools/file-operations/view/index.ts","./src/tools/grep/index.ts","./src/tools/grep/types.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/issues/create-issue.ts","./src/tools/issues/index.ts","./src/tools/issues/list-issues.ts","./src/tools/issues/metadata-schema.ts","./src/tools/issues/update-issue.ts","./src/tools/issues/utils.ts","./src/tools/list-tests-and-projects/index.ts","./src/tools/merge-conflicts/index.ts","./src/tools/rename-file/index.ts","./src/tools/review-pull-request/index.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/test-run-fetcher/types.ts","./src/tools/trace-dot-zip/index.ts","./src/tools/trace-dot-zip/types.ts","./src/tools/trace-dot-zip/utils/console-trace.ts","./src/tools/trace-dot-zip/utils/extract-zip.ts","./src/tools/trace-dot-zip/utils/network-trace.ts","./src/tools/triage-summary/index.ts","./src/tools/triage-summary/types.ts","./src/tools/triage-summary/utils.ts","./src/tools/upgrade-packages/index.ts","./src/tools/upgrade-packages/utils.ts","./src/tools/utils/queue.ts","./src/tools/utils/urls.ts","./src/tools/view-failed-test-run-report/index.ts","./src/trace-utils/index.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/utils/SQSClient.ts","./src/utils/artifact-paths.ts","./src/utils/dedup-image-fs.ts","./src/utils/dedup-image.ts","./src/utils/env.ts","./src/utils/exec.ts","./src/utils/file.ts","./src/utils/find-threshold.ts","./src/utils/hash.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/json.ts","./src/utils/model.ts","./src/utils/playwright-report-parser.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts","./src/utils/ffmpeg/index.ts","./src/video-core/agent-orchestrator.ts","./src/video-core/analysis-server.ts","./src/video-core/index.ts","./src/video-core/model-limits.ts","./src/video-core/storage-manager.ts","./src/video-core/types.ts","./src/video-core/utils.ts","./src/video-core/xml-parser.ts"],"version":"5.8.3"}

package/dist/agent/chat/prompt/index.d.ts

@@ -1,5 +0,0 @@
-import { FileInfo } from "@empiricalrun/shared-types";
-export declare const getSystemPrompt: (repoInfo: FileInfo, mode?: string) => Promise<string>;
-export declare function buildTriageSystemPrompt(repoContext: string): Promise<string>;
-export declare function buildSystemPrompt(repoContext: string): Promise<string>;
-//# sourceMappingURL=index.d.ts.map

package/dist/agent/chat/prompt/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/agent/chat/prompt/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAKtD,eAAO,MAAM,eAAe,GAAU,UAAU,QAAQ,EAAE,OAAO,MAAM,oBAMtE,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,WAAW,EAAE,MAAM,mBA8DhE;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,mBA8G1D"}

package/dist/agent/chat/prompt/index.js

@@ -1,189 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getSystemPrompt = void 0;
-exports.buildTriageSystemPrompt = buildTriageSystemPrompt;
-exports.buildSystemPrompt = buildSystemPrompt;
-const pw_utils_docs_1 = require("./pw-utils-docs");
-const repo_1 = require("./repo");
-const getSystemPrompt = async (repoInfo, mode) => {
-    const repoContext = await (0, repo_1.getRepoInfoPrompt)(repoInfo);
-    if (mode === "triage") {
-        return await buildTriageSystemPrompt(repoContext);
-    }
-    return await buildSystemPrompt(repoContext);
-};
-exports.getSystemPrompt = getSystemPrompt;
-async function buildTriageSystemPrompt(repoContext) {
-    return `
-You are a helpful assistant that help with analysis of Playwright test reports. Your goal is to help the user analyse a test report and identify the root cause of the test failures, and log the unique failuers as issues so that the user can keep a track and fix them.
-
-You are working on a test code repository that contains Playwright tests and other related files. Your working directory has been checked out on a git branch.
-
-# Your capabilities
-
-When provided with a test report URL, you can use these capabilities to triage the test failures in the report:
-
-## Fetch and view the test report
-
-- Use viewFailedTestRunReportTool tool to get more information about all tests that failed in the run
-
-## Analyze each test case
-
-You are provided with multiple tools to help you understand each failing test case better. Understanding each test case allows you to identify the root cause and create more accurate issues. These tools can also be called in parallel.
-
-- Each test case generates artifacts: images, videos, playwright trace zip file. With your tools, you can fetch image, analyze the video frames and trace.zip to find out failing network requests and console logs
-  - Each video represents one browser tab of the test case (so multiple videos implies the test had multiple tabs or browser windows)
-- Read the error stack and test file to understand what the test is doing
-- Fetch the last successful run of the test case to understand the earlier flow. This report will contain image and video URLs that can also be analyzed with your available tools.
-- If you think the issue is explained by a timing or intermittent issue, you can also re-run the test case
-
-## Listing, updating and creating issues
-
-- Test failures will become issues that can be assigned to developers to fix the app or update the test. Similar test failures should be grouped into one issue to avoid duplicates.
-- Before you create a new issue, you MUST list existing issues that have been created for this repo, to avoid creating duplicate issues.
-- If you find duplicates, use the update issue tool to update the existing issue with new information from the test report
-- When you are creating a new issue, use the description and title to clearly call out the error reason (share error stack, error message, relevant lines of code, etc.) so that a follow-up triaging session can match the issue against a new failure and avoid duplicate issues.
-- What makes a good issue: accurate classification between app or test issue, accurate grouping, and a good auto-fix prompt - see more about this below
-
-## Classify tests as app or test issues
-- An app issue is an issue in the application that is being tested. This often shows up as a network failure, or error message in the console log, or an error toast in the UI. Use the last successful run artifacts to compare the app state between the successful and failed run.
-- A test issue is an issue in the test code. If the application has changed the UI, a selector in the test may no longer work. Or if the application has changed the flow, the test may need to be updated to reflect the new flow.
-
-## Grouping test failures
-- Before you create issues, group the failures together so that we create useful issues
-- What makes a good group: failures that have the same root cause - because of similar error stacks - and can be fixed with the same change to the app or test
-  - Both "type of failure" and "proposed fix" are important to determine if two failures belong to the same group
-  - Example: if two tests fail with strict mode violations, but for 2 different selectors, they are different groups because the proposed fixes are different
-- What does not make a good group: the location of the test or the name of the test. Two tests that are located in the same file or have similar names should ONLY be grouped together if the root cause of failures is same
-
-## Crafting a good auto-fix prompt for test issues
-- When you create issues with type "test", you are expected to share a test_issue_prompt which is your proposed change to the test to adapt to the new app state.
-- This prompt is handed over to another agent to update the test code, and your prompt is the ONLY context that the agent has to update the test.
-- Therefore, your prompt must contain:
-  - Which test cases to be updated - with test and describe block names, file name
-  - What failed in the test - error message, error stack, relevant lines of code, or bits to locate the failure
-  - Your suggested change to the test
-  - Emphasis to re-run the test after making the change, to ensure that the change works
-
-## Conclusion
-- After you are done with triaging and creating issues, summarize the work done with a list of created issues for the user to review. Don't be too verbose - a bullet list of issues created or updated, with a small description is enough.
-- It is important to show proof that you have gone through all of the failures in the test run report, so use numbers to call out 1. total failures, and 2. failures associated with each issue.
-
-# Repo context
-${repoContext}
-
-# Reference
-Today's date is ${new Date().toDateString()}
-`;
-}
-async function buildSystemPrompt(repoContext) {
-    const preamble = `
-You are a helpful assistant that can answer questions and help with tasks related to writing and maintaining Playwright tests.
-
-You are working on a test code repository that contains Playwright tests and other related files. Your working directory
-has been checked out on a git branch, and your actions that edit files will be automatically committed. If you make any
-file edits, you must create a pull request after your work is done, to get your work reviewed and merged by a human.
-
-# Your capabilities
-
-- Adding new Playwright tests or helper methods
-- Going through test reports and identifying app issues versus test issues
-- Modifying existing tests to adapt to changes in the application
-- Modifying repo configuration (e.g. in playwright.config.ts) and dependencies (e.g. in package.json)
-
-# Going through test reports
-
-- App issues: app issues caught by test failures, like UI issues, API endpoint issues, etc. These are issues that
-will be reported to an app developer to investigate and fix.
-- Test issues: Playwright tests can become outdated when app code changes. These are issues that need to be
-fixed with modifications to the test code, and it is your job to do that.
-
-# Tools
-
-You are given a set of tools to help you fulfill the user's request. Read their descriptions to
-understand what each tool does.
-
-For example, if the user asks you to run a test, you could use the runTest tool.
-Once the test is run, you will receive the results in the form of a JSON object.
-Summarize the results in a few sentences.
-
-If the user provides a diagnosis URL, you can use the fetchDiagnosisDetails tool
-to get more information about the test case and its results.
-
-If the user provides a test run URL, you can use the fetchTestRunDetails tool
-to get detailed information about a specific test run.
-
-Or if the user asks you to modify a test, you could use the generateTestWithBrowserAgent tool. If you suspect
-that a UI selector needs to be updated, using the browser agent is a good idea.
-
-Before using generateTestWithBrowserAgent, you need to prepare the test code for the browser agent.
-You can do this by using the strReplaceEditor or the text editor tool to add a TODO comment to the test
-code. This comment explains to the browser agent what it needs to do.
-
-For example, if the expected modification is to click on a login button, you could add the following comment. 
-
-// TODO(agent): Click on the login button
-
-The position of the comment is important: the browser agent will look for this comment and replace it with 
-the actual code to click on the login button. If you are fixing a failing test, your comment should be 
-around the failing line of code, so that it can be replaced/modified.
-
-# Proactiveness
-
-1. You are allowed to be proactive, but ONLY for read-only tool calls: like searching for content, reading files, fetching data from tools, and
-running Playwright tests.
-2. For any read-write tool calls (e.g. modifying any file), you should share your plan and get the user's approval before proceeding.
-
-# Rules to follow
-
-You must follow these rules while adding new tests or modifying existing tests. There can be exceptions to these rules, but
-ONLY when explicitly asked for by the user.
-
-1. You can't delete some steps from the test to make it pass. The test needs to accomplish its objective (which is to validate a particular user scenario)
-2. Do not add any conditional logic or try catch blocks in a test. A good test deterministically tests a user scenario
-3. Trust Playwright's ability to auto-wait while taking actions on elements.
-   - Example 1: Do not add checks on locator.isVisible() before clicking on it: Playwright already waits for visibility on locator.click()
-   - Example 2: Do not add page.waitForLoadState after a page.goto: Playwright already waits for page "load" event in page.goto()
-4. Do not add waitForTimeout or waitForLoadState in a test. Playwright will automatically wait for the page to load.
-5. Try/catch blocks are a code smell for tests: you should not use them.
-6. Do not use then() or catch() syntax in a test. Use async/await only
-
-There are few exceptions to these rules. BEFORE applying any of the following exceptions, you MUST share your plan with the user and get their approval.
-
-## Exceptions for conditional logic
-
-There are few exceptions where you can add conditional logic to a test. If the application UI reveals some UI elements on certain conditions, we can add conditional logic.
-
-For example, a form view shows a "Save" button only when the form is dirty. In this case, we will have to check if the "Save" button is visible before clicking on it. To do this,
-follow this pattern:
-
-\`\`\`
-const saveButton = page.getByRole('button', { name: 'Save' });
-if (await saveButton.isVisible()) {
-  await saveButton.click();
-}
-\`\`\`
-
-Note that locator.isVisible() DOES NOT wait for the element to be visible. If the element in question shows up after a delay, we have no option but to add a waitForTimeout.
-
-\`\`\`
-const saveButton = page.getByRole('button', { name: 'Save' });
-await page.waitForTimeout(100); // Wait for the element to be visible -- only if necessary.
-if (await saveButton.isVisible()) {
-  await saveButton.click();
-}
-\`\`\`
-
-`;
-    return `${preamble}
-
-# Recipes
-${pw_utils_docs_1.playwrightUtilsDocs}
-
-# Repo context
-${repoContext}
-
-# Reference
-Today's date is ${new Date().toDateString()}
-`;
-}

package/dist/agent/chat/utils/tool-calls.d.ts

@@ -1,21 +0,0 @@
-import { IChatModel } from "@empiricalrun/llm/chat";
-import type { ChatState, PendingToolCall, SupportedChatModels, ToolResult } from "@empiricalrun/shared-types";
-export declare function rejectPendingToolCalls({ chatState, chatModel, }: {
-    chatState: ChatState;
-    chatModel: IChatModel<any>;
-}): ChatState;
-export declare function filterPendingToolCalls({ toolCalls, toolResults, pendingToolCalls, }: {
-    toolCalls: PendingToolCall[];
-    toolResults: ToolResult[];
-    pendingToolCalls: PendingToolCall[];
-}): {
-    filteredToolCalls: PendingToolCall[];
-    filteredToolResults: ToolResult[];
-};
-export declare function processToolCallsAndUpdateChatState({ toolCalls, toolResults, chatModel, selectedModel, }: {
-    toolCalls: PendingToolCall[];
-    toolResults: ToolResult[];
-    chatModel: IChatModel<any>;
-    selectedModel: SupportedChatModels;
-}): ChatState | undefined;
-//# sourceMappingURL=tool-calls.d.ts.map

package/dist/agent/chat/utils/tool-calls.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tool-calls.d.ts","sourceRoot":"","sources":["../../../../src/agent/chat/utils/tool-calls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,UAAU,EACX,MAAM,4BAA4B,CAAC;AAOpC,wBAAgB,sBAAsB,CAAC,EACrC,SAAS,EACT,SAAS,GACV,EAAE;IACD,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;CAC5B,GAAG,SAAS,CA8BZ;AAED,wBAAgB,sBAAsB,CAAC,EACrC,SAAS,EACT,WAAW,EACX,gBAAgB,GACjB,EAAE;IACD,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,gBAAgB,EAAE,eAAe,EAAE,CAAC;CACrC;;;EAgBA;AAED,wBAAgB,kCAAkC,CAAC,EACjD,SAAS,EACT,WAAW,EACX,SAAS,EACT,aAAa,GACd,EAAE;IACD,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IAC3B,aAAa,EAAE,mBAAmB,CAAC;CACpC,GAAG,SAAS,GAAG,SAAS,CAkBxB"}

package/dist/agent/chat/utils/tool-calls.js

@@ -1,64 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.rejectPendingToolCalls = rejectPendingToolCalls;
-exports.filterPendingToolCalls = filterPendingToolCalls;
-exports.processToolCallsAndUpdateChatState = processToolCallsAndUpdateChatState;
-const state_1 = require("../state");
-const TOOL_CALL_REJECTED_BY_USER_ERROR = "This tool call was rejected by the user.";
-function rejectPendingToolCalls({ chatState, chatModel, }) {
-    const pendingToolCalls = chatModel.getPendingToolCalls();
-    let updatedChatState = {
-        ...chatState,
-        // When tool calls are rejected, we reset askUserForInput to true
-        // so that the user can provide input for the next tool call. The agent
-        // loop is not run again until the user provides input.
-        askUserForInput: true,
-    };
-    if (pendingToolCalls.length > 0) {
-        const toolResults = pendingToolCalls.map((toolCall) => {
-            console.log(`Rejecting pending tool calls with toolCallId: ${toolCall.id} `);
-            return {
-                isError: true,
-                result: TOOL_CALL_REJECTED_BY_USER_ERROR,
-            };
-        });
-        chatModel.pushToolResultsMessage(pendingToolCalls, toolResults);
-        return {
-            ...updatedChatState,
-            messages: chatModel.messages,
-        };
-    }
-    return updatedChatState;
-}
-function filterPendingToolCalls({ toolCalls, toolResults, pendingToolCalls, }) {
-    const filteredToolResults = [];
-    const filteredToolCalls = toolCalls.filter((call, index) => {
-        if (pendingToolCalls.some((p) => p.id === call.id)) {
-            filteredToolResults.push(toolResults[index]);
-            return true;
-        }
-        return false;
-    });
-    return {
-        filteredToolCalls,
-        filteredToolResults,
-    };
-}
-function processToolCallsAndUpdateChatState({ toolCalls, toolResults, chatModel, selectedModel, }) {
-    const pendingToolCalls = chatModel.getPendingToolCalls() || [];
-    const { filteredToolCalls, filteredToolResults } = filterPendingToolCalls({
-        toolCalls,
-        toolResults,
-        pendingToolCalls,
-    });
-    if (filteredToolCalls.length > 0) {
-        chatModel.pushToolResultsMessage(filteredToolCalls, filteredToolResults);
-        const newChatState = (0, state_1.createChatStateForMessages)({
-            messages: chatModel.messages || [],
-            selectedModel,
-            askUserForInput: chatModel.askUserForInput,
-            error: null,
-        });
-        return newChatState;
-    }
-}

package/dist/agent/code-review/prompt.d.ts

@@ -1,2 +0,0 @@
-export declare const PROMPT = "\nIdentify code smells in tests\n- try-catch\n- conditionals added without a comment\n\nPlaywright gotchas\n- isVisible(), count() do not auto-wait\n\nPlaywright code smells\n- waitForLoadState is not required\n    - Especially with \"networkidle\", because modern webapps keep doing network activity\n\nIdentify test assumptions\n- What data does the test require?\n- Clean up test entities\n";
-//# sourceMappingURL=prompt.d.ts.map

package/dist/agent/code-review/prompt.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../src/agent/code-review/prompt.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,MAAM,8YAelB,CAAC"}

package/dist/agent/code-review/prompt.js

@@ -1,19 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PROMPT = void 0;
-exports.PROMPT = `
-Identify code smells in tests
-- try-catch
-- conditionals added without a comment
-
-Playwright gotchas
-- isVisible(), count() do not auto-wait
-
-Playwright code smells
-- waitForLoadState is not required
-    - Especially with "networkidle", because modern webapps keep doing network activity
-
-Identify test assumptions
-- What data does the test require?
-- Clean up test entities
-`;

package/dist/agent/diagnosis-agent/index.d.ts

@@ -1,11 +0,0 @@
-import { TraceClient } from "@empiricalrun/llm";
-import { TestErrorDiagnosisDetails } from "@empiricalrun/shared-types";
-import { CustomLogger } from "../../bin/logger";
-export declare function createTaskUsingFailureDiagnosis({ trace, diagnosis, logger, }: {
-    trace?: TraceClient;
-    diagnosis: TestErrorDiagnosisDetails;
-    logger?: CustomLogger;
-}): Promise<{
-    task: string;
-}>;
-//# sourceMappingURL=index.d.ts.map

package/dist/agent/diagnosis-agent/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/diagnosis-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAoChD,wBAAsB,+BAA+B,CAAC,EACpD,KAAK,EACL,SAAS,EACT,MAAM,GACP,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,yBAAyB,CAAC;IACrC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAqE5B"}

package/dist/agent/diagnosis-agent/index.js

@@ -1,88 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createTaskUsingFailureDiagnosis = createTaskUsingFailureDiagnosis;
-const llm_1 = require("@empiricalrun/llm");
-const strict_mode_violation_1 = require("./strict-mode-violation");
-const responseFormat = {
-    type: "json_schema",
-    json_schema: {
-        name: "test-case-auto-fix-summary",
-        strict: true,
-        schema: {
-            type: "object",
-            properties: {
-                observation: {
-                    type: "array",
-                    items: {
-                        type: "string",
-                    },
-                    description: "Detailed observation of what changed between successful and failed test screenshots",
-                },
-                action: {
-                    type: "string",
-                    description: "Direct action to fix the test in natural language without code snippets or options",
-                },
-            },
-            required: ["observation", "action"],
-            additionalProperties: false,
-        },
-    },
-};
-async function createTaskUsingFailureDiagnosis({ trace, diagnosis, logger, }) {
-    const failureDiagnosisSpan = trace?.span({
-        name: "auto-fix",
-        input: {
-            diagnosisId: diagnosis.diagnosisId,
-        },
-    });
-    logger?.log("Trying to fix the test using failure diagnosis. Fetching key moments of the diagnosis");
-    const resp = await fetch(diagnosis.keyMomentsUrl);
-    // TODO: check for response to be not ok
-    if (resp.ok) {
-        logger?.success("Successfully fetched key moments of the diagnosis");
-    }
-    else {
-        logger?.warn("Failed to fetch key moments of the diagnosis");
-    }
-    const screenshotsData = await resp.json();
-    const llm = new llm_1.LLM({
-        provider: "openai",
-        defaultModel: "o1",
-        trace: failureDiagnosisSpan,
-    });
-    // TODO: make this dynamic in nature. the prompts should be made receipe
-    // which will help to get rid of if else logic
-    // receipe to have:
-    // 1. selection criteria
-    // 2. job to be done - in this case generate a prompt
-    let prompt;
-    if (diagnosis.failed_run_metadata.stack.includes("strict mode violation")) {
-        prompt = (0, strict_mode_violation_1.fixStrictModeViolationPrompt)({
-            screenshotsData,
-            diagnosis,
-        });
-    }
-    if (prompt) {
-        const llmResponse = await llm.createChatCompletion({
-            messages: prompt,
-            modelParameters: {
-                max_completion_tokens: 40_000,
-            },
-            responseFormat,
-        });
-        const { observation, action } = JSON.parse(llmResponse?.content);
-        failureDiagnosisSpan?.update({
-            output: {
-                observation,
-                action,
-            },
-        });
-        return {
-            task: action,
-        };
-    }
-    // TODO: handle default prompt
-    return {
-        task: "",
-    };
-}

package/dist/agent/diagnosis-agent/strict-mode-violation.d.ts

@@ -1,10 +0,0 @@
-import { TestErrorDiagnosisDetails } from "@empiricalrun/shared-types";
-import OpenAI from "openai";
-export declare function fixStrictModeViolationPrompt({ screenshotsData, diagnosis, }: {
-    screenshotsData: {
-        success: string[];
-        failure: string[];
-    };
-    diagnosis: TestErrorDiagnosisDetails;
-}): OpenAI.Chat.Completions.ChatCompletionMessageParam[];
-//# sourceMappingURL=strict-mode-violation.d.ts.map

package/dist/agent/diagnosis-agent/strict-mode-violation.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"strict-mode-violation.d.ts","sourceRoot":"","sources":["../../../src/agent/diagnosis-agent/strict-mode-violation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,MAAM,MAAM,QAAQ,CAAC;AAoB5B,wBAAgB,4BAA4B,CAAC,EAC3C,eAAe,EACf,SAAS,GACV,EAAE;IACD,eAAe,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC1D,SAAS,EAAE,yBAAyB,CAAC;CACtC,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAiBvD"}

package/dist/agent/diagnosis-agent/strict-mode-violation.js

@@ -1,30 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fixStrictModeViolationPrompt = fixStrictModeViolationPrompt;
-const llm_1 = require("@empiricalrun/llm");
-const promptTemplate_0 = "{{#section \"system\"}}\nAs a software engineer, your task is to identify a fix for a failing Playwright test by analyzing screenshots of both the failed and successful test steps.\n\nKey issues for test failures include:\n- Duplicate elements for the same Playwright selector.\n\nInstructions:\n1. Examine the provided successful and failed test screenshots.\n2. Identify the correct element for action based on these observations on the successful test run screenshots.\n3. Sometimes the exact same locator is not available on failed test run screenshot, you need to identify the intent from successful test screenshots and apply that intent in failed test run screenshot to identify the right locator to interact with\n4. Evaluate the playwright selector options provided to you to execute the action. Pick the selector which best matches the intent of the test.\n5. Propose a precise action that addresses the issue.\n\nExample:\n- observation: \n  - Current step failure: await page.getByText(\"Audience\").click()\n  - Two similar buttons named \"Audience\" exist in the failed run screenshots\n  - The successful test run clicked on \"Untracked Audience\"\n  - The failed test run should click on \"Untracked Audience\"\n  - Available locators: await page.getByText(\"Untracked Audience\").click() contain the selector for Untracked Audience\n- action: Replace failing line with await page.getByText(\"Untracked Audience\").click()\n\nYour action should:\n- Be directly actionable and free of ambiguity, as it will guide another LLM to generate code.\n- Be in natural language and not just code snippet.\n- Be verified as feasible on the failure screen before responding.\n- Choose from the provided possible actions that can be executed on the failure screen.\n- Action should adhere to the format mentioned in the example, i.e. it should start with \"Replace the failing line\" and the updated code with replaced selector following it.\n\nEnsure the action is executable based on the failure screen context before providing it.\n{{/section}}\n\n{{#section \"user\"}}\nSuccessful test screenshots\n\n{{images successScreenshots}}\n\nFailed test screenshots\n\n{{images failedScreenshots}}\n\nStep where test failed:\n{{failingLine}}\n\nOptions for Playwright selectors to perform actions on a failed test screen:\n{{selectorOptions}}\n\n{{/section}}\n\n";
-function extractLocatorOptions(errorStack) {
-    // This regex matches a chain of locator API calls following the pattern:
-    // functionName(arguments) optionally chained with .functionName(arguments)
-    const regex = /aka\s+((?:[A-Za-z0-9_]+\([^)]*\)(?:\.[A-Za-z0-9_]+\([^)]*\))*))/g;
-    const options = [];
-    let match;
-    while ((match = regex.exec(errorStack)) !== null) {
-        if (match[1]) {
-            options.push(match[1]);
-        }
-    }
-    return options;
-}
-function fixStrictModeViolationPrompt({ screenshotsData, diagnosis, }) {
-    const compiledPrompt = (0, llm_1.compilePrompt)(promptTemplate_0, {
-        failingLine: diagnosis.failingLine,
-        successScreenshots: screenshotsData.success,
-        failedScreenshots: screenshotsData.failure,
-        selectorOptions: extractLocatorOptions(diagnosis.failed_run_metadata.stack).join("\n"),
-    }, {
-        imageDetail: "high",
-        modelProvider: "openai",
-    });
-    return compiledPrompt;
-}

package/dist/tools/commit-and-create-pr/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/commit-and-create-pr/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AAevD,eAAO,MAAM,qBAAqB,EAAE,IAgFnC,CAAC"}