@empiricalrun/test-gen 0.81.0 → 0.81.1

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.81.1
+
+### Patch Changes
+
+- Updated dependencies [76d0872]
+  - @empiricalrun/shared-types@0.14.0
+  - @empiricalrun/llm@0.27.0
+  - @empiricalrun/cua@0.4.1
+  - @empiricalrun/dashboard-client@0.3.0
+  - @empiricalrun/reporter@0.29.0
+  - @empiricalrun/test-run@0.17.0
+  - videostil@0.3.6
+
 ## 0.81.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.81.0",
+  "version": "0.81.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -89,15 +89,15 @@
     "typescript": "^5.3.3",
     "zod": "^4.0.1",
     "@empiricalrun/ast-parser": "^0.0.10",
-    "@empiricalrun/cua": "^0.4.0",
+    "@empiricalrun/cua": "^0.4.1",
     "@empiricalrun/dashboard-client": "^0.3.0",
-    "@empiricalrun/shared-types": "0.13.0",
-    "@empiricalrun/llm": "^0.26.0",
+    "@empiricalrun/shared-types": "0.14.0",
+    "@empiricalrun/llm": "^0.27.0",
     "@empiricalrun/r2-uploader": "^0.9.1",
     "@empiricalrun/trace-utils": "^0.1.0",
     "@empiricalrun/reporter": "^0.29.0",
     "@empiricalrun/test-run": "^0.17.0",
-    "videostil": "0.3.5"
+    "videostil": "0.3.6"
   },
   "devDependencies": {
     "@playwright/test": "1.60.0",
@@ -119,11 +119,11 @@
   },
   "scripts": {
     "dev": "tspc --build --watch",
-    "build": "tspc --build && cp -r src/browser-injected-scripts dist",
+    "build": "tspc --build",
     "clean": "tspc --build --clean",
     "lint": "biome check --unsafe",
     "test": "vitest run",
-    "test-browser": "pnpm build && RUNNING_BROWSER_TESTS_FOR_TEST_GEN=1 npx playwright test",
+    "test-browser": "pnpm build && npx playwright test",
     "test:watch": "vitest",
     "test:watch-files": "vitest $0 --watch"
   }

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/index.ts","./src/logger.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/cli.ts","./src/agent/index.ts","./src/agent/base/index.ts","./src/agent/browsing/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/chat/prompt/pw-utils-docs.ts","./src/agent/chat/prompt/repo.ts","./src/agent/chat/prompt/skills.ts","./src/agent/chat/prompt/test-case-def.ts","./src/agent/chat/prompt/trace-utils-docs.ts","./src/agent/code-review/index.ts","./src/agent/code-review/types.ts","./src/agent/code-review/xml-parser.ts","./src/agent/code-review/executor/index.ts","./src/agent/fast-triage/index.ts","./src/agent/master/action-tool-calls.ts","./src/agent/master/element-annotation.ts","./src/agent/master/execute-browser-action.ts","./src/agent/master/next-action.ts","./src/agent/master/planner.ts","./src/agent/master/run.ts","./src/agent/master/scroller.ts","./src/agent/master/with-hints.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/planner/run-time-planner.ts","./src/agent/planner/run.ts","./src/agent/triage/index.ts","./src/artifacts/index.ts","./src/artifacts/utils.ts","./src/auth/cli-auth.ts","./src/auth/index.ts","./src/auth/token-store.ts","./src/bin/index.ts","./src/bin/setup.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/browser-injected-scripts/annotate-elements.spec.ts","./src/constants/index.ts","./src/dashboard/client.ts","./src/dashboard/index.ts","./src/dashboard/tool-response-from-sandbox.ts","./src/dashboard/tool-response.ts","./src/dashboard/totp.ts","./src/dashboard/types.ts","./src/errors/index.ts","./src/file/client.ts","./src/file/server.ts","./src/file-info/index.ts","./src/file-info/adapters/file-system/index.ts","./src/file-info/adapters/file-system/reader.ts","./src/file-info/adapters/github/reader.ts","./src/generate-summary/frame-sampling.ts","./src/generate-summary/generate-error-stack-summary.ts","./src/generate-summary/generate-failed-step-screenshot-diff-summary.ts","./src/generate-summary/generate-grouped-summary.ts","./src/generate-summary/merge-summary.ts","./src/generate-summary/pick-videos-for-comparison.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/recorder/env-variables.ts","./src/recorder/index.ts","./src/recorder/request.ts","./src/recorder/temp-files.ts","./src/recorder/upload.ts","./src/recorder/validation.ts","./src/telemetry/index.ts","./src/test-build/index.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/index.ts","./src/tools/list-environments.ts","./src/tools/run-test.ts","./src/tools/test-gen-browser.ts","./src/tools/analyse-video/index.ts","./src/tools/api-client/index.ts","./src/tools/create-pull-request/index.ts","./src/tools/create-pull-request/utils.ts","./src/tools/definitions/analyse-video.ts","./src/tools/definitions/delete-file.ts","./src/tools/definitions/download-build.ts","./src/tools/definitions/grep.ts","./src/tools/definitions/list-tests-and-projects.ts","./src/tools/definitions/merge-conflicts.ts","./src/tools/definitions/publish-markdown-report.ts","./src/tools/definitions/rename-file.ts","./src/tools/definitions/run-test.ts","./src/tools/definitions/safe-bash.ts","./src/tools/definitions/str_replace_editor.ts","./src/tools/definitions/test-gen-browser.ts","./src/tools/definitions/trace-dot-zip.ts","./src/tools/definitions/utils.ts","./src/tools/delete-file/index.ts","./src/tools/download-build/index.ts","./src/tools/executor/base.ts","./src/tools/executor/index.ts","./src/tools/executor/types.ts","./src/tools/executor/utils/checkpoint.ts","./src/tools/executor/utils/git.ts","./src/tools/executor/utils/index.ts","./src/tools/executor/utils/pr-description.ts","./src/tools/fetch-file/index.ts","./src/tools/fetch-file/utils.ts","./src/tools/fetch-session-diff/index.ts","./src/tools/file-operations/create.ts","./src/tools/file-operations/index.ts","./src/tools/file-operations/insert.ts","./src/tools/file-operations/replace.ts","./src/tools/file-operations/shared/helpers.ts","./src/tools/file-operations/view/index.ts","./src/tools/firecrawl-scrape/index.ts","./src/tools/grep/index.ts","./src/tools/grep/types.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/issues-v1/index.ts","./src/tools/issues-v1/list-issues.ts","./src/tools/issues-v1/utils.ts","./src/tools/issues-v2/create-issue.ts","./src/tools/issues-v2/set-issue-description.ts","./src/tools/issues-v2/update-issue.ts","./src/tools/list-tests-and-projects/index.ts","./src/tools/merge-conflicts/index.ts","./src/tools/publish-markdown-report/index.ts","./src/tools/rename-file/index.ts","./src/tools/safe-bash/index.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/trace-dot-zip/index.ts","./src/tools/trace-dot-zip/types.ts","./src/tools/trace-dot-zip/utils/console-trace.ts","./src/tools/trace-dot-zip/utils/extract-screenshots.ts","./src/tools/trace-dot-zip/utils/extract-steps.ts","./src/tools/trace-dot-zip/utils/extract-zip.ts","./src/tools/trace-dot-zip/utils/network-trace.ts","./src/tools/triage-summary/index.ts","./src/tools/triage-summary/types.ts","./src/tools/triage-summary/utils.ts","./src/tools/utils/apply-line-limit.ts","./src/tools/utils/validate-schema.ts","./src/tools/utils/validators.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/utils/env.ts","./src/utils/exec.ts","./src/utils/file.ts","./src/utils/hash.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/json.ts","./src/utils/model.ts","./src/utils/playwright-test-id.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts","./src/utils/url-validation.ts","./src/utils/dedup/dedup-image.ts","./src/utils/dedup/find-threshold.ts","./src/video-core/index.ts","./src/video-core/model-limits.ts","./src/video-core/storage-manager.ts","./src/video-core/types.ts","./src/video-core/utils.ts"],"version":"5.8.3"}
+{"root":["./src/index.ts","./src/logger.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/cli.ts","./src/agent/index.ts","./src/agent/base/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/chat/prompt/pw-utils-docs.ts","./src/agent/chat/prompt/repo.ts","./src/agent/chat/prompt/skills.ts","./src/agent/chat/prompt/test-case-def.ts","./src/agent/chat/prompt/trace-utils-docs.ts","./src/agent/code-review/index.ts","./src/agent/code-review/types.ts","./src/agent/code-review/xml-parser.ts","./src/agent/code-review/executor/index.ts","./src/agent/fast-triage/index.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/triage/index.ts","./src/artifacts/index.ts","./src/artifacts/utils.ts","./src/auth/cli-auth.ts","./src/auth/index.ts","./src/auth/token-store.ts","./src/bin/index.ts","./src/bin/setup.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/constants/index.ts","./src/dashboard/client.ts","./src/dashboard/index.ts","./src/dashboard/tool-response-from-sandbox.ts","./src/dashboard/tool-response.ts","./src/dashboard/totp.ts","./src/dashboard/types.ts","./src/errors/index.ts","./src/file/client.ts","./src/file/server.ts","./src/file-info/index.ts","./src/file-info/adapters/file-system/index.ts","./src/file-info/adapters/file-system/reader.ts","./src/file-info/adapters/github/reader.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/recorder/env-variables.ts","./src/recorder/index.ts","./src/recorder/request.ts","./src/recorder/temp-files.ts","./src/recorder/upload.ts","./src/recorder/validation.ts","./src/telemetry/index.ts","./src/test-build/index.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/index.ts","./src/tools/list-environments.ts","./src/tools/run-test.ts","./src/tools/test-gen-browser.ts","./src/tools/analyse-video/index.ts","./src/tools/api-client/index.ts","./src/tools/create-pull-request/index.ts","./src/tools/create-pull-request/utils.ts","./src/tools/definitions/analyse-video.ts","./src/tools/definitions/delete-file.ts","./src/tools/definitions/download-build.ts","./src/tools/definitions/grep.ts","./src/tools/definitions/list-tests-and-projects.ts","./src/tools/definitions/merge-conflicts.ts","./src/tools/definitions/publish-markdown-report.ts","./src/tools/definitions/rename-file.ts","./src/tools/definitions/run-test.ts","./src/tools/definitions/safe-bash.ts","./src/tools/definitions/str_replace_editor.ts","./src/tools/definitions/test-gen-browser.ts","./src/tools/definitions/trace-dot-zip.ts","./src/tools/definitions/utils.ts","./src/tools/delete-file/index.ts","./src/tools/download-build/index.ts","./src/tools/executor/base.ts","./src/tools/executor/index.ts","./src/tools/executor/types.ts","./src/tools/executor/utils/checkpoint.ts","./src/tools/executor/utils/git.ts","./src/tools/executor/utils/index.ts","./src/tools/executor/utils/pr-description.ts","./src/tools/fetch-file/index.ts","./src/tools/fetch-file/utils.ts","./src/tools/fetch-session-diff/index.ts","./src/tools/file-operations/create.ts","./src/tools/file-operations/index.ts","./src/tools/file-operations/insert.ts","./src/tools/file-operations/replace.ts","./src/tools/file-operations/shared/helpers.ts","./src/tools/file-operations/view/index.ts","./src/tools/firecrawl-scrape/index.ts","./src/tools/grep/index.ts","./src/tools/grep/types.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/issues-v1/index.ts","./src/tools/issues-v1/list-issues.ts","./src/tools/issues-v1/utils.ts","./src/tools/issues-v2/create-issue.ts","./src/tools/issues-v2/set-issue-description.ts","./src/tools/issues-v2/update-issue.ts","./src/tools/list-tests-and-projects/index.ts","./src/tools/merge-conflicts/index.ts","./src/tools/publish-markdown-report/index.ts","./src/tools/rename-file/index.ts","./src/tools/safe-bash/index.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/trace-dot-zip/index.ts","./src/tools/trace-dot-zip/types.ts","./src/tools/trace-dot-zip/utils/console-trace.ts","./src/tools/trace-dot-zip/utils/extract-screenshots.ts","./src/tools/trace-dot-zip/utils/extract-steps.ts","./src/tools/trace-dot-zip/utils/extract-zip.ts","./src/tools/trace-dot-zip/utils/network-trace.ts","./src/tools/triage-summary/index.ts","./src/tools/triage-summary/types.ts","./src/tools/triage-summary/utils.ts","./src/tools/utils/apply-line-limit.ts","./src/tools/utils/validate-schema.ts","./src/tools/utils/validators.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/utils/exec.ts","./src/utils/file.ts","./src/utils/hash.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/json.ts","./src/utils/model.ts","./src/utils/playwright-test-id.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts","./src/utils/url-validation.ts","./src/utils/dedup/dedup-image.ts","./src/utils/dedup/find-threshold.ts","./src/video-core/index.ts","./src/video-core/model-limits.ts","./src/video-core/storage-manager.ts","./src/video-core/types.ts","./src/video-core/utils.ts"],"version":"5.8.3"}

package/dist/agent/browsing/index.d.ts

@@ -1,15 +0,0 @@
-import { LLM, TraceClient } from "@empiricalrun/llm";
-import { Page } from "playwright";
-import { PlaywrightActions } from "../../actions";
-export type BrowserAgentAction = {
-    action: string;
-    code: string | undefined;
-};
-export declare function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace, }: {
-    action: string;
-    page: Page;
-    actions: PlaywrightActions;
-    llm?: LLM;
-    trace?: TraceClient;
-}): Promise<BrowserAgentAction | undefined>;
-//# sourceMappingURL=index.d.ts.map

package/dist/agent/browsing/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAKlD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CA0D1C"}

package/dist/agent/browsing/index.js

@@ -1,60 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.executeTaskUsingBrowsingAgent = executeTaskUsingBrowsingAgent;
-const llm_1 = require("@empiricalrun/llm");
-const constants_1 = require("../../constants");
-const promptTemplate_0 = "{{#section \"system\"}}\nYou are a browser automation agent who is given a task to generate code for navigation and assertion. This task is your\ngoal and you must achieve it.\n\nYou will be provided with already executed actions and basis that you need to pick the next step to achieve the task.\nRemember that the goal must be achieved.\n\nYou will be provided with the web page snapshot in the form of Document Object Model. Based on the goal and available\ntool calls you need to pick the appropriate tool call.\n\nInstructions:\n- Take actions one at a time. Do not try to take multiple actions\n- You can respond with multiple assertions in one shot\n- Do not repeat the same actions again otherwise your response will be marked INVALID\n- Avoid repeating errors which we got while executing the last action\n- Stick to the task provided to you and mark the task done once the task is complete\n- Do not execute any action which is not mentioned in the task\n- Do not repeat actions which are already executed more than twice otherwise your response will be marked INVALID\n- Always refer to \"Executed actions\" before deciding your next action for completion of the task.\n- End the task done if all actions required for task are executed\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\nCurrent page snapshot:\n{{pageSnapshot}}\n{{/section}}";
-const html_1 = require("../../utils/html");
-async function executeTaskUsingBrowsingAgent({ action, page, actions, llm, trace, }) {
-    let generatedCodeSteps;
-    const tools = actions.getBrowsingActionSchemas();
-    const browsingAgentSpan = trace?.span({
-        name: `browsing-agent`,
-        input: {
-            action,
-        },
-    });
-    const pageContentSpan = browsingAgentSpan?.span({
-        name: "page-content",
-    });
-    const pageContent = await page.content();
-    pageContentSpan?.end({ output: { pageContent } });
-    const sanitizationSpan = browsingAgentSpan?.span({
-        name: "page-sanitization",
-    });
-    const pageSnapshot = (0, html_1.sanitizeHtml)(pageContent);
-    sanitizationSpan?.end({ output: { pageSnapshot } });
-    const promptSpan = browsingAgentSpan?.span({ name: "page-prompt" });
-    const messages = (0, llm_1.compilePrompt)(promptTemplate_0, { pageSnapshot, task: action });
-    promptSpan?.end({ output: { messages } });
-    llm =
-        llm ||
-            new llm_1.LLM({
-                provider: constants_1.DEFAULT_MODEL_PROVIDER,
-            });
-    const completion = await llm.createChatCompletion({
-        model: "o3-mini-2025-01-31",
-        messages,
-        tools,
-        trace: browsingAgentSpan,
-        modelParameters: {
-            tool_choice: "required",
-        },
-    });
-    const toolCalls = completion?.tool_calls || [];
-    const toolCallsSpan = browsingAgentSpan?.span({ name: "tool-calls" });
-    for (const i in toolCalls) {
-        const toolCall = toolCalls[i];
-        if ("function" in toolCall) {
-            const args = JSON.parse(toolCall.function.arguments);
-            const code = await actions.executeAction(toolCall.function.name, JSON.parse(toolCall.function.arguments), toolCallsSpan);
-            generatedCodeSteps = {
-                // Passing reason as action, in order to pass the correct action that took place to runtime planner
-                action: args.reason,
-                code,
-            };
-        }
-    }
-    toolCallsSpan?.end({ output: { toolCalls, generatedCodeSteps } });
-    return generatedCodeSteps;
-}

package/dist/agent/master/action-tool-calls.d.ts

@@ -1,42 +0,0 @@
-export declare enum ActionType {
-    FILL = "fill",
-    PAGE_GOTO = "page_goto",
-    CLICK = "click",
-    PRESS_ACTION = "keyboard_press_on_element",
-    ASSERT_TEXT = "assert_text",
-    HOVER = "hover_element",
-    SCROLL = "scroll",
-    OBSERVATION = "observation",
-    UNKNOWN = "unknown"
-}
-export declare function isValidActionType(value: string): value is ActionType;
-export declare function getActionToolCalls(): {
-    type: string;
-    function: {
-        name: string;
-        description: string;
-        parameters: {
-            type: string;
-            properties: {
-                reason: {
-                    type: string;
-                    description: string;
-                };
-                action: {
-                    type: string;
-                    description: string;
-                };
-                element_description: {
-                    type: string;
-                    description: string;
-                };
-                is_element_visible: {
-                    type: string;
-                    description: string;
-                };
-            };
-            required: string[];
-        };
-    };
-}[];
-//# sourceMappingURL=action-tool-calls.d.ts.map

package/dist/agent/master/action-tool-calls.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"action-tool-calls.d.ts","sourceRoot":"","sources":["../../../src/agent/master/action-tool-calls.ts"],"names":[],"mappings":"AAAA,oBAAY,UAAU;IACpB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,YAAY,8BAA8B;IAC1C,WAAW,gBAAgB;IAC3B,KAAK,kBAAkB;IACvB,MAAM,WAAW;IACjB,WAAW,gBAAgB;IAC3B,OAAO,YAAY;CACpB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,UAAU,CAEpE;AAqDD,wBAAgB,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA8CjC"}

package/dist/agent/master/action-tool-calls.js

@@ -1,87 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ActionType = void 0;
-exports.isValidActionType = isValidActionType;
-exports.getActionToolCalls = getActionToolCalls;
-var ActionType;
-(function (ActionType) {
-    ActionType["FILL"] = "fill";
-    ActionType["PAGE_GOTO"] = "page_goto";
-    ActionType["CLICK"] = "click";
-    ActionType["PRESS_ACTION"] = "keyboard_press_on_element";
-    ActionType["ASSERT_TEXT"] = "assert_text";
-    ActionType["HOVER"] = "hover_element";
-    ActionType["SCROLL"] = "scroll";
-    ActionType["OBSERVATION"] = "observation";
-    ActionType["UNKNOWN"] = "unknown";
-})(ActionType || (exports.ActionType = ActionType = {}));
-function isValidActionType(value) {
-    return Object.values(ActionType).includes(value);
-}
-const createActionCall = (name, description, additionalProperties = {}) => ({
-    type: "function",
-    function: {
-        name,
-        description,
-        parameters: {
-            type: "object",
-            properties: {
-                reason: {
-                    type: "string",
-                    description: "explain how this action will help to complete the task. the reason should align with the task provided",
-                },
-                action: {
-                    type: "string",
-                    description: `explain the next action in natural language.
-                The next action should be as atomic as possible, precise and should contain enough details about the action to be performed.
-                E.g. each click, key press, input, assert, observation should be a separate action.
-                Each action should take the task to completion, if not the action is invalid.
-                If the element is not visible then action should be scroll.`,
-                },
-                element_description: {
-                    type: "string",
-                    description: "The description of the element on which action needs to be taken, including its position, appearance, etc.",
-                },
-                is_element_visible: {
-                    type: "boolean",
-                    description: "A boolean to indicate whether the concerned element is visible in the screenshot.",
-                },
-                ...additionalProperties,
-            },
-            required: [
-                "reason",
-                "action",
-                "element_description",
-                "is_element_visible",
-                ...Object.keys(additionalProperties),
-            ],
-        },
-    },
-});
-function getActionToolCalls() {
-    return [
-        createActionCall(ActionType.ASSERT_TEXT, "assert whether the given element on the page is visible", {
-            assertion_text: {
-                type: "string",
-                description: "The text for which the visibility needs to be asserted.",
-            },
-        }),
-        createActionCall(ActionType.FILL, "fill the input element with a particular text"),
-        createActionCall(ActionType.CLICK, "click an element on the page"),
-        createActionCall(ActionType.PAGE_GOTO, "navigate the page to a new URL"),
-        createActionCall(ActionType.PRESS_ACTION, "Focuses the matching element and presses a combination of the keys"),
-        createActionCall(ActionType.HOVER, "hover over an element on the page"),
-        createActionCall(ActionType.SCROLL, "Scroll the page since element is not visible in the screenshot", {
-            element: {
-                type: "string",
-                description: "Element to look for after scrolling to complete the task.",
-            },
-        }),
-        createActionCall(ActionType.OBSERVATION, "Observation based on the task provided. If the task doesn't explicitly mention any browser interaction, and instead, asks questions to be answered — use this tool call", {
-            observation: {
-                type: "string",
-                description: "Observation based on the task provided.",
-            },
-        }),
-    ];
-}

package/dist/agent/master/element-annotation.d.ts

@@ -1,30 +0,0 @@
-import { LLM, TraceClient } from "@empiricalrun/llm";
-import { TestGenConfigOptions } from "@empiricalrun/shared-types/agent-workflow";
-import { Page } from "playwright";
-import { ActionType } from "./action-tool-calls";
-export declare function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, preference, }: {
-    elementDescription: string;
-    annotations: string;
-    annotatedScreenshot: string;
-    trace?: TraceClient;
-    llm?: LLM;
-    options?: TestGenConfigOptions;
-    preference: AnnotationPreference;
-}): Promise<string | undefined>;
-export type AnnotationPreference = {
-    actionType: "all" | ActionType.FILL | ActionType.ASSERT_TEXT | ActionType.SCROLL;
-    assertionText?: string | undefined;
-};
-export declare function getAnnotationKeys({ page, preference, trace, }: {
-    page: Page;
-    preference: AnnotationPreference;
-    trace?: TraceClient;
-}): Promise<{
-    annotationKeys: {
-        elementID: string;
-        text: string;
-    }[];
-    annotationBuffer: Buffer;
-    annotatedPageScreenshot: string;
-}>;
-//# sourceMappingURL=element-annotation.d.ts.map

package/dist/agent/master/element-annotation.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAQlC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAgBF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAmFD"}

package/dist/agent/master/element-annotation.js

@@ -1,175 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getElementAnnotation = getElementAnnotation;
-exports.getAnnotationKeys = getAnnotationKeys;
-const llm_1 = require("@empiricalrun/llm");
-const constants_1 = require("../../constants");
-const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements\npresent in the image.\n\nYou will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation\nis done by drawing a red box around the element and a small yellow box on it which contains unique element id.\n\nYou are given a Annotations which contains list of unique element id and description of the element separated by \":\".\n\nYou are also given the description of the element on which the action needs to be taken. The description includes\ninformation about how the element looks, it's position etc.\n\nYour task is to provide the annotation of the element on which the action needs to be performed based on the element\ndescription.\n\nFollow steps to fulfil your task:\n- Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each\nelement.\n- For describing each element Id\n-- iterate over each element Id in annotation list\n-- check if the description is already present for the element Id in the Annotation provided to you. If present skip\ndescribing it and use it as is.\n-- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or\nicon enclosed in the element.\n- Respond with the mapped element Ids as \"enriched_annotations\"\n- Based on the description provided to you and the enriched annotations, first identify the element Id whose description\nmatches the task provided\n\nNote:\n- Ensure providing the description of all the elements in the list.\n- Don't update the description if its already present in the given annotations\n- Replace all the \"NA\" with description of the element. Its position, how does it look like etc.\n- There should be no \"NA\" present in any of the element description\n{{/section}}\n\n{{#section \"user\"}}\nElement description:\n{{elementDescription}}\n\nAnnotations:\n{{annotations}}\n\n{{image annotatedScreenshot}}\n{{/section}}";
-const annotationToolAction = {
-    name: "element_annotation",
-    schema: {
-        type: "function",
-        function: {
-            name: "element_annotation",
-            description: "Handles annotations for elements and enriched annotations.",
-            parameters: {
-                type: "object",
-                properties: {
-                    enriched_annotations: {
-                        type: "array",
-                        description: "List of enriched annotations for elements.",
-                        items: {
-                            type: "object",
-                            properties: {
-                                element_id: {
-                                    type: "string",
-                                    description: "Unique element ID for the element.",
-                                },
-                                description: {
-                                    type: "string",
-                                    description: "Detailed description for the element. Must not be NA",
-                                },
-                            },
-                            required: ["element_id", "description"],
-                        },
-                    },
-                    reason: {
-                        type: "string",
-                        description: "Explain why this element is selected. The reason should be clear and align with the task or purpose.",
-                    },
-                    element: {
-                        type: "string",
-                        description: "Detailed description of the DOM element's visual characteristics and position.",
-                    },
-                    element_annotation: {
-                        type: "string",
-                        description: "Return the unique element ID for the element on which the action needs to be performed.",
-                    },
-                },
-                required: [
-                    "enriched_annotations",
-                    "reason",
-                    "element",
-                    "element_annotation",
-                ],
-            },
-        },
-    },
-};
-async function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, preference, }) {
-    const annotationsSpan = trace?.span({
-        name: "get-element-annotation",
-        input: {
-            elementDescription,
-            annotations,
-            annotatedScreenshot,
-            preference,
-        },
-    });
-    const messages = (0, llm_1.compilePrompt)(promptTemplate_0, {
-        elementDescription,
-        annotations,
-        annotatedScreenshot,
-    }, options);
-    llm =
-        llm ||
-            new llm_1.LLM({
-                provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
-                defaultModel: options?.model || constants_1.DEFAULT_MODEL,
-            });
-    const completion = await llm.createChatCompletion({
-        messages,
-        modelParameters: {
-            ...constants_1.DEFAULT_MODEL_PARAMETERS,
-            ...options?.modelParameters,
-            tool_choice: "required",
-            temperature: 0.5,
-        },
-        trace: annotationsSpan,
-        traceName: "get-element-from-action",
-        //@ts-ignore
-        tools: [annotationToolAction.schema],
-    });
-    const toolCall = completion?.tool_calls?.[0];
-    annotationsSpan?.end({ output: toolCall });
-    if (toolCall && "function" in toolCall) {
-        const args = JSON.parse(toolCall.function.arguments);
-        return args.element_annotation;
-    }
-    return;
-}
-async function getAnnotationKeys({ page, preference, trace, }) {
-    const annotatedElements = await page.evaluate(({ preference, options }) => {
-        // @ts-ignore
-        window.annotationInstance = annotateElementsWithPreference({
-            options: options,
-            preference: preference,
-        });
-        const annotations = Object.entries(
-        // @ts-ignore
-        window.annotationInstance.annotations).map(([key, value]) => {
-            // Helper function to safely get text content
-            const getTextContent = (node) => {
-                try {
-                    return node?.innerText?.trim() || null;
-                }
-                catch {
-                    return null;
-                }
-            };
-            // Helper function to safely get placeholder
-            const getPlaceholder = (node) => {
-                try {
-                    return node?.placeholder?.trim() || null;
-                }
-                catch {
-                    return null;
-                }
-            };
-            return {
-                elementID: key,
-                innerText: getTextContent(value.node),
-                innerHTML: value.node.innerHTML,
-                outerHTML: value.node.outerHTML,
-                placeholder: getPlaceholder(value.node),
-            };
-        });
-        const fullPageHTML = document.documentElement.outerHTML;
-        return { annotations, fullPageHTML };
-    }, { preference, options: {} });
-    const annotationKeys = await enrichAnnotations(annotatedElements.annotations, annotatedElements.fullPageHTML, trace);
-    const annotationBuffer = await page.screenshot({
-    // path: `screenshots/screenshot-${screenshotIndex++}.png`,
-    });
-    const annotatedPageScreenshot = annotationBuffer.toString("base64");
-    await page.evaluate(() => {
-        console.log({
-            // @ts-ignore
-            disable: window?.annotationInstance?.disable,
-        });
-        if (
-        // @ts-ignore
-        window?.annotationInstance &&
-            // @ts-ignore
-            window?.annotationInstance?.destroy) {
-            // @ts-ignore
-            window?.annotationInstance?.destroy();
-        }
-    });
-    return {
-        annotationKeys,
-        annotationBuffer,
-        annotatedPageScreenshot,
-    };
-}
-async function enrichAnnotations(annotatedElements, _pageHtml, _trace) {
-    const results = [];
-    for (const element of annotatedElements) {
-        const text = element.innerText?.trim() || element.placeholder?.trim();
-        results.push({
-            elementID: element.elementID,
-            text: text || "NA",
-        });
-    }
-    return results;
-}

package/dist/agent/master/execute-browser-action.d.ts

@@ -1,24 +0,0 @@
-import { LLM, TraceClient } from "@empiricalrun/llm";
-import type { Page } from "@playwright/test";
-import { PlaywrightActions } from "../../actions";
-export declare function executeBrowserAction({ page, nextAction, flags, actions, llm, trace, }: {
-    page: Page;
-    nextAction: {
-        actionType: string;
-        toolCallArgs: string;
-    };
-    flags: {
-        useActionSpecificAnnotations: boolean;
-    };
-    actions: PlaywrightActions;
-    trace?: TraceClient | undefined;
-    llm: LLM;
-}): Promise<{
-    generatedCodeSteps: string[];
-    output: {
-        action: string;
-        reason: string;
-        elementAnnotation?: string;
-    };
-}>;
-//# sourceMappingURL=execute-browser-action.d.ts.map

package/dist/agent/master/execute-browser-action.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAUlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IAChC,GAAG,EAAE,GAAG,CAAC;CACV;;;gBAIW,MAAM;gBACN,MAAM;4BACM,MAAM;;GA+H7B"}