npm - libretto - Versions diffs - 0.3.0 → 0.3.1 - Mend

libretto 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (162) hide show

package/README.md +41 -125
package/dist/cli/cli.js +298 -0
package/dist/cli/commands/ai.js +21 -0
package/dist/cli/commands/browser.js +82 -0
package/dist/cli/commands/execution.js +490 -0
package/dist/cli/commands/init.js +166 -0
package/dist/cli/commands/logs.js +93 -0
package/dist/cli/commands/snapshot.js +106 -0
package/dist/cli/core/ai-config.js +149 -0
package/dist/cli/core/browser.js +652 -0
package/dist/cli/core/context.js +117 -0
package/dist/cli/core/pause-signals.js +29 -0
package/dist/cli/core/session-telemetry.js +491 -0
package/dist/cli/core/session.js +183 -0
package/dist/cli/core/snapshot-analyzer.js +492 -0
package/dist/cli/core/telemetry.js +362 -0
package/dist/cli/index.js +13 -0
package/dist/cli/workers/run-integration-runtime.js +234 -0
package/dist/cli/workers/run-integration-worker-protocol.js +12 -0
package/dist/cli/workers/run-integration-worker.js +67 -0
package/dist/index.cjs +144 -0
package/dist/index.d.cts +21 -0
package/dist/index.d.ts +21 -0
package/dist/index.js +114 -0
package/dist/runtime/download/download.cjs +70 -0
package/dist/runtime/download/download.d.cts +35 -0
package/dist/runtime/download/download.d.ts +35 -0
package/dist/runtime/download/download.js +45 -0
package/dist/runtime/download/index.cjs +30 -0
package/dist/runtime/download/index.d.cts +3 -0
package/dist/runtime/download/index.d.ts +3 -0
package/dist/runtime/download/index.js +8 -0
package/dist/runtime/extract/extract.cjs +88 -0
package/dist/runtime/extract/extract.d.cts +23 -0
package/dist/runtime/extract/extract.d.ts +23 -0
package/dist/runtime/extract/extract.js +64 -0
package/dist/runtime/extract/index.cjs +28 -0
package/dist/runtime/extract/index.d.cts +5 -0
package/dist/runtime/extract/index.d.ts +5 -0
package/dist/runtime/extract/index.js +4 -0
package/dist/runtime/network/index.cjs +28 -0
package/dist/runtime/network/index.d.cts +4 -0
package/dist/runtime/network/index.d.ts +4 -0
package/dist/runtime/network/index.js +6 -0
package/dist/runtime/network/network.cjs +91 -0
package/dist/runtime/network/network.d.cts +28 -0
package/dist/runtime/network/network.d.ts +28 -0
package/dist/runtime/network/network.js +67 -0
package/dist/runtime/recovery/agent.cjs +223 -0
package/dist/runtime/recovery/agent.d.cts +13 -0
package/dist/runtime/recovery/agent.d.ts +13 -0
package/dist/runtime/recovery/agent.js +199 -0
package/dist/runtime/recovery/errors.cjs +124 -0
package/dist/runtime/recovery/errors.d.cts +31 -0
package/dist/runtime/recovery/errors.d.ts +31 -0
package/dist/runtime/recovery/errors.js +100 -0
package/dist/runtime/recovery/index.cjs +34 -0
package/dist/runtime/recovery/index.d.cts +7 -0
package/dist/runtime/recovery/index.d.ts +7 -0
package/dist/runtime/recovery/index.js +10 -0
package/dist/runtime/recovery/recovery.cjs +55 -0
package/dist/runtime/recovery/recovery.d.cts +12 -0
package/dist/runtime/recovery/recovery.d.ts +12 -0
package/dist/runtime/recovery/recovery.js +31 -0
package/dist/shared/config/config.cjs +44 -0
package/dist/shared/config/config.d.cts +10 -0
package/dist/shared/config/config.d.ts +10 -0
package/dist/shared/config/config.js +18 -0
package/dist/shared/config/index.cjs +32 -0
package/dist/shared/config/index.d.cts +1 -0
package/dist/shared/config/index.d.ts +1 -0
package/dist/shared/config/index.js +10 -0
package/dist/shared/debug/index.cjs +30 -0
package/dist/shared/debug/index.d.cts +1 -0
package/dist/shared/debug/index.d.ts +1 -0
package/dist/shared/debug/index.js +5 -0
package/dist/shared/debug/pause.cjs +90 -0
package/dist/shared/debug/pause.d.cts +16 -0
package/dist/shared/debug/pause.d.ts +16 -0
package/dist/shared/debug/pause.js +55 -0
package/dist/shared/instrumentation/errors.cjs +81 -0
package/dist/shared/instrumentation/errors.d.cts +12 -0
package/dist/shared/instrumentation/errors.d.ts +12 -0
package/dist/shared/instrumentation/errors.js +57 -0
package/dist/shared/instrumentation/index.cjs +35 -0
package/dist/shared/instrumentation/index.d.cts +6 -0
package/dist/shared/instrumentation/index.d.ts +6 -0
package/dist/shared/instrumentation/index.js +12 -0
package/dist/shared/instrumentation/instrument.cjs +206 -0
package/dist/shared/instrumentation/instrument.d.cts +32 -0
package/dist/shared/instrumentation/instrument.d.ts +32 -0
package/dist/shared/instrumentation/instrument.js +190 -0
package/dist/shared/llm/ai-sdk-adapter.cjs +67 -0
package/dist/shared/llm/ai-sdk-adapter.d.cts +22 -0
package/dist/shared/llm/ai-sdk-adapter.d.ts +22 -0
package/dist/shared/llm/ai-sdk-adapter.js +43 -0
package/dist/shared/llm/client.cjs +139 -0
package/dist/shared/llm/client.d.cts +6 -0
package/dist/shared/llm/client.d.ts +6 -0
package/dist/shared/llm/client.js +115 -0
package/dist/shared/llm/index.cjs +31 -0
package/dist/shared/llm/index.d.cts +5 -0
package/dist/shared/llm/index.d.ts +5 -0
package/dist/shared/llm/index.js +6 -0
package/dist/shared/llm/types.cjs +16 -0
package/dist/shared/llm/types.d.cts +66 -0
package/dist/shared/llm/types.d.ts +66 -0
package/dist/shared/llm/types.js +0 -0
package/dist/shared/logger/index.cjs +37 -0
package/dist/shared/logger/index.d.cts +2 -0
package/dist/shared/logger/index.d.ts +2 -0
package/dist/shared/logger/index.js +13 -0
package/dist/shared/logger/logger.cjs +232 -0
package/dist/shared/logger/logger.d.cts +86 -0
package/dist/shared/logger/logger.d.ts +86 -0
package/dist/shared/logger/logger.js +207 -0
package/dist/shared/logger/sinks.cjs +160 -0
package/dist/shared/logger/sinks.d.cts +9 -0
package/dist/shared/logger/sinks.d.ts +9 -0
package/dist/shared/logger/sinks.js +124 -0
package/dist/shared/paths/paths.cjs +104 -0
package/dist/shared/paths/paths.d.cts +10 -0
package/dist/shared/paths/paths.d.ts +10 -0
package/dist/shared/paths/paths.js +73 -0
package/dist/shared/run/api.cjs +28 -0
package/dist/shared/run/api.d.cts +2 -0
package/dist/shared/run/api.d.ts +2 -0
package/dist/shared/run/api.js +4 -0
package/dist/shared/run/browser.cjs +98 -0
package/dist/shared/run/browser.d.cts +22 -0
package/dist/shared/run/browser.d.ts +22 -0
package/dist/shared/run/browser.js +74 -0
package/dist/shared/state/index.cjs +38 -0
package/dist/shared/state/index.d.cts +2 -0
package/dist/shared/state/index.d.ts +2 -0
package/dist/shared/state/index.js +16 -0
package/dist/shared/state/session-state.cjs +85 -0
package/dist/shared/state/session-state.d.cts +34 -0
package/dist/shared/state/session-state.d.ts +34 -0
package/dist/shared/state/session-state.js +56 -0
package/dist/shared/visualization/ghost-cursor.cjs +174 -0
package/dist/shared/visualization/ghost-cursor.d.cts +37 -0
package/dist/shared/visualization/ghost-cursor.d.ts +37 -0
package/dist/shared/visualization/ghost-cursor.js +145 -0
package/dist/shared/visualization/highlight.cjs +134 -0
package/dist/shared/visualization/highlight.d.cts +22 -0
package/dist/shared/visualization/highlight.d.ts +22 -0
package/dist/shared/visualization/highlight.js +108 -0
package/dist/shared/visualization/index.cjs +45 -0
package/dist/shared/visualization/index.d.cts +3 -0
package/dist/shared/visualization/index.d.ts +3 -0
package/dist/shared/visualization/index.js +24 -0
package/dist/shared/workflow/workflow.cjs +47 -0
package/dist/shared/workflow/workflow.d.cts +21 -0
package/dist/shared/workflow/workflow.d.ts +21 -0
package/dist/shared/workflow/workflow.js +21 -0
package/package.json +11 -70
package/bin/libretto.mjs +0 -18
package/scripts/postinstall.mjs +0 -48
/package/{skill → .agents/skills/libretto}/SKILL.md +0 -0
/package/{skill → .agents/skills/libretto}/code-generation-rules.md +0 -0
/package/{skill → .agents/skills/libretto}/integration-approach-selection.md +0 -0

package/dist/runtime/extract/extract.d.ts ADDED Viewed

@@ -0,0 +1,23 @@
+import { Page } from 'playwright';
+import z from 'zod';
+import { MinimalLogger } from '../../shared/logger/logger.js';
+import { LLMClient } from '../../shared/llm/types.js';
+type ExtractOptions<T extends z.ZodType> = {
+    page: Page;
+    instruction: string;
+    schema: T;
+    llmClient: LLMClient;
+    logger?: MinimalLogger;
+    /** Optional CSS selector to scope extraction to a specific element. */
+    selector?: string;
+};
+/**
+ * Generic AI-powered data extraction from page elements.
+ * Takes a screenshot (full-page via CDP or scoped to an element),
+ * captures DOM content, and uses an LLM to extract structured data
+ * matching the provided Zod schema.
+ */
+declare function extractFromPage<T extends z.ZodType>(options: ExtractOptions<T>): Promise<z.infer<T>>;
+export { type ExtractOptions, extractFromPage };

package/dist/runtime/extract/extract.js ADDED Viewed

@@ -0,0 +1,64 @@
+import { defaultLogger } from "../../shared/logger/logger.js";
+async function extractFromPage(options) {
+  const { page, instruction, schema, selector, logger = defaultLogger, llmClient } = options;
+  let screenshot;
+  let domContent;
+  if (selector) {
+    const element = page.locator(selector);
+    await element.waitFor({ state: "visible", timeout: 1e4 });
+    const screenshotBuffer = await element.screenshot();
+    screenshot = screenshotBuffer.toString("base64");
+    try {
+      domContent = await element.innerHTML();
+      if (domContent.length > 3e4) {
+        domContent = domContent.slice(0, 3e4) + "\n... [truncated]";
+      }
+    } catch {
+      domContent = void 0;
+    }
+  } else {
+    const cdpClient = await page.context().newCDPSession(page);
+    await cdpClient.send("Page.enable");
+    const { data } = await cdpClient.send("Page.captureScreenshot", {
+      format: "png"
+    });
+    screenshot = data;
+    try {
+      const htmlContent = await page.content();
+      domContent = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
+    } catch {
+      domContent = void 0;
+    }
+  }
+  const prompt = `You are analyzing a screenshot${selector ? " of a specific element" : ""} from a web page to extract structured data.
+Instruction: ${instruction}
+${domContent ? `Here is the HTML content for additional context:
+<html>
+${domContent}
+</html>` : ""}
+Extract the requested information from the screenshot and return it in the specified format. Be precise and only extract what is visible.`;
+  const result = await llmClient.generateObjectFromMessages({
+    schema,
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: prompt },
+          { type: "image", image: `data:image/png;base64,${screenshot}` }
+        ]
+      }
+    ],
+    temperature: 0
+  });
+  logger.info("extractFromPage completed", {
+    selector,
+    instruction: instruction.slice(0, 100)
+  });
+  return result;
+}
+export {
+  extractFromPage
+};

package/dist/runtime/extract/index.cjs ADDED Viewed

@@ -0,0 +1,28 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var extract_exports = {};
+__export(extract_exports, {
+  extractFromPage: () => import_extract.extractFromPage
+});
+module.exports = __toCommonJS(extract_exports);
+var import_extract = require("./extract.js");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  extractFromPage
+});

package/dist/runtime/extract/index.d.cts ADDED Viewed

@@ -0,0 +1,5 @@
+export { ExtractOptions, extractFromPage } from './extract.cjs';
+import 'playwright';
+import 'zod';
+import '../../shared/logger/logger.cjs';
+import '../../shared/llm/types.cjs';

package/dist/runtime/extract/index.d.ts ADDED Viewed

@@ -0,0 +1,5 @@
+export { ExtractOptions, extractFromPage } from './extract.js';
+import 'playwright';
+import 'zod';
+import '../../shared/logger/logger.js';
+import '../../shared/llm/types.js';

package/dist/runtime/extract/index.js ADDED Viewed

@@ -0,0 +1,4 @@
+import { extractFromPage } from "./extract.js";
+export {
+  extractFromPage
+};

package/dist/runtime/network/index.cjs ADDED Viewed

@@ -0,0 +1,28 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var network_exports = {};
+__export(network_exports, {
+  pageRequest: () => import_network.pageRequest
+});
+module.exports = __toCommonJS(network_exports);
+var import_network = require("./network.js");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  pageRequest
+});

package/dist/runtime/network/index.d.cts ADDED Viewed

@@ -0,0 +1,4 @@
+export { PageRequestOptions, RequestConfig, pageRequest } from './network.cjs';
+import 'playwright';
+import 'zod';
+import '../../shared/logger/logger.cjs';

package/dist/runtime/network/index.d.ts ADDED Viewed

@@ -0,0 +1,4 @@
+export { PageRequestOptions, RequestConfig, pageRequest } from './network.js';
+import 'playwright';
+import 'zod';
+import '../../shared/logger/logger.js';

package/dist/runtime/network/index.js ADDED Viewed

@@ -0,0 +1,6 @@
+import {
+  pageRequest
+} from "./network.js";
+export {
+  pageRequest
+};

package/dist/runtime/network/network.cjs ADDED Viewed

@@ -0,0 +1,91 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var network_exports = {};
+__export(network_exports, {
+  pageRequest: () => pageRequest
+});
+module.exports = __toCommonJS(network_exports);
+async function pageRequest(page, config, options) {
+  const { url, method = "GET", headers = {}, body, bodyType = "json", responseType = "json" } = config;
+  const { logger, schema } = options ?? {};
+  const startTime = Date.now();
+  const fetchHeaders = { ...headers };
+  let fetchBody;
+  if (body !== void 0) {
+    if (bodyType === "form") {
+      fetchHeaders["Content-Type"] = "application/x-www-form-urlencoded";
+      if (typeof body === "string") {
+        fetchBody = body;
+      } else {
+        fetchBody = new URLSearchParams(
+          Object.entries(body).map(([k, v]) => [k, String(v)])
+        ).toString();
+      }
+    } else {
+      fetchHeaders["Content-Type"] = "application/json";
+      fetchBody = typeof body === "string" ? body : JSON.stringify(body);
+    }
+  }
+  const result = await page.evaluate(
+    async ({ url: url2, method: method2, headers: headers2, body: body2, responseType: responseType2 }) => {
+      const res = await fetch(url2, {
+        method: method2,
+        headers: headers2,
+        body: body2 ?? void 0
+      });
+      const status = res.status;
+      const ok = res.ok;
+      let data;
+      if (responseType2 === "json") {
+        data = await res.json();
+      } else {
+        data = await res.text();
+      }
+      return { status, ok, data };
+    },
+    { url, method, headers: fetchHeaders, body: fetchBody, responseType }
+  );
+  const duration = Date.now() - startTime;
+  if (!result.ok) {
+    logger?.warn("network:request:error", {
+      method,
+      url,
+      status: result.status,
+      duration,
+      body: typeof result.data === "string" ? result.data.slice(0, 500) : void 0
+    });
+    throw new Error(
+      `pageRequest failed: ${method} ${url} returned ${result.status}`
+    );
+  }
+  logger?.info("network:request", {
+    method,
+    url,
+    status: result.status,
+    duration
+  });
+  if (schema) {
+    return schema.parse(result.data);
+  }
+  return result.data;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  pageRequest
+});

package/dist/runtime/network/network.d.cts ADDED Viewed

@@ -0,0 +1,28 @@
+import { Page } from 'playwright';
+import z from 'zod';
+import { MinimalLogger } from '../../shared/logger/logger.cjs';
+type RequestConfig = {
+    url: string;
+    method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+    headers?: Record<string, string>;
+    body?: Record<string, any> | string;
+    /** How to serialize the body. Defaults to "json". */
+    bodyType?: "json" | "form";
+    /** How to parse the response. Defaults to "json". */
+    responseType?: "json" | "text" | "xml";
+};
+type PageRequestOptions<T extends z.ZodType | undefined = undefined> = {
+    logger?: MinimalLogger;
+    /** Optional Zod schema to validate the response body. */
+    schema?: T;
+};
+type PageRequestResult<T extends z.ZodType | undefined> = T extends z.ZodType ? z.infer<T> : any;
+/**
+ * Executes a fetch() call inside the browser context via page.evaluate().
+ * Provides typed request config, automatic response parsing, optional Zod
+ * validation, and logging.
+ */
+declare function pageRequest<T extends z.ZodType | undefined = undefined>(page: Page, config: RequestConfig, options?: PageRequestOptions<T>): Promise<PageRequestResult<T>>;
+export { type PageRequestOptions, type RequestConfig, pageRequest };

package/dist/runtime/network/network.d.ts ADDED Viewed

@@ -0,0 +1,28 @@
+import { Page } from 'playwright';
+import z from 'zod';
+import { MinimalLogger } from '../../shared/logger/logger.js';
+type RequestConfig = {
+    url: string;
+    method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+    headers?: Record<string, string>;
+    body?: Record<string, any> | string;
+    /** How to serialize the body. Defaults to "json". */
+    bodyType?: "json" | "form";
+    /** How to parse the response. Defaults to "json". */
+    responseType?: "json" | "text" | "xml";
+};
+type PageRequestOptions<T extends z.ZodType | undefined = undefined> = {
+    logger?: MinimalLogger;
+    /** Optional Zod schema to validate the response body. */
+    schema?: T;
+};
+type PageRequestResult<T extends z.ZodType | undefined> = T extends z.ZodType ? z.infer<T> : any;
+/**
+ * Executes a fetch() call inside the browser context via page.evaluate().
+ * Provides typed request config, automatic response parsing, optional Zod
+ * validation, and logging.
+ */
+declare function pageRequest<T extends z.ZodType | undefined = undefined>(page: Page, config: RequestConfig, options?: PageRequestOptions<T>): Promise<PageRequestResult<T>>;
+export { type PageRequestOptions, type RequestConfig, pageRequest };

package/dist/runtime/network/network.js ADDED Viewed

@@ -0,0 +1,67 @@
+async function pageRequest(page, config, options) {
+  const { url, method = "GET", headers = {}, body, bodyType = "json", responseType = "json" } = config;
+  const { logger, schema } = options ?? {};
+  const startTime = Date.now();
+  const fetchHeaders = { ...headers };
+  let fetchBody;
+  if (body !== void 0) {
+    if (bodyType === "form") {
+      fetchHeaders["Content-Type"] = "application/x-www-form-urlencoded";
+      if (typeof body === "string") {
+        fetchBody = body;
+      } else {
+        fetchBody = new URLSearchParams(
+          Object.entries(body).map(([k, v]) => [k, String(v)])
+        ).toString();
+      }
+    } else {
+      fetchHeaders["Content-Type"] = "application/json";
+      fetchBody = typeof body === "string" ? body : JSON.stringify(body);
+    }
+  }
+  const result = await page.evaluate(
+    async ({ url: url2, method: method2, headers: headers2, body: body2, responseType: responseType2 }) => {
+      const res = await fetch(url2, {
+        method: method2,
+        headers: headers2,
+        body: body2 ?? void 0
+      });
+      const status = res.status;
+      const ok = res.ok;
+      let data;
+      if (responseType2 === "json") {
+        data = await res.json();
+      } else {
+        data = await res.text();
+      }
+      return { status, ok, data };
+    },
+    { url, method, headers: fetchHeaders, body: fetchBody, responseType }
+  );
+  const duration = Date.now() - startTime;
+  if (!result.ok) {
+    logger?.warn("network:request:error", {
+      method,
+      url,
+      status: result.status,
+      duration,
+      body: typeof result.data === "string" ? result.data.slice(0, 500) : void 0
+    });
+    throw new Error(
+      `pageRequest failed: ${method} ${url} returned ${result.status}`
+    );
+  }
+  logger?.info("network:request", {
+    method,
+    url,
+    status: result.status,
+    duration
+  });
+  if (schema) {
+    return schema.parse(result.data);
+  }
+  return result.data;
+}
+export {
+  pageRequest
+};

package/dist/runtime/recovery/agent.cjs ADDED Viewed

@@ -0,0 +1,223 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var agent_exports = {};
+__export(agent_exports, {
+  executeRecoveryAgent: () => executeRecoveryAgent
+});
+module.exports = __toCommonJS(agent_exports);
+var import_logger = require("../../shared/logger/logger.js");
+var import_zod = require("zod");
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+const KEY_MAPPINGS = {
+  ENTER: "Enter",
+  RETURN: "Enter",
+  TAB: "Tab",
+  SPACE: " ",
+  BACKSPACE: "Backspace",
+  DELETE: "Delete",
+  ESCAPE: "Escape",
+  ESC: "Escape",
+  UP: "ArrowUp",
+  DOWN: "ArrowDown",
+  LEFT: "ArrowLeft",
+  RIGHT: "ArrowRight",
+  HOME: "Home",
+  END: "End",
+  PAGEUP: "PageUp",
+  PAGEDOWN: "PageDown",
+  CTRL: "Control",
+  CONTROL: "Control",
+  ALT: "Alt",
+  SHIFT: "Shift",
+  META: "Meta",
+  CMD: "Meta",
+  COMMAND: "Meta"
+};
+function mapKeyName(key) {
+  return KEY_MAPPINGS[key.toUpperCase()] ?? key;
+}
+async function executeBrowserAction(page, action, logger = import_logger.defaultLogger) {
+  switch (action.type) {
+    case "click": {
+      const { x, y, button = "left" } = action;
+      const playwrightButton = button === "wheel" || button === "back" || button === "forward" ? "left" : button;
+      await page.mouse.click(x, y, { button: playwrightButton });
+      logger.info(`Clicked at (${x}, ${y}) with ${button} button`);
+      break;
+    }
+    case "double_click": {
+      const { x, y } = action;
+      await page.mouse.dblclick(x, y);
+      logger.info(`Double-clicked at (${x}, ${y})`);
+      break;
+    }
+    case "scroll": {
+      const { x, y, scroll_x, scroll_y } = action;
+      await page.mouse.move(x, y);
+      await page.evaluate(`window.scrollBy(${scroll_x}, ${scroll_y})`);
+      logger.info(`Scrolled at (${x}, ${y}) by (${scroll_x}, ${scroll_y})`);
+      break;
+    }
+    case "keypress": {
+      for (const key of action.keys) {
+        const mapped = mapKeyName(key);
+        await page.keyboard.press(mapped);
+        logger.info(`Pressed key: ${key} (mapped to ${mapped})`);
+      }
+      break;
+    }
+    case "type": {
+      await page.keyboard.type(action.text);
+      logger.info(`Typed text: ${action.text}`);
+      break;
+    }
+    case "wait": {
+      await delay(2e3);
+      logger.info("Waited 2 seconds");
+      break;
+    }
+    case "screenshot": {
+      logger.info("Screenshot action (no-op, taken automatically)");
+      break;
+    }
+    case "drag": {
+      const { path } = action;
+      const start = path[0];
+      const end = path[path.length - 1];
+      if (path.length >= 2 && start && end) {
+        await page.mouse.move(start.x, start.y);
+        await page.mouse.down();
+        for (let i = 1; i < path.length; i++) {
+          const point = path[i];
+          if (point) await page.mouse.move(point.x, point.y);
+        }
+        await page.mouse.up();
+        logger.info(`Dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
+      }
+      break;
+    }
+    case "move": {
+      const { x, y } = action;
+      await page.mouse.move(x, y);
+      logger.info(`Moved mouse to (${x}, ${y})`);
+      break;
+    }
+    case "done": {
+      break;
+    }
+  }
+}
+const recoveryActionSchema = import_zod.z.object({
+  reasoning: import_zod.z.string().describe("Your reasoning about what you see and what action to take"),
+  action: import_zod.z.discriminatedUnion("type", [
+    import_zod.z.object({
+      type: import_zod.z.literal("click"),
+      x: import_zod.z.number(),
+      y: import_zod.z.number()
+    }),
+    import_zod.z.object({
+      type: import_zod.z.literal("type"),
+      text: import_zod.z.string()
+    }),
+    import_zod.z.object({
+      type: import_zod.z.literal("keypress"),
+      keys: import_zod.z.array(import_zod.z.string())
+    }),
+    import_zod.z.object({
+      type: import_zod.z.literal("scroll"),
+      x: import_zod.z.number(),
+      y: import_zod.z.number(),
+      scroll_x: import_zod.z.number(),
+      scroll_y: import_zod.z.number()
+    }),
+    import_zod.z.object({
+      type: import_zod.z.literal("wait")
+    }),
+    import_zod.z.object({
+      type: import_zod.z.literal("done")
+    })
+  ])
+});
+async function executeRecoveryAgent(page, instruction, logger, llmClient) {
+  if (!llmClient) {
+    return;
+  }
+  const log = logger ?? import_logger.defaultLogger;
+  log.info("Executing vision-based recovery agent", { instruction });
+  const viewport = page.viewportSize();
+  if (!viewport) {
+    throw new Error("Viewport size not found");
+  }
+  let screenshot;
+  try {
+    screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
+  } catch (screenshotError) {
+    log.warn("Failed to take screenshot for recovery agent, skipping", {
+      screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
+    });
+    throw new Error("Failed to take screenshot for recovery agent");
+  }
+  const maxSteps = 3;
+  for (let step = 1; step <= maxSteps; step++) {
+    const result = await llmClient.generateObjectFromMessages({
+      schema: recoveryActionSchema,
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: `You are an expert browser support agent. Your job is to resolve issues when browser automation encounters unexpected website behavior (e.g., popups blocking interaction).
+Your task: ${instruction}
+Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
+Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`
+            },
+            {
+              type: "image",
+              image: `data:image/png;base64,${screenshot}`
+            }
+          ]
+        }
+      ],
+      temperature: 0
+    });
+    log.info(`Recovery step ${step}/${maxSteps}`, {
+      reasoning: result.reasoning,
+      action: result.action
+    });
+    if (result.action.type === "done") {
+      log.info("Recovery agent completed - no more actions needed");
+      break;
+    }
+    await executeBrowserAction(page, result.action, log);
+    await delay(2e3);
+    screenshot = (await page.screenshot({ fullPage: false })).toString(
+      "base64"
+    );
+  }
+  log.info("Recovery agent execution completed");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  executeRecoveryAgent
+});

package/dist/runtime/recovery/agent.d.cts ADDED Viewed

@@ -0,0 +1,13 @@
+import { Page } from 'playwright';
+import { MinimalLogger } from '../../shared/logger/logger.cjs';
+import { LLMClient } from '../../shared/llm/types.cjs';
+import 'zod';
+/**
+ * Executes a vision-based recovery agent to recover from browser automation failures.
+ * Takes a screenshot, sends it to the LLM with the instruction, and executes
+ * the LLM's suggested browser actions.
+ */
+declare function executeRecoveryAgent(page: Page, instruction: string, logger?: MinimalLogger, llmClient?: LLMClient): Promise<void>;
+export { executeRecoveryAgent };

package/dist/runtime/recovery/agent.d.ts ADDED Viewed

@@ -0,0 +1,13 @@
+import { Page } from 'playwright';
+import { MinimalLogger } from '../../shared/logger/logger.js';
+import { LLMClient } from '../../shared/llm/types.js';
+import 'zod';
+/**
+ * Executes a vision-based recovery agent to recover from browser automation failures.
+ * Takes a screenshot, sends it to the LLM with the instruction, and executes
+ * the LLM's suggested browser actions.
+ */
+declare function executeRecoveryAgent(page: Page, instruction: string, logger?: MinimalLogger, llmClient?: LLMClient): Promise<void>;
+export { executeRecoveryAgent };