libretto 0.2.0 → 0.2.2

package/dist/cli/core/session-telemetry.js

@@ -0,0 +1,491 @@
+async function installSessionTelemetry(options) {
+  const STATIC_EXT_RE = /\.(css|js|png|jpg|jpeg|gif|woff|woff2|ttf|ico|svg)(\?|$)/i;
+  const { context, initialPage, logAction, logNetwork } = options;
+  const includeUserDomActions = options.includeUserDomActions ?? false;
+  const pageIdCache = /* @__PURE__ */ new WeakMap();
+  const wrappedPages = /* @__PURE__ */ new WeakSet();
+  const exposedPages = /* @__PURE__ */ new WeakSet();
+  const resolvePageId = async (page) => {
+    if (pageIdCache.has(page)) return pageIdCache.get(page);
+    const cdpSession = await context.newCDPSession(page);
+    try {
+      const targetInfo = await cdpSession.send("Target.getTargetInfo");
+      const targetId = targetInfo?.targetInfo?.targetId;
+      if (typeof targetId !== "string" || targetId.length === 0) {
+        throw new Error(`Could not resolve target id for page at URL "${page.url()}".`);
+      }
+      pageIdCache.set(page, targetId);
+      return targetId;
+    } finally {
+      await cdpSession.detach();
+    }
+  };
+  const emitAction = (entry) => {
+    logAction({
+      ts: (/* @__PURE__ */ new Date()).toISOString(),
+      ...entry
+    });
+  };
+  const emitNetwork = (entry) => {
+    logNetwork({
+      ts: (/* @__PURE__ */ new Date()).toISOString(),
+      ...entry
+    });
+  };
+  const markApiActionInProgress = async (page, inProgress) => {
+    await page.evaluate((flag) => {
+      window.__btApiActionInProgress = flag;
+    }, inProgress);
+  };
+  const wrapLocator = (locator, page, pageId) => {
+    if (locator.__librettoActionLogged) return locator;
+    locator.__librettoActionLogged = true;
+    const locatorActionMethods = [
+      "click",
+      "dblclick",
+      "fill",
+      "type",
+      "press",
+      "check",
+      "uncheck",
+      "selectOption",
+      "hover",
+      "focus",
+      "scrollIntoViewIfNeeded",
+      "waitFor",
+      "innerHTML",
+      "innerText",
+      "textContent",
+      "inputValue",
+      "isChecked",
+      "isDisabled",
+      "isEditable",
+      "isEnabled",
+      "isHidden",
+      "isVisible",
+      "count",
+      "boundingBox",
+      "screenshot",
+      "evaluate",
+      "evaluateAll",
+      "evaluateHandle",
+      "getAttribute",
+      "dispatchEvent",
+      "setInputFiles",
+      "selectText",
+      "dragTo",
+      "highlight",
+      "tap"
+    ];
+    const locatorReturningMethods = [
+      "first",
+      "last",
+      "locator",
+      "getByRole",
+      "getByText",
+      "getByLabel",
+      "getByPlaceholder",
+      "getByAltText",
+      "getByTitle",
+      "getByTestId",
+      "filter",
+      "and",
+      "or"
+    ];
+    for (const actMethod of locatorActionMethods) {
+      if (typeof locator[actMethod] !== "function") continue;
+      const originalAction = locator[actMethod].bind(locator);
+      locator[actMethod] = async (...actionArgs) => {
+        const start = Date.now();
+        await markApiActionInProgress(page, true);
+        try {
+          const result = await originalAction(...actionArgs);
+          emitAction({
+            pageId,
+            action: actMethod,
+            source: "agent",
+            selector: "locator",
+            value: actionArgs[0] !== void 0 ? String(actionArgs[0]).slice(0, 100) : void 0,
+            duration: Date.now() - start,
+            success: true
+          });
+          return result;
+        } catch (error) {
+          emitAction({
+            pageId,
+            action: actMethod,
+            source: "agent",
+            selector: "locator",
+            duration: Date.now() - start,
+            success: false,
+            error: error?.message ?? String(error)
+          });
+          throw error;
+        } finally {
+          await markApiActionInProgress(page, false);
+        }
+      };
+    }
+    for (const method of locatorReturningMethods) {
+      if (typeof locator[method] !== "function") continue;
+      const originalMethod = locator[method].bind(locator);
+      locator[method] = (...args) => {
+        const child = originalMethod(...args);
+        return wrapLocator(child, page, pageId);
+      };
+    }
+    if (typeof locator.nth === "function") {
+      const originalNth = locator.nth.bind(locator);
+      locator.nth = (index) => {
+        const child = originalNth(index);
+        return wrapLocator(child, page, pageId);
+      };
+    }
+    if (typeof locator.all === "function") {
+      const originalAll = locator.all.bind(locator);
+      locator.all = async () => {
+        const items = await originalAll();
+        return items.map((item) => wrapLocator(item, page, pageId));
+      };
+    }
+    return locator;
+  };
+  const installUserDomTracking = async (page, pageId) => {
+    if (exposedPages.has(page)) return;
+    exposedPages.add(page);
+    await page.exposeFunction("__btActionLog", (jsonStr) => {
+      const parsed = JSON.parse(jsonStr);
+      emitAction({
+        pageId,
+        source: "user",
+        ...parsed
+      });
+    });
+    await page.addInitScript(() => {
+      if (window.__btDomListenersInstalled) return;
+      window.__btDomListenersInstalled = true;
+      const identify = (el) => {
+        if (!el || !el.tagName) return "";
+        const testId = el.getAttribute("data-testid");
+        if (testId) return `[data-testid="${testId}"]`;
+        const role = el.getAttribute("role") || "";
+        const id = el.id;
+        if (role && id) return `${role}#${id}`;
+        const label = el.getAttribute("aria-label") || (el.textContent || "").trim().slice(0, 30) || "";
+        if (role && label) return `${role} "${label}"`;
+        const tag = el.tagName.toLowerCase();
+        const cls = el.className && typeof el.className === "string" ? "." + el.className.trim().split(/\s+/).slice(0, 2).join(".") : "";
+        return `${tag}${cls}`;
+      };
+      let clickTimer = null;
+      let pendingClick = null;
+      document.addEventListener(
+        "click",
+        (event) => {
+          if (window.__btApiActionInProgress) return;
+          const target = event.target;
+          const selector = identify(target);
+          if (target?.type === "checkbox") {
+            window.__btActionLog(
+              JSON.stringify({
+                action: target.checked ? "check" : "uncheck",
+                selector,
+                success: true
+              })
+            );
+            return;
+          }
+          pendingClick = { selector };
+          if (clickTimer) clearTimeout(clickTimer);
+          clickTimer = setTimeout(() => {
+            if (pendingClick) {
+              window.__btActionLog(
+                JSON.stringify({
+                  action: "click",
+                  selector: pendingClick.selector,
+                  success: true
+                })
+              );
+            }
+            pendingClick = null;
+            clickTimer = null;
+          }, 200);
+        },
+        true
+      );
+      document.addEventListener(
+        "dblclick",
+        (event) => {
+          if (window.__btApiActionInProgress) return;
+          if (clickTimer) {
+            clearTimeout(clickTimer);
+            clickTimer = null;
+            pendingClick = null;
+          }
+          const selector = identify(event.target);
+          window.__btActionLog(
+            JSON.stringify({ action: "dblclick", selector, success: true })
+          );
+        },
+        true
+      );
+      const inputTimers = /* @__PURE__ */ new WeakMap();
+      document.addEventListener(
+        "input",
+        (event) => {
+          if (window.__btApiActionInProgress) return;
+          const target = event.target;
+          const selector = identify(target);
+          if (target.tagName === "SELECT") {
+            window.__btActionLog(
+              JSON.stringify({
+                action: "selectOption",
+                selector,
+                value: target.value,
+                success: true
+              })
+            );
+            return;
+          }
+          const existing = inputTimers.get(target);
+          if (existing) clearTimeout(existing);
+          inputTimers.set(
+            target,
+            setTimeout(() => {
+              inputTimers.delete(target);
+              window.__btActionLog(
+                JSON.stringify({
+                  action: "fill",
+                  selector,
+                  value: (target.value || "").slice(0, 100),
+                  success: true
+                })
+              );
+            }, 500)
+          );
+        },
+        true
+      );
+      const specialKeys = /* @__PURE__ */ new Set([
+        "Enter",
+        "Escape",
+        "Tab",
+        "Backspace",
+        "Delete",
+        "ArrowUp",
+        "ArrowDown",
+        "ArrowLeft",
+        "ArrowRight",
+        "Home",
+        "End",
+        "PageUp",
+        "PageDown",
+        "F1",
+        "F2",
+        "F3",
+        "F4",
+        "F5",
+        "F6",
+        "F7",
+        "F8",
+        "F9",
+        "F10",
+        "F11",
+        "F12"
+      ]);
+      document.addEventListener(
+        "keydown",
+        (event) => {
+          if (window.__btApiActionInProgress) return;
+          const isShortcut = event.ctrlKey || event.metaKey || event.altKey;
+          if (!isShortcut && !specialKeys.has(event.key)) return;
+          const selector = identify(event.target);
+          const keyDesc = (event.ctrlKey ? "Ctrl+" : "") + (event.metaKey ? "Meta+" : "") + (event.altKey ? "Alt+" : "") + (event.shiftKey ? "Shift+" : "") + event.key;
+          window.__btActionLog(
+            JSON.stringify({
+              action: "press",
+              selector,
+              value: keyDesc,
+              success: true
+            })
+          );
+        },
+        true
+      );
+      let scrollTimer = null;
+      document.addEventListener(
+        "scroll",
+        () => {
+          if (window.__btApiActionInProgress) return;
+          if (scrollTimer) clearTimeout(scrollTimer);
+          scrollTimer = setTimeout(() => {
+            scrollTimer = null;
+            window.__btActionLog(
+              JSON.stringify({
+                action: "scroll",
+                selector: "document",
+                value: `y=${window.scrollY}`,
+                success: true
+              })
+            );
+          }, 300);
+        },
+        true
+      );
+    });
+  };
+  const wrapPageActions = (page, pageId) => {
+    if (wrappedPages.has(page)) return;
+    wrappedPages.add(page);
+    const pageActions = [
+      "click",
+      "dblclick",
+      "fill",
+      "type",
+      "press",
+      "check",
+      "uncheck",
+      "selectOption",
+      "hover",
+      "focus"
+    ];
+    const navActions = ["goto", "reload", "goBack", "goForward"];
+    const locatorFactories = [
+      "locator",
+      "getByRole",
+      "getByText",
+      "getByLabel",
+      "getByPlaceholder",
+      "getByAltText",
+      "getByTitle",
+      "getByTestId"
+    ];
+    for (const method of pageActions) {
+      const originalMethod = page[method].bind(page);
+      page[method] = async (...args) => {
+        const start = Date.now();
+        await markApiActionInProgress(page, true);
+        try {
+          const result = await originalMethod(...args);
+          emitAction({
+            pageId,
+            action: method,
+            source: "agent",
+            selector: typeof args[0] === "string" ? args[0] : void 0,
+            value: args[1] !== void 0 ? String(args[1]).slice(0, 100) : void 0,
+            duration: Date.now() - start,
+            success: true
+          });
+          return result;
+        } catch (error) {
+          emitAction({
+            pageId,
+            action: method,
+            source: "agent",
+            selector: typeof args[0] === "string" ? args[0] : void 0,
+            duration: Date.now() - start,
+            success: false,
+            error: error?.message ?? String(error)
+          });
+          throw error;
+        } finally {
+          await markApiActionInProgress(page, false);
+        }
+      };
+    }
+    for (const method of navActions) {
+      const originalMethod = page[method].bind(page);
+      page[method] = async (...args) => {
+        const start = Date.now();
+        try {
+          const result = await originalMethod(...args);
+          emitAction({
+            pageId,
+            action: method,
+            source: "agent",
+            url: typeof args[0] === "string" ? args[0] : page.url(),
+            duration: Date.now() - start,
+            success: true
+          });
+          return result;
+        } catch (error) {
+          emitAction({
+            pageId,
+            action: method,
+            source: "agent",
+            url: typeof args[0] === "string" ? args[0] : void 0,
+            duration: Date.now() - start,
+            success: false,
+            error: error?.message ?? String(error)
+          });
+          throw error;
+        }
+      };
+    }
+    for (const factory of locatorFactories) {
+      const originalFactory = page[factory].bind(page);
+      page[factory] = (...factoryArgs) => {
+        const locator = originalFactory(...factoryArgs);
+        return wrapLocator(locator, page, pageId);
+      };
+    }
+  };
+  const installForPage = async (page) => {
+    const pageId = await resolvePageId(page);
+    wrapPageActions(page, pageId);
+    if (includeUserDomActions) {
+      await installUserDomTracking(page, pageId);
+    }
+    page.on("response", async (response) => {
+      const request = response.request();
+      const url = request.url();
+      if (STATIC_EXT_RE.test(url) || url.startsWith("chrome-extension://")) return;
+      emitNetwork({
+        pageId,
+        method: request.method(),
+        url,
+        status: response.status(),
+        contentType: response.headers()["content-type"] ?? null,
+        postData: request.method() === "POST" || request.method() === "PUT" || request.method() === "PATCH" ? (request.postData() ?? "").substring(0, 2e3) : void 0,
+        responseBody: null,
+        size: null,
+        durationMs: null
+      });
+    });
+    page.on("framenavigated", (frame) => {
+      if (frame !== page.mainFrame()) return;
+      emitAction({
+        pageId,
+        action: "navigate",
+        source: "agent",
+        url: frame.url(),
+        success: true
+      });
+    });
+    page.on("popup", (popup) => {
+      emitAction({
+        pageId,
+        action: "popup",
+        source: "agent",
+        url: popup.url(),
+        success: true
+      });
+    });
+    page.on("dialog", (dialog) => {
+      emitAction({
+        pageId,
+        action: "dialog",
+        source: "agent",
+        value: `${dialog.type()}: ${dialog.message().slice(0, 500)}`,
+        success: true
+      });
+    });
+  };
+  await installForPage(initialPage);
+  context.on("page", (newPage) => {
+    void installForPage(newPage);
+  });
+}
+export {
+  installSessionTelemetry
+};

package/dist/cli/core/telemetry.js

@@ -20,6 +20,9 @@ function readNetworkLog(session, opts = {}) {
     const re = new RegExp(opts.filter, "i");
     entries = entries.filter((e) => re.test(e.url));
   }
+  if (opts.pageId) {
+    entries = entries.filter((e) => e.pageId === opts.pageId);
+  }
   const last = opts.last ?? 20;
   if (entries.length > last) {
     entries = entries.slice(-last);
@@ -79,6 +82,9 @@ function readActionLog(session, opts = {}) {
       (e) => re.test(e.action) || re.test(e.selector || "") || re.test(e.value || "") || re.test(e.url || "")
     );
   }
+  if (opts.pageId) {
+    entries = entries.filter((e) => e.pageId === opts.pageId);
+  }
   const last = opts.last ?? 20;
   if (entries.length > last) {
     entries = entries.slice(-last);
@@ -157,7 +163,7 @@ function formatHint(method, args) {
   const formatted = args.map((a) => JSON.stringify(a)).join(", ");
   return `${method}(${formatted})`;
 }
-function wrapLocator(locator, hint, session, page, onActivity) {
+function wrapLocator(locator, hint, session, page, pageId, onActivity) {
   if (locator.__librettoActionLogged) return locator;
   locator.__librettoActionLogged = true;
   for (const actMethod of LOCATOR_ACTION_METHODS) {
@@ -174,6 +180,7 @@ function wrapLocator(locator, hint, session, page, onActivity) {
       try {
         const result = await origAct(...actArgs);
         parentLogAction(session, {
+          pageId,
           action: actMethod,
           source: "agent",
           selector: hint,
@@ -185,6 +192,7 @@ function wrapLocator(locator, hint, session, page, onActivity) {
         return result;
       } catch (err) {
         parentLogAction(session, {
+          pageId,
           action: actMethod,
           source: "agent",
           selector: hint,
@@ -210,7 +218,7 @@ function wrapLocator(locator, hint, session, page, onActivity) {
     locator[method] = (...args) => {
       const child = origMethod(...args);
       const childHint = args.length > 0 ? `${hint}.${formatHint(method, args)}` : `${hint}.${method}()`;
-      return wrapLocator(child, childHint, session, page, onActivity);
+      return wrapLocator(child, childHint, session, page, pageId, onActivity);
     };
   }
   if (typeof locator.nth === "function") {
@@ -218,7 +226,7 @@ function wrapLocator(locator, hint, session, page, onActivity) {
     locator.nth = (index) => {
       const child = origNth(index);
       const childHint = `${hint}.nth(${index})`;
-      return wrapLocator(child, childHint, session, page, onActivity);
+      return wrapLocator(child, childHint, session, page, pageId, onActivity);
     };
   }
   if (typeof locator.all === "function") {
@@ -227,13 +235,13 @@ function wrapLocator(locator, hint, session, page, onActivity) {
       const items = await origAll();
       return items.map((item, i) => {
         const childHint = `${hint}.all()[${i}]`;
-        return wrapLocator(item, childHint, session, page, onActivity);
+        return wrapLocator(item, childHint, session, page, pageId, onActivity);
       });
     };
   }
   return locator;
 }
-function wrapPageForActionLogging(page, session, onActivity) {
+function wrapPageForActionLogging(page, session, pageId, onActivity) {
   const PAGE_ACTIONS = [
     "click",
     "dblclick",
@@ -260,6 +268,7 @@ function wrapPageForActionLogging(page, session, onActivity) {
       try {
         const result = await orig(...args);
         parentLogAction(session, {
+          pageId,
           action: method,
           source: "agent",
           selector: typeof args[0] === "string" ? args[0] : void 0,
@@ -271,6 +280,7 @@ function wrapPageForActionLogging(page, session, onActivity) {
         return result;
       } catch (err) {
         parentLogAction(session, {
+          pageId,
           action: method,
           source: "agent",
           selector: typeof args[0] === "string" ? args[0] : void 0,
@@ -297,6 +307,7 @@ function wrapPageForActionLogging(page, session, onActivity) {
       try {
         const result = await orig(...args);
         parentLogAction(session, {
+          pageId,
           action: method,
           source: "agent",
           url: typeof args[0] === "string" ? args[0] : page.url(),
@@ -307,6 +318,7 @@ function wrapPageForActionLogging(page, session, onActivity) {
         return result;
       } catch (err) {
         parentLogAction(session, {
+          pageId,
           action: method,
           source: "agent",
           url: typeof args[0] === "string" ? args[0] : void 0,
@@ -334,7 +346,7 @@ function wrapPageForActionLogging(page, session, onActivity) {
     page[factory] = (...factoryArgs) => {
       const locator = orig(...factoryArgs);
       const hint = formatHint(factory, factoryArgs);
-      return wrapLocator(locator, hint, session, page, onActivity);
+      return wrapLocator(locator, hint, session, page, pageId, onActivity);
     };
   }
 }

package/dist/cli/workers/run-integration-runtime.js

@@ -7,8 +7,13 @@ import {
   launchBrowser
 } from "../../index.js";
 import { getProfilePath, normalizeDomain } from "../core/browser.js";
-import { getSessionDir } from "../core/context.js";
+import {
+  getSessionActionsLogPath,
+  getSessionDir,
+  getSessionNetworkLogPath
+} from "../core/context.js";
 import { getPauseSignalPaths, removeSignalIfExists } from "../core/pause-signals.js";
+import { installSessionTelemetry } from "../core/session-telemetry.js";
 const LIBRETTO_WORKFLOW_BRAND = /* @__PURE__ */ Symbol.for("libretto.workflow");
 const RESUME_POLL_INTERVAL_MS = 250;
 function mirrorStdoutToFile(filePath) {
@@ -136,6 +141,19 @@ async function runIntegrationInternal(args, options) {
     headless: args.headless,
     storageStatePath
   });
+  const actionsLogPath = getSessionActionsLogPath(args.session);
+  const networkLogPath = getSessionNetworkLogPath(args.session);
+  await installSessionTelemetry({
+    context: browserSession.context,
+    initialPage: browserSession.page,
+    includeUserDomActions: true,
+    logAction: (entry) => {
+      appendFileSync(actionsLogPath, JSON.stringify(entry) + "\n");
+    },
+    logNetwork: (entry) => {
+      appendFileSync(networkLogPath, JSON.stringify(entry) + "\n");
+    }
+  });
   const workflowContext = {
     logger: integrationLogger,
     page: browserSession.page,

package/dist/index.cjs

@@ -29,8 +29,8 @@ __export(index_exports, {
   attemptWithRecovery: () => import_recovery.attemptWithRecovery,
   clearHighlights: () => import_highlight.clearHighlights,
   createFileLogSink: () => import_sinks.createFileLogSink,
-  createRunner: () => import_runner.createRunner,
   debugPause: () => import_pause.debugPause,
+  defaultLogger: () => import_logger.defaultLogger,
   detectSubmissionError: () => import_errors.detectSubmissionError,
   downloadAndSave: () => import_download.downloadAndSave,
   downloadViaClick: () => import_download.downloadViaClick,
@@ -58,12 +58,9 @@ __export(index_exports, {
   serializeSessionState: () => import_state.serializeSessionState,
   shouldPauseBeforeMutation: () => import_config.shouldPauseBeforeMutation,
   showHighlight: () => import_highlight.showHighlight,
-  step: () => import_step.step,
   workflow: () => import_workflow.workflow
 });
 module.exports = __toCommonJS(index_exports);
-var import_step = require("./runtime/step/step.js");
-var import_runner = require("./runtime/step/runner.js");
 var import_logger = require("./shared/logger/logger.js");
 var import_sinks = require("./shared/logger/sinks.js");
 var import_state = require("./shared/state/index.js");
@@ -93,8 +90,8 @@ var import_workflow = require("./shared/workflow/workflow.js");
   attemptWithRecovery,
   clearHighlights,
   createFileLogSink,
-  createRunner,
   debugPause,
+  defaultLogger,
   detectSubmissionError,
   downloadAndSave,
   downloadViaClick,
@@ -122,6 +119,5 @@ var import_workflow = require("./shared/workflow/workflow.js");
   serializeSessionState,
   shouldPauseBeforeMutation,
   showHighlight,
-  step,
   workflow
 });

package/dist/index.d.cts

@@ -1,7 +1,4 @@
-export { step } from './runtime/step/step.cjs';
-export { Runner, createRunner } from './runtime/step/runner.cjs';
-export { DebugBundle, RecoveryHandler, RunnerConfig, Step, StepContext, StepHandler, StepHistoryEntry, StepOptions } from './runtime/step/types.cjs';
-export { LogOptions, Logger, LoggerApi, LoggerSink } from './shared/logger/logger.cjs';
+export { LogOptions, Logger, LoggerApi, LoggerSink, MinimalLogger, defaultLogger } from './shared/logger/logger.cjs';
 export { createFileLogSink, jsonlConsoleSink, prettyConsoleSink } from './shared/logger/sinks.cjs';
 export { LLMClient, Message, MessageContentPart } from './shared/llm/types.cjs';
 export { SESSION_STATE_VERSION, SessionState, SessionStateFile, SessionStateFileSchema, SessionStatus, SessionStatusSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState } from './shared/state/session-state.cjs';
@@ -18,5 +15,5 @@ export { GhostCursorOptions, ensureGhostCursor, ghostClick, hideGhostCursor, mov
 export { HighlightOptions, clearHighlights, ensureHighlightLayer, showHighlight } from './shared/visualization/highlight.cjs';
 export { BrowserSession, LaunchBrowserArgs, launchBrowser } from './shared/run/browser.cjs';
 export { LIBRETTO_WORKFLOW_BRAND, LibrettoAuthProfile, LibrettoWorkflow, LibrettoWorkflowContext, LibrettoWorkflowHandler, LibrettoWorkflowMetadata, workflow } from './shared/workflow/workflow.cjs';
-import 'playwright';
 import 'zod';
+import 'playwright';