libretto 0.6.11 → 0.6.13

package/dist/cli/core/daemon/daemon.js

@@ -2,51 +2,133 @@ import {
   chromium
 } from "playwright";
 import { mkdir } from "node:fs/promises";
-import { appendFileSync } from "node:fs";
+import { appendFileSync, existsSync, writeFileSync } from "node:fs";
 import { installSessionTelemetry } from "../session-telemetry.js";
+import {
+  createIpcPeer
+} from "../../../shared/ipc/ipc.js";
+import {
+  createIpcSocketServer,
+  listenOnIpcSocket
+} from "../../../shared/ipc/socket-transport.js";
 import {
   createLoggerForSession,
   getSessionDir,
   getSessionNetworkLogPath,
-  getSessionActionsLogPath
+  getSessionActionsLogPath,
+  getSessionProviderClosePath,
+  getSessionStatePath
 } from "../context.js";
 import {
-  DaemonServer,
   getDaemonSocketPath
 } from "./ipc.js";
 import { wrapPageForActionLogging } from "../telemetry.js";
+import {
+  getProfilePath,
+  hasProfile,
+  normalizeDomain,
+  normalizeUrl
+} from "../browser.js";
 import { handlePages } from "./pages.js";
 import { handleExec, handleReadonlyExec } from "./exec.js";
-import { handleSnapshot } from "./snapshot.js";
+import { handleCompactSnapshot } from "./snapshot.js";
+import { librettoCommand } from "../../../shared/package-manager.js";
+import { snapshot } from "../../../shared/snapshot/capture-snapshot.js";
+import { diffSnapshots } from "../../../shared/snapshot/diff-snapshots.js";
+import {
+  installPageStabilityWaiter,
+  preparePageStabilityWait,
+  waitForPageStable
+} from "../../../shared/snapshot/wait-for-page-stable.js";
+import { getCloudProviderApi } from "../providers/index.js";
 import {
-  isConnectConfig
-} from "./config.js";
+  getAbsoluteIntegrationPath,
+  loadDefaultWorkflow
+} from "../workflow-runtime.js";
+import { WorkflowController } from "../workflow-runner/runner.js";
 function isOperationalPage(page) {
   const url = page.url();
   return !url.startsWith("devtools://") && !url.startsWith("chrome-error://");
 }
+async function waitForSessionState(session) {
+  const deadline = Date.now() + 2e3;
+  while (Date.now() < deadline) {
+    if (existsSync(getSessionStatePath(session))) return;
+    await new Promise((resolve) => setTimeout(resolve, 25));
+  }
+  throw new Error(
+    `Session state was not written before workflow start for "${session}".`
+  );
+}
+class UserFacingStartupError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "UserFacingStartupError";
+  }
+}
+function getMissingLocalAuthProfileError(args) {
+  return [
+    `Local auth profile not found for domain "${args.normalizedDomain}".`,
+    `Expected profile file: ${args.profilePath}`,
+    "To create it:",
+    `  1. libretto open https://${args.normalizedDomain} --headed --session ${args.session}`,
+    "  2. Log in manually in the browser window.",
+    `  3. libretto save ${args.normalizedDomain} --session ${args.session}`
+  ].join("\n");
+}
+function resolveAuthProfileStorageStatePath(args) {
+  if (!args.authProfileDomain) return void 0;
+  const normalizedDomain = normalizeDomain(
+    normalizeUrl(args.authProfileDomain)
+  );
+  const profilePath = getProfilePath(normalizedDomain);
+  if (!hasProfile(normalizedDomain)) {
+    throw new UserFacingStartupError(
+      getMissingLocalAuthProfileError({
+        normalizedDomain,
+        profilePath,
+        session: args.session
+      })
+    );
+  }
+  return profilePath;
+}
 const PROTOCOL_VERSION = 1;
 const REQUEST_TIMEOUT_MS = 6e4;
 class BrowserDaemon {
-  constructor(session, externallyManaged, browser, context, page, ipcServer, logger) {
+  constructor(session, experiments, externallyManaged, browser, context, page, logger, providerSession) {
     this.session = session;
+    this.experiments = experiments;
     this.externallyManaged = externallyManaged;
     this.browser = browser;
     this.context = context;
     this.page = page;
-    this.ipcServer = ipcServer;
+    this.providerSession = providerSession;
     this.logger = logger.withScope("child");
   }
   logger;
   execState = {};
   pageById = /* @__PURE__ */ new Map();
-  shuttingDown = false;
+  shutdownHandlers = [];
+  connectedClis = /* @__PURE__ */ new Set();
+  workflowController;
+  shutdownPromise;
+  latestCompactSnapshotByPage = /* @__PURE__ */ new WeakMap();
   trackPage(page) {
     const id = `page-${Math.random().toString(36).slice(2, 5)}`;
     this.pageById.set(id, page);
     page.on("close", () => this.pageById.delete(id));
     return id;
   }
+  async installCompactSnapshotWaiter(page) {
+    const result = await preparePageStabilityWait(page, { timeoutMs: 1e3 });
+    if (!result.ok) {
+      this.logger.warn("compact-snapshot-waiter-install-incomplete", {
+        session: this.session,
+        diagnostics: result.diagnostics
+      });
+    }
+  }
   // ── Shared initialization ──────────────────────────────────────────
   /**
    * Common setup after the mode-specific code has obtained a browser,
@@ -56,12 +138,15 @@ class BrowserDaemon {
   static async initialize(args) {
     const {
       session,
+      experiments,
       externallyManaged,
       browser,
       context,
       page,
       initialPages,
-      navigateUrl
+      navigateUrl,
+      readyProvider,
+      providerSession
     } = args;
     await mkdir(getSessionDir(session), { recursive: true });
     const networkLogFile = getSessionNetworkLogPath(session);
@@ -85,34 +170,70 @@ class BrowserDaemon {
         error: err instanceof Error ? err.message : String(err)
       });
     }
+    await context.addInitScript(installPageStabilityWaiter);
     const socketPath = getDaemonSocketPath(session);
-    let handler;
-    const ipcServer = new DaemonServer(
-      socketPath,
-      (request) => handler(request)
-    );
     const daemon = new BrowserDaemon(
       session,
+      experiments,
       externallyManaged,
       browser,
       context,
       page,
-      ipcServer,
-      logger
+      logger,
+      providerSession
     );
     for (const p of initialPages) {
       wrapPageForActionLogging(p, session);
       daemon.trackPage(p);
     }
+    await Promise.all(
+      initialPages.map(
+        (initialPage) => daemon.installCompactSnapshotWaiter(initialPage)
+      )
+    );
     context.on("page", (newPage) => {
       wrapPageForActionLogging(newPage, session);
       daemon.trackPage(newPage);
+      void daemon.installCompactSnapshotWaiter(newPage);
     });
     if (navigateUrl) {
       await page.goto(navigateUrl);
     }
-    handler = (request) => daemon.handleRequest(request);
-    await ipcServer.listen();
+    const ipcServer = createIpcSocketServer((transport) => {
+      const cli = createIpcPeer(
+        transport,
+        daemon.createIpcHandlers()
+      );
+      const stopTracking = transport.onClose?.(() => {
+        daemon.connectedClis.delete(cli);
+        stopTracking?.();
+      });
+      daemon.connectedClis.add(cli);
+    });
+    daemon.registerShutdownHandler(async (options) => {
+      if (!options.keepIpcClientsAlive) {
+        for (const cli of daemon.connectedClis) {
+          cli.destroy();
+        }
+        daemon.connectedClis.clear();
+      }
+      if (options.keepIpcClientsAlive) {
+        ipcServer.close((error) => {
+          if (error) {
+            daemon.logger.warn("ipc-server-close-failed", {
+              session,
+              error
+            });
+          }
+        });
+        return;
+      }
+      await new Promise((resolve, reject) => {
+        ipcServer.close((error) => error ? reject(error) : resolve());
+      });
+    });
+    await listenOnIpcSocket(ipcServer, socketPath);
+    process.send?.({ type: "ready", socketPath, provider: readyProvider });
     daemon.logger.info("ipc-server-listening", { socketPath });
     browser.on("disconnected", () => {
       void daemon.shutdown("browser-disconnected-exiting", false);
@@ -120,20 +241,25 @@ class BrowserDaemon {
     return daemon;
   }
   // ── Launch mode ────────────────────────────────────────────────────
-  static async launchBrowser(config) {
+  static async launchBrowser(args) {
+    const { session, browser: config } = args;
     const windowPositionArg = config.windowPosition ? `--window-position=${config.windowPosition.x},${config.windowPosition.y}` : void 0;
     const browser = await chromium.launch({
       headless: !config.headed,
       args: [
         "--disable-blink-features=AutomationControlled",
-        `--remote-debugging-port=${config.port}`,
+        ...config.remoteDebuggingPort ? [`--remote-debugging-port=${config.remoteDebuggingPort}`] : [],
         "--remote-debugging-address=127.0.0.1",
         "--no-focus-on-check",
         ...windowPositionArg ? [windowPositionArg] : []
       ]
     });
+    const storageStatePath = config.storageStatePath ?? resolveAuthProfileStorageStatePath({
+      authProfileDomain: args.workflow?.authProfileDomain,
+      session
+    });
     const context = await browser.newContext({
-      ...config.storageStatePath ? { storageState: config.storageStatePath } : {},
+      ...storageStatePath ? { storageState: storageStatePath } : {},
       viewport: {
         width: config.viewport.width,
         height: config.viewport.height
@@ -144,136 +270,404 @@ class BrowserDaemon {
     page.setDefaultTimeout(3e4);
     page.setDefaultNavigationTimeout(45e3);
     const daemon = await BrowserDaemon.initialize({
-      session: config.session,
+      session,
+      experiments: args.experiments,
       externallyManaged: false,
       browser,
       context,
       page,
       initialPages: [page],
-      navigateUrl: config.url
+      navigateUrl: config.initialUrl
     });
     daemon.logger.info("child-launched", {
-      port: config.port,
+      port: config.remoteDebuggingPort,
       pid: process.pid,
-      session: config.session
+      session
     });
     return daemon;
   }
   // ── Connect mode ───────────────────────────────────────────────────
-  static async connectToEndpoint(config) {
+  static async connectToEndpoint(args) {
+    const { session, browser: config } = args;
     const browser = await chromium.connectOverCDP(config.cdpEndpoint);
     const contexts = browser.contexts();
     const context = contexts.length > 0 ? contexts[0] : await browser.newContext();
     const operationalPages = context.pages().filter(isOperationalPage);
     const page = operationalPages.length > 0 ? operationalPages[operationalPages.length - 1] : await context.newPage();
     const daemon = await BrowserDaemon.initialize({
-      session: config.session,
+      session,
+      experiments: args.experiments,
       externallyManaged: true,
       browser,
       context,
       page,
       initialPages: operationalPages.length > 0 ? operationalPages : [page],
-      navigateUrl: config.url
+      navigateUrl: config.initialUrl
     });
     daemon.logger.info("child-connected", {
       cdpEndpoint: config.cdpEndpoint,
-      url: config.url,
+      url: config.initialUrl,
       pid: process.pid,
-      session: config.session
+      session
     });
     return daemon;
   }
+  static async connectToProvider(args) {
+    const { session, browser: config } = args;
+    const provider = getCloudProviderApi(config.providerName);
+    const providerSession = await provider.createSession();
+    try {
+      const browser = await chromium.connectOverCDP(
+        providerSession.cdpEndpoint
+      );
+      const contexts = browser.contexts();
+      const context = contexts.length > 0 ? contexts[0] : await browser.newContext();
+      const operationalPages = context.pages().filter(isOperationalPage);
+      const page = operationalPages.length > 0 ? operationalPages[operationalPages.length - 1] : await context.newPage();
+      const daemon = await BrowserDaemon.initialize({
+        session,
+        experiments: args.experiments,
+        externallyManaged: true,
+        browser,
+        context,
+        page,
+        initialPages: operationalPages.length > 0 ? operationalPages : [page],
+        navigateUrl: config.initialUrl,
+        readyProvider: {
+          name: config.providerName,
+          sessionId: providerSession.sessionId,
+          cdpEndpoint: providerSession.cdpEndpoint,
+          liveViewUrl: providerSession.liveViewUrl
+        },
+        providerSession: {
+          provider,
+          name: config.providerName,
+          sessionId: providerSession.sessionId
+        }
+      });
+      daemon.logger.info("child-provider-connected", {
+        provider: config.providerName,
+        sessionId: providerSession.sessionId,
+        url: config.initialUrl,
+        pid: process.pid,
+        session
+      });
+      return daemon;
+    } catch (error) {
+      await provider.closeSession(providerSession.sessionId);
+      throw error;
+    }
+  }
   // ── Lifecycle ──────────────────────────────────────────────────────
-  async shutdown(reason, closeBrowser) {
-    if (this.shuttingDown) return;
-    this.shuttingDown = true;
-    this.logger.info(reason, { session: this.session });
-    await this.ipcServer.close();
-    if (!closeBrowser) return;
-    if (this.externallyManaged) {
-      try {
-        this.browser._connection?.close();
-      } catch {
+  registerShutdownHandler(handler) {
+    this.shutdownHandlers.push(handler);
+  }
+  shutdown(reason, closeBrowser, options = {}) {
+    if (this.shutdownPromise) return this.shutdownPromise;
+    this.shutdownPromise = this.performShutdown(reason, closeBrowser, options);
+    return this.shutdownPromise;
+  }
+  async performShutdown(reason, closeBrowser, options) {
+    let replayUrl;
+    try {
+      this.logger.info(reason, { session: this.session });
+      for (const handler of this.shutdownHandlers) {
+        await handler(options);
+      }
+      if (closeBrowser) {
+        if (this.externallyManaged) {
+          try {
+            this.browser._connection?.close();
+          } catch {
+          }
+        } else {
+          await this.browser.close();
+        }
+      }
+      if (this.providerSession) {
+        const result = await this.providerSession.provider.closeSession(
+          this.providerSession.sessionId
+        );
+        replayUrl = result.replayUrl;
+        if (result.replayUrl) {
+          this.logger.info("provider-recording", {
+            session: this.session,
+            provider: this.providerSession.name,
+            sessionId: this.providerSession.sessionId,
+            replayUrl: result.replayUrl
+          });
+        }
+        writeFileSync(
+          getSessionProviderClosePath(this.session),
+          JSON.stringify(
+            {
+              provider: this.providerSession.name,
+              sessionId: this.providerSession.sessionId,
+              replayUrl: result.replayUrl
+            },
+            null,
+            2
+          ),
+          "utf8"
+        );
+      }
+      return replayUrl ? { replayUrl } : {};
+    } finally {
+      if (options.keepIpcClientsAlive) {
+        setImmediate(() => {
+          for (const cli of this.connectedClis) {
+            cli.destroy();
+          }
+          this.connectedClis.clear();
+        });
       }
-    } else {
-      await this.browser.close();
     }
   }
   // ── Page resolution ────────────────────────────────────────────────
   resolveTargetPage(pageId) {
     if (!pageId) {
-      if (this.pageById.size > 1) {
+      if (this.page.isClosed()) {
+        const openPages = Array.from(this.pageById.values());
+        if (openPages.length === 1) return openPages[0];
         throw new Error(
-          `Multiple pages are open in session "${this.session}". Pass --page <id> to target a page (run "libretto pages --session ${this.session}" to list ids).`
+          `The primary page for session "${this.session}" is closed. Run "${librettoCommand(`pages --session ${this.session}`)}" to choose a page id.`
         );
       }
-      if (this.pageById.size === 1) {
-        return this.pageById.values().next().value;
-      }
       return this.page;
     }
     const page = this.pageById.get(pageId);
     if (!page) {
       throw new Error(
-        `Page "${pageId}" was not found in session "${this.session}". Run "libretto pages --session ${this.session}" to list ids.`
+        `Page "${pageId}" was not found in session "${this.session}". Run "${librettoCommand(`pages --session ${this.session}`)}" to list ids.`
       );
     }
     return page;
   }
-  // ── IPC handler ────────────────────────────────────────────────────
-  async handleRequest(request) {
-    if (request.command === "ping") {
-      return { protocolVersion: PROTOCOL_VERSION };
+  // ── IPC handlers ───────────────────────────────────────────────────
+  createIpcHandlers() {
+    return {
+      ping: () => ({ protocolVersion: PROTOCOL_VERSION }),
+      pages: () => this.withRequestTimeout(() => handlePages(this.pageById, this.page)),
+      exec: (args) => this.runExec(args),
+      readonlyExec: (args) => this.runReadonlyExec(args),
+      snapshot: (args) => this.runSnapshot(args),
+      getWorkflowStatus: () => this.getWorkflowStatus(),
+      resumeWorkflow: () => this.resumeWorkflow(),
+      close: () => this.shutdown("ipc-close", true, {
+        keepIpcClientsAlive: true
+      })
+    };
+  }
+  async runSnapshot(args) {
+    const targetPage = this.resolveTargetPage(args.pageId);
+    const result = await this.withRequestTimeout(
+      () => handleCompactSnapshot(
+        targetPage,
+        this.session,
+        this.logger,
+        {
+          pageId: args.pageId,
+          cachedSnapshot: this.latestCompactSnapshotByPage.get(targetPage),
+          useCachedSnapshot: args.useCachedSnapshot
+        }
+      )
+    );
+    if (!args.useCachedSnapshot) {
+      this.latestCompactSnapshotByPage.set(targetPage, result.snapshot);
     }
+    return result;
+  }
+  async withRequestTimeout(operation) {
     let timerId;
-    return Promise.race([
-      this.dispatchCommand(request).finally(() => clearTimeout(timerId)),
-      new Promise((_resolve, reject) => {
-        timerId = setTimeout(
-          () => reject(
-            new Error(`Request timed out after ${REQUEST_TIMEOUT_MS}ms`)
-          ),
-          REQUEST_TIMEOUT_MS
-        );
-      })
-    ]);
+    const timeout = new Promise((_resolve, reject) => {
+      timerId = setTimeout(
+        () => reject(new Error(`Request timed out after ${REQUEST_TIMEOUT_MS}ms`)),
+        REQUEST_TIMEOUT_MS
+      );
+    });
+    try {
+      return await Promise.race([operation(), timeout]);
+    } finally {
+      if (timerId) clearTimeout(timerId);
+    }
   }
-  async dispatchCommand(request) {
-    switch (request.command) {
-      case "pages":
-        return handlePages(this.pageById, this.page);
-      case "exec":
-        return handleExec(
-          this.resolveTargetPage(request.pageId),
-          request.code,
+  async runExec(args) {
+    return this.runCompactExec(args);
+  }
+  async runCompactExec(args) {
+    let targetPage;
+    try {
+      targetPage = this.resolveTargetPage(args.pageId);
+      const page = targetPage;
+      const data = await this.withRequestTimeout(async () => {
+        const before = this.latestCompactSnapshotByPage.get(page) ?? await snapshot(page);
+        const result = await handleExec(
+          page,
+          args.code,
           this.context,
           this.browser,
           this.execState,
           this.session,
-          request.visualize
-        );
-      case "readonly-exec":
-        return handleReadonlyExec(
-          this.resolveTargetPage(request.pageId),
-          request.code
+          args.visualize
         );
-      case "snapshot":
-        return handleSnapshot(
-          this.resolveTargetPage(request.pageId),
-          this.session,
-          this.logger,
-          request.pageId
-        );
-      default:
-        throw new Error(
-          `Unknown command: ${request.command}`
+        try {
+          const waitResult = await waitForPageStable(page);
+          if (!waitResult.ok) {
+            this.logger.warn("compact-exec-stability-wait-incomplete", {
+              session: this.session,
+              pageId: args.pageId,
+              diagnostics: waitResult.diagnostics
+            });
+          }
+          const after = await snapshot(page);
+          const snapshotDiff = diffSnapshots(before, after);
+          this.latestCompactSnapshotByPage.set(page, after);
+          return { ...result, snapshotDiff };
+        } catch (error) {
+          this.latestCompactSnapshotByPage.delete(page);
+          this.logger.warn("compact-exec-diff-failed", {
+            session: this.session,
+            pageId: args.pageId,
+            error: error instanceof Error ? error.message : String(error)
+          });
+          return result;
+        }
+      });
+      return { ok: true, data };
+    } catch (error) {
+      if (targetPage) this.latestCompactSnapshotByPage.delete(targetPage);
+      return this.createExecErrorResult(error);
+    }
+  }
+  async runReadonlyExec(args) {
+    try {
+      const data = await this.withRequestTimeout(
+        () => handleReadonlyExec(this.resolveTargetPage(args.pageId), args.code)
+      );
+      return { ok: true, data };
+    } catch (error) {
+      return this.createExecErrorResult(error);
+    }
+  }
+  createExecErrorResult(error) {
+    return {
+      ok: false,
+      message: error instanceof Error ? error.message : String(error),
+      output: error instanceof Error ? error.output : void 0
+    };
+  }
+  startWorkflow(args) {
+    if (this.workflowController) {
+      throw new Error("Workflow controller has already started.");
+    }
+    this.workflowController = new WorkflowController({
+      session: this.session,
+      headed: args.headed,
+      page: this.page,
+      context: this.context,
+      logger: this.logger,
+      onLog: (event) => {
+        void this.broadcast("workflowOutput", event);
+      },
+      onOutcome: (outcome) => {
+        if (outcome.state === "paused") {
+          void this.broadcast("workflowPaused", {
+            pausedAt: outcome.pausedAt,
+            url: outcome.url
+          });
+          return;
+        }
+        void this.broadcast(
+          "workflowFinished",
+          outcome.result === "completed" ? { result: "completed", completedAt: outcome.completedAt } : {
+            result: "failed",
+            message: outcome.message,
+            phase: outcome.phase
+          }
         );
+      }
+    });
+    this.workflowController.start({
+      integrationPath: args.workflow.integrationPath,
+      params: args.workflow.params,
+      visualize: args.workflow.visualize,
+      loadedWorkflow: args.loadedWorkflow
+    });
+  }
+  getWorkflowStatus() {
+    return this.workflowController?.getStatus() ?? { state: "idle" };
+  }
+  resumeWorkflow() {
+    if (!this.workflowController) {
+      throw new Error("Workflow is not paused.");
+    }
+    this.workflowController.resume();
+  }
+  async broadcast(name, message) {
+    const results = await Promise.allSettled(
+      Array.from(this.connectedClis, (cli) => {
+        const call = cli.call[name];
+        return call(message);
+      })
+    );
+    for (const result of results) {
+      if (result.status === "rejected") {
+        this.logger.warn("workflow-event-failed", {
+          event: name,
+          error: result.reason instanceof Error ? result.reason.message : String(result.reason)
+        });
+      }
     }
   }
 }
 async function main() {
   const config = JSON.parse(process.argv[2]);
-  const daemon = isConnectConfig(config) ? await BrowserDaemon.connectToEndpoint(config) : await BrowserDaemon.launchBrowser(config);
+  const headed = config.browser.kind === "launch" ? config.browser.headed : false;
+  let loadedWorkflow;
+  if (config.workflow) {
+    try {
+      loadedWorkflow = await loadDefaultWorkflow(
+        getAbsoluteIntegrationPath(config.workflow.integrationPath)
+      );
+    } catch (error) {
+      throw new UserFacingStartupError(
+        error instanceof Error ? error.message : String(error)
+      );
+    }
+  }
+  const daemon = config.browser.kind === "provider" ? await BrowserDaemon.connectToProvider({
+    session: config.session,
+    experiments: config.experiments,
+    browser: config.browser
+  }) : config.browser.kind === "connect" ? await BrowserDaemon.connectToEndpoint({
+    session: config.session,
+    experiments: config.experiments,
+    browser: config.browser
+  }) : await BrowserDaemon.launchBrowser({
+    session: config.session,
+    experiments: config.experiments,
+    browser: config.browser,
+    workflow: config.workflow
+  });
+  if (config.workflow) {
+    void waitForSessionState(config.session).then(
+      () => daemon.startWorkflow({
+        workflow: config.workflow,
+        headed,
+        loadedWorkflow
+      })
+    ).catch((error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      daemon.logger.error("workflow-failed", {
+        error: message
+      });
+      return daemon.broadcast("workflowFinished", {
+        result: "failed",
+        message,
+        phase: "setup"
+      }).finally(() => daemon.shutdown("workflow-start-failed", true));
+    });
+  }
   process.on("SIGTERM", () => {
     void daemon.shutdown("child-sigterm", true);
   });
@@ -295,4 +689,17 @@ async function main() {
     });
   });
 }
-await main();
+function reportStartupError(error) {
+  if (error instanceof UserFacingStartupError) {
+    process.send?.({
+      type: "startup-error",
+      message: error.message
+    });
+  }
+  process.exit(1);
+}
+try {
+  await main();
+} catch (error) {
+  reportStartupError(error);
+}