libretto 0.6.12 → 0.6.14

package/dist/cli/core/experiments.js

@@ -2,34 +2,7 @@ import {
   readLibrettoConfig,
   writeLibrettoConfig
 } from "./config.js";
-const EXPERIMENTS = {
-  "compact-snapshot-format": {
-    title: "Compact snapshot format",
-    oneSentenceDescription: "Use compact accessibility snapshots and exec page-change diffs without an AI sub-agent.",
-    docs: [
-      "Compact snapshot format changes how agents should use snapshot and exec after the experiment is enabled.",
-      "",
-      "Compared with the skill's documented behavior:",
-      "  - Run libretto snapshot --session <name> without --objective or --context.",
-      "  - Snapshot output is a screenshot path plus a compact accessibility tree; it does not use the PNG + HTML + AI analysis path.",
-      "  - Run libretto snapshot <ref> --session <name> to inspect a subtree from the latest full compact snapshot.",
-      "  - Run libretto exec normally; after successful mutations, Libretto prints page-change diffs from compact snapshots without AI analysis.",
-      "  - If a session was already open before enabling the experiment, close and reopen it before relying on this behavior.",
-      "",
-      "Full compact snapshot:",
-      "  libretto snapshot --session <name>",
-      "",
-      "Cached subtree snapshot:",
-      "  libretto snapshot <ref> --session <name>",
-      "",
-      "Run an unscoped snapshot before using refs. Subtree snapshots capture a fresh screenshot but reuse the latest cached tree.",
-      "",
-      "Notes:",
-      "  - Use ref forms printed in the tree, such as l16. Numeric-suffix aliases such as e16 also match l16."
-    ].join("\n"),
-    defaultValue: false
-  }
-};
+const EXPERIMENTS = {};
 function isExperimentName(name) {
   return Object.hasOwn(EXPERIMENTS, name);
 }

package/dist/cli/core/providers/index.js

@@ -2,7 +2,14 @@ import { readLibrettoConfig } from "../config.js";
 import { createBrowserbaseProvider } from "./browserbase.js";
 import { createKernelProvider } from "./kernel.js";
 import { createLibrettoCloudProvider } from "./libretto-cloud.js";
-const VALID_PROVIDERS = /* @__PURE__ */ new Set(["local", "kernel", "browserbase", "libretto-cloud"]);
+const VALID_PROVIDERS = /* @__PURE__ */ new Set([
+  "local",
+  "kernel",
+  "browserbase",
+  "libretto-cloud"
+]);
+const DEFAULT_PROVIDER_STARTUP_TIMEOUT_MS = 6e4;
+const LIBRETTO_CLOUD_STARTUP_TIMEOUT_MS = 10 * 6e4;
 function assertValidProviderName(value, source) {
   if (!VALID_PROVIDERS.has(value)) {
     throw new Error(
@@ -32,9 +39,7 @@ function getCloudProviderApi(name) {
     case "browserbase":
       return createBrowserbaseProvider();
     case "libretto-cloud":
-      console.warn(
-        "Note: The libretto-cloud provider is in alpha."
-      );
+      console.warn("Note: The libretto-cloud provider is in alpha.");
       return createLibrettoCloudProvider();
     default:
       throw new Error(
@@ -42,7 +47,11 @@ function getCloudProviderApi(name) {
       );
   }
 }
+function getProviderStartupTimeoutMs(providerName) {
+  return providerName === "libretto-cloud" ? LIBRETTO_CLOUD_STARTUP_TIMEOUT_MS : DEFAULT_PROVIDER_STARTUP_TIMEOUT_MS;
+}
 export {
   getCloudProviderApi,
+  getProviderStartupTimeoutMs,
   resolveProviderName
 };

package/dist/cli/core/providers/libretto-cloud.js

@@ -1,15 +1,19 @@
-import { HOSTED_API_URL } from "../auth-fetch.js";
+import { resolveHostedApiUrl } from "../auth-fetch.js";
+const DEFAULT_POLL_INTERVAL_MS = 2e3;
+const DEFAULT_BROWSER_SESSION_TIMEOUT_SECONDS = 3600;
+const QUEUE_WAIT_TIMEOUT_MS = 10 * 6e4;
 function createLibrettoCloudProvider() {
   const apiKey = process.env.LIBRETTO_API_KEY;
   if (!apiKey)
     throw new Error(
       "LIBRETTO_API_KEY is required for the Libretto Cloud provider."
     );
-  const endpoint = HOSTED_API_URL;
+  const endpoint = resolveHostedApiUrl();
   return {
     async createSession() {
-      const timeoutSeconds = Number(
-        process.env.LIBRETTO_TIMEOUT_SECONDS ?? 7200
+      const browserSessionTimeoutSeconds = readPositiveNumberEnv(
+        "LIBRETTO_TIMEOUT_SECONDS",
+        DEFAULT_BROWSER_SESSION_TIMEOUT_SECONDS
       );
       const resp = await fetch(`${endpoint}/v1/sessions/create`, {
         method: "POST",
@@ -18,42 +22,190 @@ function createLibrettoCloudProvider() {
           "Content-Type": "application/json"
         },
         body: JSON.stringify({
-          json: { timeout_seconds: timeoutSeconds }
+          json: { timeout_seconds: browserSessionTimeoutSeconds }
         })
       });
       if (!resp.ok) {
         const body = await resp.text();
-        throw new Error(
-          `Libretto Cloud API error (${resp.status}): ${body}`
-        );
+        throw new Error(`Libretto Cloud API error (${resp.status}): ${body}`);
       }
       const { json } = await resp.json();
+      const startupCleanup = createStartupSessionCleanup(
+        endpoint,
+        apiKey,
+        json.session_id
+      );
+      let readySession;
+      try {
+        readySession = await waitForCloudSessionReady({
+          endpoint,
+          apiKey,
+          session: json,
+          timeoutMs: QUEUE_WAIT_TIMEOUT_MS,
+          isCancelled: startupCleanup.isCancelled
+        });
+      } catch (error) {
+        if (startupCleanup.isCancelled()) {
+          await startupCleanup.waitForClose();
+        } else {
+          await closeCloudSession(endpoint, apiKey, json.session_id).catch(
+            () => {
+            }
+          );
+        }
+        throw error;
+      } finally {
+        startupCleanup.dispose();
+      }
       return {
-        sessionId: json.session_id,
-        cdpEndpoint: json.cdp_url,
-        liveViewUrl: json.live_view_url ?? void 0
+        sessionId: readySession.session_id,
+        cdpEndpoint: readySession.cdp_url,
+        liveViewUrl: readySession.live_view_url ?? void 0
       };
     },
     async closeSession(sessionId) {
-      const resp = await fetch(`${endpoint}/v1/sessions/close`, {
-        method: "POST",
-        headers: {
-          "x-api-key": apiKey,
-          "Content-Type": "application/json"
-        },
-        body: JSON.stringify({ json: { session_id: sessionId } })
-      });
-      if (!resp.ok) {
-        const body = await resp.text();
-        throw new Error(
-          `Libretto Cloud API error closing session ${sessionId} (${resp.status}): ${body}`
-        );
-      }
-      const { json } = await resp.json();
+      const json = await closeCloudSession(endpoint, apiKey, sessionId);
       return { replayUrl: json.replay_url ?? void 0 };
     }
   };
 }
+async function waitForCloudSessionReady(args) {
+  let session = args.session;
+  if (args.isCancelled?.()) {
+    throw new Error(
+      `Libretto Cloud session ${session.session_id} was cancelled before browser capacity was available.`
+    );
+  }
+  if (session.cdp_url) {
+    return { ...session, cdp_url: session.cdp_url };
+  }
+  sendStartupStatus(
+    `Libretto Cloud browser session queued (session: ${session.session_id}). Waiting for browser capacity...`
+  );
+  const pollIntervalMs = readPositiveNumberEnv(
+    "LIBRETTO_CLOUD_SESSION_POLL_INTERVAL_MS",
+    DEFAULT_POLL_INTERVAL_MS
+  );
+  const deadline = Date.now() + args.timeoutMs;
+  while (Date.now() < deadline) {
+    if (args.isCancelled?.()) {
+      throw new Error(
+        `Libretto Cloud session ${session.session_id} was cancelled before browser capacity was available.`
+      );
+    }
+    await sleep(pollIntervalMs);
+    if (args.isCancelled?.()) {
+      throw new Error(
+        `Libretto Cloud session ${session.session_id} was cancelled before browser capacity was available.`
+      );
+    }
+    session = await getCloudSession(
+      args.endpoint,
+      args.apiKey,
+      session.session_id
+    );
+    if (session.cdp_url) {
+      sendStartupStatus(
+        `Libretto Cloud browser capacity available (session: ${session.session_id}). Connecting...`
+      );
+      return { ...session, cdp_url: session.cdp_url };
+    }
+    if (!["queued", "starting"].includes(session.status)) {
+      throw new Error(
+        `Libretto Cloud session ${session.session_id} entered status "${session.status}" before a CDP URL was available.`
+      );
+    }
+  }
+  throw new Error(
+    `Timed out waiting for Libretto Cloud browser capacity after ${Math.ceil(args.timeoutMs / 1e3)}s (session: ${session.session_id}).`
+  );
+}
+async function getCloudSession(endpoint, apiKey, sessionId) {
+  const resp = await fetch(`${endpoint}/v1/sessions/get`, {
+    method: "POST",
+    headers: {
+      "x-api-key": apiKey,
+      "Content-Type": "application/json"
+    },
+    body: JSON.stringify({ json: { session_id: sessionId } })
+  });
+  if (!resp.ok) {
+    const body = await resp.text();
+    throw new Error(
+      `Libretto Cloud API error reading session ${sessionId} (${resp.status}): ${body}`
+    );
+  }
+  const { json } = await resp.json();
+  return json;
+}
+async function closeCloudSession(endpoint, apiKey, sessionId) {
+  const resp = await fetch(`${endpoint}/v1/sessions/close`, {
+    method: "POST",
+    headers: {
+      "x-api-key": apiKey,
+      "Content-Type": "application/json"
+    },
+    body: JSON.stringify({ json: { session_id: sessionId } })
+  });
+  if (!resp.ok) {
+    const body = await resp.text();
+    throw new Error(
+      `Libretto Cloud API error closing session ${sessionId} (${resp.status}): ${body}`
+    );
+  }
+  const { json } = await resp.json();
+  return json;
+}
+function createStartupSessionCleanup(endpoint, apiKey, sessionId) {
+  let cancelled = false;
+  let closePromise = null;
+  const requestClose = (reason) => {
+    if (cancelled) return;
+    cancelled = true;
+    sendStartupStatus(
+      `Libretto Cloud browser session cancelled (${reason}). Cleaning up queued session...`
+    );
+    closePromise = closeCloudSession(endpoint, apiKey, sessionId).then(
+      () => {
+      },
+      () => {
+      }
+    );
+  };
+  const onDisconnect = () => requestClose("parent command disconnected");
+  const onSigint = () => requestClose("received SIGINT");
+  const onSigterm = () => requestClose("received SIGTERM");
+  if (typeof process.send === "function") {
+    process.once("disconnect", onDisconnect);
+  }
+  process.once("SIGINT", onSigint);
+  process.once("SIGTERM", onSigterm);
+  return {
+    isCancelled: () => cancelled,
+    waitForClose: async () => {
+      await closePromise;
+    },
+    dispose: () => {
+      process.off("disconnect", onDisconnect);
+      process.off("SIGINT", onSigint);
+      process.off("SIGTERM", onSigterm);
+    }
+  };
+}
+function readPositiveNumberEnv(name, fallback) {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+function sendStartupStatus(message) {
+  if (typeof process.send === "function") {
+    process.send({ type: "startup-status", message });
+  }
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 export {
   createLibrettoCloudProvider
 };

package/dist/cli/index.js

@@ -1,8 +1,6 @@
 #!/usr/bin/env node
 import { runLibrettoCLI } from "./cli.js";
-import { runClose } from "./commands/browser.js";
 void runLibrettoCLI();
 export {
-  runClose,
   runLibrettoCLI
 };

package/dist/cli/router.js

@@ -1,4 +1,3 @@
-import { aiCommands } from "./commands/ai.js";
 import { authCommands } from "./commands/auth.js";
 import { billingCommands } from "./commands/billing.js";
 import { browserCommands } from "./commands/browser.js";
@@ -9,15 +8,19 @@ import { setupCommand } from "./commands/setup.js";
 import { statusCommand } from "./commands/status.js";
 import { snapshotCommand } from "./commands/snapshot.js";
 import { librettoCommand } from "../shared/package-manager.js";
-import { SimpleCLI } from "./framework/simple-cli.js";
+import { SimpleCLI } from "affordance";
 const cliRoutes = {
   ...browserCommands,
-  deploy: deployCommand,
+  cloud: SimpleCLI.group({
+    description: "Libretto Cloud commands",
+    routes: {
+      deploy: deployCommand,
+      auth: authCommands,
+      billing: billingCommands
+    }
+  }),
   experiments: experimentsCommand,
   ...executionCommands,
-  ai: aiCommands,
-  auth: authCommands,
-  billing: billingCommands,
   setup: setupCommand,
   status: statusCommand,
   snapshot: snapshotCommand

package/dist/shared/instrumentation/instrument.js

@@ -82,12 +82,12 @@ function wrapLocatorActions(locator, page, opts) {
       try {
         const result = await orig(...args);
         if (opts.visualize) {
-          enqueue(page, () => visualizeAfterAction(page));
+          void enqueue(page, () => visualizeAfterAction(page));
         }
         return result;
       } catch (err) {
         if (opts.visualize) {
-          enqueue(page, () => visualizeAfterAction(page));
+          void enqueue(page, () => visualizeAfterAction(page));
         }
         if (POINTER_ACTIONS.has(method) && isTimeoutError(err)) {
           await enrichTimeoutError(err, locator, page);
@@ -226,12 +226,12 @@ async function installInstrumentation(page, options) {
       try {
         const result = await orig(...args);
         if (visualize) {
-          enqueue(page, () => visualizeAfterAction(page));
+          void enqueue(page, () => visualizeAfterAction(page));
         }
         return result;
       } catch (err) {
         if (visualize) {
-          enqueue(page, () => visualizeAfterAction(page));
+          void enqueue(page, () => visualizeAfterAction(page));
         }
         if (POINTER_ACTIONS.has(method) && isTimeoutError(err) && typeof args[0] === "string") {
           await enrichTimeoutError(err, page.locator(args[0]), page);

package/dist/shared/ipc/socket-transport.d.ts

@@ -5,5 +5,6 @@ declare function connectToIpcSocket(socketPath: string): Promise<IpcTransport<Ip
 declare function createIpcSocketServer(onConnection: (transport: IpcTransport<IpcProtocolMessage>) => void): Server;
 declare function listenForIpcConnections(socketPath: string, onConnection: (transport: IpcTransport<IpcProtocolMessage>) => void): Promise<Server>;
 declare function listenOnIpcSocket(server: Server, socketPath: string): Promise<void>;
+declare function isWindowsNamedPipePath(socketPath: string): boolean;
 
-export { connectToIpcSocket, createIpcSocketServer, listenForIpcConnections, listenOnIpcSocket };
+export { connectToIpcSocket, createIpcSocketServer, isWindowsNamedPipePath, listenForIpcConnections, listenOnIpcSocket };

package/dist/shared/ipc/socket-transport.js

@@ -1,10 +1,9 @@
-import { rm } from "node:fs/promises";
+import { mkdir, rm } from "node:fs/promises";
 import {
   createServer,
   createConnection
 } from "node:net";
 import { dirname } from "node:path";
-import { mkdir } from "node:fs/promises";
 function createJsonSocketTransport(socket) {
   socket.setEncoding("utf8");
   return {
@@ -82,12 +81,11 @@ async function listenForIpcConnections(socketPath, onConnection) {
   return server;
 }
 async function listenOnIpcSocket(server, socketPath) {
-  await mkdir(dirname(socketPath), { recursive: true });
-  await rm(socketPath, { force: true });
+  await prepareIpcSocketPath(socketPath);
   const originalClose = server.close.bind(server);
   server.close = ((callback) => {
     return originalClose((error) => {
-      void rm(socketPath, { force: true }).finally(() => callback?.(error));
+      void removeStaleSocketFile(socketPath).finally(() => callback?.(error));
     });
   });
   await new Promise((resolve, reject) => {
@@ -104,6 +102,18 @@ async function listenOnIpcSocket(server, socketPath) {
     server.listen(socketPath);
   });
 }
+function isWindowsNamedPipePath(socketPath) {
+  return socketPath.startsWith("\\\\.\\pipe\\");
+}
+async function prepareIpcSocketPath(socketPath) {
+  if (isWindowsNamedPipePath(socketPath)) return;
+  await mkdir(dirname(socketPath), { recursive: true });
+  await removeStaleSocketFile(socketPath);
+}
+async function removeStaleSocketFile(socketPath) {
+  if (isWindowsNamedPipePath(socketPath)) return;
+  await rm(socketPath, { force: true });
+}
 async function connectSocket(socketPath) {
   const socket = createConnection(socketPath);
   return new Promise((resolve, reject) => {
@@ -138,6 +148,7 @@ function isRecord(value) {
 export {
   connectToIpcSocket,
   createIpcSocketServer,
+  isWindowsNamedPipePath,
   listenForIpcConnections,
   listenOnIpcSocket
 };

package/dist/shared/ipc/socket-transport.spec.js

@@ -6,6 +6,7 @@ import { expect, test as base } from "vitest";
 import { createIpcPeer } from "./ipc.js";
 import {
   connectToIpcSocket,
+  isWindowsNamedPipePath,
   listenForIpcConnections
 } from "./socket-transport.js";
 const test = base.extend({
@@ -47,6 +48,10 @@ test("sends concurrent calls over one socket", async ({ socketPath }) => {
   });
   await expect(stat(socketPath)).rejects.toThrow();
 });
+test("recognizes Windows named pipe paths", () => {
+  expect(isWindowsNamedPipePath("\\\\.\\pipe\\libretto-abc123")).toBe(true);
+  expect(isWindowsNamedPipePath("/tmp/libretto-501-abc123.sock")).toBe(false);
+});
 test("rejects pending calls when the socket closes", async ({ socketPath }) => {
   const serverPeers = [];
   const server = await listenForIpcConnections(socketPath, (transport) => {

package/docs/releasing.md

@@ -24,7 +24,7 @@ This repo does not publish from local machines and does not push directly to `ma
 
 GitHub Actions needs these repository secrets:
 
-- `OPENAI_API_KEY`: used by the existing test suite during the release workflow.
+- `OPENAI_API_KEY`: used by the existing test suite during the release workflow and by the eval workflow's default `openai/gpt-5.5` model.
 
 The release workflow uses a GitHub Actions environment named `release`. Create that environment in the repository settings (no required reviewers — access is controlled by branch protection on `main` instead).
 
@@ -69,7 +69,7 @@ The root `scripts/prepare-release.sh` script does the following:
 6. Commits the version bump.
 7. Pushes the branch and opens a PR to `main` with the `release` label.
 
-Release PRs also run the eval workflow. That workflow compares the current eval score against the latest successful `main` baseline and fails if the score drifts by more than 5 percentage points in either direction.
+Release PRs also run the eval workflow. That workflow records score, duration, token, cost, and tool-call metrics for review. Scores are informational: low scores do not fail the workflow, but setup/runtime failures and zero completed records do.
 
 ## Merge behavior
 
@@ -89,11 +89,13 @@ This makes the workflow safe to re-run after partial failures. For example, if n
 
 `.github/workflows/evals.yml` now runs automatically for release PRs and for qualifying pushes to `main`.
 
-- On `main`, it records the current eval summary as the baseline artifact for future release PRs.
-- On release PRs, it runs evals again and compares the overall score against the latest successful `main` baseline.
-- If the score moves outside a `+/-5%` window, the eval job fails and flags the release PR.
+- It runs `pnpm evals --no-auth --output <runner-temp>/eval-run` so CI only runs cases that do not require local auth profiles.
+- It validates and renders the CI report with `pnpm evals summary <runner-temp>/eval-run`.
+- It writes a GitHub step summary and a sticky PR comment with aggregate score, duration, token, cost, and tool-call metrics.
+- It uploads summary files and per-case result records from the run output directory. Raw transcripts and local profile files are not uploaded.
+- It fails when the eval runner crashes, required setup is missing, result records are malformed, or zero completed records are produced.
 
-If no successful baseline artifact exists yet, the release PR eval job reports that and skips the comparison for that run.
+There is no baseline comparison gate yet. Add one only after the eval records and metrics are stable enough to compare reliably.
 
 ## Changelog behavior
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libretto",
-  "version": "0.6.12",
+  "version": "0.6.14",
   "description": "AI-powered browser automation library and CLI built on Playwright",
   "license": "MIT",
   "homepage": "https://libretto.sh",
@@ -9,7 +9,6 @@
     "url": "https://github.com/saffron-health/libretto"
   },
   "type": "module",
-  "packageManager": "pnpm@10.33.0",
   "publishConfig": {
     "access": "public"
   },
@@ -37,6 +36,7 @@
     "sync-skills": "pnpm run sync:mirrors",
     "check:skills": "pnpm run check:mirrors",
     "build": "tsup --config tsup.config.ts",
+    "lint": "lintcn lint --tsconfig tsconfig.json",
     "type-check": "tsc --noEmit",
     "test": "turbo run test:vitest --filter=libretto --log-order=grouped",
     "test:vitest": "vitest run",
@@ -85,6 +85,7 @@
     "vitest": "^4.1.5"
   },
   "dependencies": {
+    "affordance": "^0.1.0",
     "ai": "^6.0.116",
     "esbuild": "^0.27.0",
     "playwright": "^1.58.2",