cdp-toolkit 1.10.0 → 2.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cdp-toolkit",
-  "version": "1.10.0",
+  "version": "2.0.0",
   "description": "A lightweight chrome-devtools-mcp alternative: an MCP server + CLI that drives the tabs you name, one target per call over one direct WebSocket, with a bounded timeout on every call, so a stuck page can't wedge your agent or force a /mcp restart. Chrome over CDP and Firefox over WebDriver BiDi, with tools a backend can't run absent from tools/list rather than throwing mid-task. 45 tools including built-in network mocking, a cookie group, tab leases with human-activity detection, real HTML5 drag-and-drop, raw mouse/scroll dispatch, downloads, permission grants, and tab-to-video screen recording so several agents can drive one browser without stealing each other's tabs or a human's. No Puppeteer, no all-target fan-out.",
   "type": "module",
   "license": "MIT",
@@ -79,16 +79,18 @@
     "mcp:smoke": "bun run ./test/mcp-smoke.ts",
     "firefox:smoke": "bun run ./test/firefox-smoke.ts",
     "firefox:attach:smoke": "bun run ./test/firefox-attach-smoke.ts",
+    "firefox:coord:smoke": "bun run ./test/firefox-session-coord-smoke.ts",
     "lease:smoke": "bun run ./test/lease-smoke.ts",
     "staleness:smoke": "bun run ./test/staleness-smoke.ts",
     "input:smoke": "bun run ./test/input-smoke.ts",
     "extension:smoke": "bun run ./test/extension-smoke.ts"
   },
   "devDependencies": {
+    "@modelcontextprotocol/client": "^2.0.0",
     "@types/node": "22.10.10",
     "typescript": "5.7.3"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "1.29.0"
+    "@modelcontextprotocol/server": "^2.0.0"
   }
 }

package/src/bidi/driver.ts

@@ -40,11 +40,16 @@ import {
   type ScreenshotOptions, type ScreenshotResult, type ScrollOptions, type SnapshotNode, type UidStability,
   type BandConsumer, type CaptureDelivery,
 } from "../driver.ts";
-import { assertLeaseOk } from "../leases.ts";
+import { assertLeaseOk, defaultLabel } from "../leases.ts";
 import { resolveLiveLabel } from "../origins.ts";
 import { isWorkerSelector, WORKER_SELECTOR_UNSUPPORTED_MESSAGE } from "../workers.ts";
 import { BEACON_FUNCTION_DECLARATION, BEACON_READ_EXPRESSION, BEACON_SOURCE, recordDispatch } from "../activity.ts";
 import { BidiConnection, BidiError, connectBidiSessionUrl } from "./client.ts";
+import {
+  acquireSessionSlot, releaseSessionSlot, SessionSlotBusyError,
+  type AcquireResult, type SessionSlotHandle,
+} from "./session-lease.ts";
+import { defaultMarionettePort, forceClearOrphanSession } from "./marionette.ts";
 import { takeStampedSnapshot } from "./snapshot.ts";
 import { selectRule, buildFulfillParams, effectiveAction } from "../tools/network_mock.ts";
 import type {
@@ -131,6 +136,20 @@ function mapBidiError(e: unknown): DriverError {
     if (code === "unsupported operation") return driverError("unsupported", e.message);
     if (!code && /timed out/i.test(e.message)) return driverError("timeout", e.message);
     if (!code && /(disposed|not open|connection closed)/i.test(e.message)) return driverError("disconnected", e.message);
+    // NS_ERROR_FILE_ACCESS_DENIED from browsingContext.navigate means the browser process was
+    // refused at the OS level while opening the file:// target — not a protocol failure. Two
+    // mechanisms produce it (#6): (1) the automation prefs that relax file:// handling were never
+    // applied — launch.ts now seeds them via user.js, so this should not surface for a
+    // toolkit-launched Firefox; (2) kernel-level confinement of the browser itself (Ubuntu's snap
+    // Firefox is the common one: a snap process cannot see host paths outside its mount
+    // namespace), which no preference can bypass. The error must name the way out rather than
+    // relay the raw nsresult.
+    if (/NS_ERROR_FILE_ACCESS_DENIED/i.test(e.message)) {
+      return driverError(
+        "page-error",
+        `${e.message} — the browser process was not allowed to open this file:// path. For a snap- or AppArmor-confined Firefox (Ubuntu default) that is kernel-level and no preference fixes it: serve the directory over a local http server (e.g. python3 -m http.server) and navigate to the http://127.0.0.1:<port>/ URL instead. If this Firefox was launched by cdp-toolkit and still fails, please report it at https://github.com/sblattj/cdp-toolkit/issues.`,
+      );
+    }
     return driverError("page-error", e.message);
   }
   return driverError("page-error", e instanceof Error ? e.message : String(e));
@@ -320,39 +339,174 @@ const BIDI_CAPABILITIES: ReadonlySet<Capability> = new Set<Capability>([
 // the only identity both modes share. getConnection retains a holder on every call (fresh or
 // reused); callers release() it.
 const connections = new Map<string, BidiConnection>();
+// This process's OWN Firefox BiDi session-slot ownership, keyed by endpoint (see session-lease.ts).
+// The slot LOCK is a file on disk — that is the cross-process source of truth — and this map is our
+// in-process record of which endpoints WE currently hold a live claim on, so dispose() releases
+// exactly the slots we took, and a re-dial after a dead socket reuses our own claim instead of
+// contending with our own pid. Populated in getConnection's create body; cleared on dispose() and
+// on a terminal dial failure that never established a session.
+const slotHandles = new Map<string, SessionSlotHandle>();
+// In-flight create dedup, keyed by endpoint. Two page() acquisitions can both cache-miss and call
+// getConnection concurrently; without this both would run the acquire+dial body, and the second
+// would either contend for the slot we just took (same pid → wait then SessionSlotBusyError against
+// ourselves) or open a second BiDi session. The first caller stores its create-promise here and
+// every other awaits it; cleared in a finally once the create settles.
+const connecting = new Map<string, Promise<BidiConnection>>();
+
+/**
+ * Release a slot claim we took in THIS getConnection call after a dial that never established a
+ * live session, so a later attempt re-acquires cleanly. When `owned` is true we did NOT acquire in
+ * this call (we reused a pre-existing claim — our previous socket died without dispose), so the
+ * handle predates this call and must be left intact: we still own Firefox's slot, only the socket
+ * failed to re-establish.
+ */
+async function releaseSlotOnDialFailure(endpoint: string, owned: boolean): Promise<void> {
+  if (owned) return;
+  const h = slotHandles.get(endpoint);
+  if (h) {
+    await releaseSessionSlot(h).catch(() => undefined);
+    slotHandles.delete(endpoint);
+  }
+}
+
+/**
+ * The actionable error for issue #4: Firefox is holding an orphaned BiDi session on this endpoint
+ * that Marionette recovery could NOT auto-clear (Firefox launched without --marionette, or the
+ * Marionette channel timed out / could not confirm). Distinguished from the live-busy case (issue
+ * #5, a LIVE lease holder) because here NO live process holds our lease — a session Firefox itself
+ * stranded is wedging the slot, and the fix is to enable Marionette or restart Firefox.
+ */
+function orphanNotClearedError(endpoint: string): DriverError {
+  return driverError("disconnected",
+    `Firefox holds an orphaned WebDriver BiDi session on ${endpoint} that could not be auto-cleared. ` +
+    `Marionette recovery needs Firefox launched with --marionette (default port 2828, or set ` +
+    `CDP_FIREFOX_MARIONETTE_PORT); otherwise restart Firefox to free the slot. (A client killed without ` +
+    `session.end can leave a session Firefox does not reap.)`);
+}
+
+// getConnection: the ONE Firefox-session choke point. Beyond memoizing a BidiConnection per endpoint
+// it now runs the cross-process SESSION-slot coordination (session-lease.ts) plus Marionette orphan
+// recovery (marionette.ts). Firefox serves exactly ONE WebDriver-BiDi session per browser, and it
+// never reaps an abandoned one (verified: still held 10s after an abrupt socket death). Two ills
+// follow that this closes: (#5) two LIVE processes attaching to one user-launched Firefox used to
+// hard-wedge on the raw "Maximum number of active sessions" with no way to tell a live holder from a
+// crash; the lease now names the LIVE holder and says wait-and-retry. (#4) a process killed without
+// session.end used to leave the slot occupied until Firefox restarted; when the lease shows the
+// prior holder's PID is DEAD, or Firefox itself still refuses on a re-dial, a blind Marionette
+// WebDriver:DeleteSession (which shares Firefox's one session slot) force-clears the orphan WITHOUT
+// killing the user's browser, and we retry. Marionette recovery is best-effort and degrades to a
+// clear, actionable error when Firefox has no --marionette port (the launch-mode norm).
 async function getConnection(endpoint: string, timeoutMs?: number): Promise<BidiConnection> {
   const existing = connections.get(endpoint);
   if (existing && existing.isOpen) { existing.retain(); return existing; }
   // A cached connection whose socket has died (ws.onclose fired) can only reject every
   // send() with "connection not open" — reusing it would strand the endpoint permanently.
-  // Evict it so the code below re-dials and creates a fresh BiDi session. No cleanup is
-  // needed: a dead connection already had `closed` set and rejectAll() cleared its per-call
-  // timers, and BidiConnection holds no heartbeat/keepalive resource to leak (see client.ts).
+  // Evict it so the code below re-dials and creates a fresh BiDi session. We deliberately do NOT
+  // release the session slot here: a dead socket is transient and WE still own Firefox's slot, so
+  // the create body below re-dials WITHOUT re-acquiring (its `owned` branch), avoiding a needless
+  // contention against our own pid. No other cleanup is needed: a dead connection already had
+  // `closed` set and rejectAll() cleared its per-call timers, and BidiConnection holds no
+  // heartbeat/keepalive resource to leak (see client.ts).
   if (existing) connections.delete(endpoint);
-  // Firefox 153's default unhandledPromptBehavior is "dismiss": a page-blocking alert()/confirm()
-  // gets auto-dismissed before handleDialog() ever runs, and Firefox's own userPromptOpened event
-  // reports handler:"dismiss" on it (verified: a bare session.new left "no such alert" on the very
-  // next handleUserPrompt call). Setting "ignore" here is what makes handleDialog() able to see
-  // and answer a live prompt at all.
-  const conn = await connectBidiSessionUrl(endpoint, {
-    ...(timeoutMs !== undefined ? { timeoutMs } : {}),
-    capabilities: { alwaysMatch: { unhandledPromptBehavior: { default: "ignore" } } },
-  }).catch((e: unknown) => {
-    // Firefox allows exactly ONE active BiDi session and never reaps an abandoned one (verified:
-    // still held 10s after an abrupt socket death). A client killed without session.end therefore
-    // leaves the slot occupied until Firefox restarts, and the raw wire message ("Maximum number of
-    // active sessions") tells the user nothing about how to get unstuck. Force-ending a session we
-    // do not own is not an option (there is no id to end), so this maps the error to instructions.
-    if (e instanceof BidiError && /maximum number of active sessions/i.test(e.message)) {
-      throw driverError("disconnected",
-        `Firefox already has an active WebDriver BiDi session on ${endpoint}; it allows only one. ` +
-        `Close the other client, or restart Firefox, then retry. (A previously killed client can leave a ` +
-        `session that Firefox does not reap.)`);
+
+  // Concurrency: dedup in-flight creates on this endpoint. An awaiter shares the create's outcome
+  // but needs its OWN holder — the create's connectBidiSessionUrl retains exactly once, for the
+  // caller that started it, so every other caller retains again for itself (callers all release()).
+  const inflight = connecting.get(endpoint);
+  if (inflight) {
+    const conn = await inflight;
+    conn.retain();
+    return conn;
+  }
+
+  const create = (async (): Promise<BidiConnection> => {
+    const mport = defaultMarionettePort();
+    // Do we ALREADY hold this endpoint's slot? (e.g. our previous socket died without dispose, so
+    // the eviction above kept the claim.) If so, re-dial WITHOUT re-acquiring: acquireSessionSlot
+    // against our own live pid would just poll to its wait ceiling and then throw
+    // SessionSlotBusyError against ourselves. We already own Firefox's slot; only the socket needs
+    // re-establishing (and if Firefox stranded our old session, the re-dial's retry path clears it).
+    const owned = slotHandles.has(endpoint);
+    if (!owned) {
+      // Cross-process coordination (session-lease.ts): take the endpoint's single BiDi session slot,
+      // classifying any current holder. A LIVE holder (another agent legitimately driving this
+      // Firefox) throws SessionSlotBusyError; a DEAD holder's orphan is stolen and surfaced as
+      // staleHolder so we can force-clear Firefox's stranded session before we call session.new.
+      let acq: AcquireResult;
+      try {
+        acq = await acquireSessionSlot(endpoint, { label: defaultLabel(), marionettePort: mport });
+      } catch (e) {
+        if (e instanceof SessionSlotBusyError) {
+          // Issue #5: a distinguishable LIVE holder, NOT a wedge. Map to an actionable disconnected
+          // error that names who holds the one session and why Firefox can serve only one per browser.
+          const who = e.holderLabel ? `'${e.holderLabel}' (pid ${e.holderPid})` : "another live process";
+          throw driverError("disconnected",
+            `Firefox's single WebDriver BiDi session on ${endpoint} is held by a LIVE process ${who}. ` +
+            `Firefox serves exactly ONE BiDi session per browser instance, so this server cannot take it ` +
+            `while that process is running. Wait for it to release the session and retry, or point this ` +
+            `server at a different endpoint/browser.`);
+        }
+        throw e;
+      }
+      slotHandles.set(endpoint, acq.handle);
+      if (acq.staleHolder) {
+        // Issue #4 auto-recovery: we stole a DEAD holder's lease, but Firefox itself may still hold
+        // that holder's orphaned BiDi session. Force-clear it over Marionette BEFORE dialing, or
+        // session.new below will still hit "Maximum number of active sessions". Best-effort: a false
+        // result means Firefox has no --marionette (launch mode, or the user never enabled it); we
+        // still dial, and the re-dial retry path below surfaces a clear error if the orphan persists.
+        await forceClearOrphanSession(acq.staleHolder.marionettePort ?? mport);
+      }
+    }
+
+    // Firefox 153's default unhandledPromptBehavior is "dismiss": a page-blocking alert()/confirm()
+    // gets auto-dismissed before handleDialog() ever runs, and Firefox's own userPromptOpened event
+    // reports handler:"dismiss" on it (verified: a bare session.new left "no such alert" on the very
+    // next handleUserPrompt call). Setting "ignore" here is what makes handleDialog() able to see
+    // and answer a live prompt at all.
+    const dialOpts = {
+      ...(timeoutMs !== undefined ? { timeoutMs } : {}),
+      capabilities: { alwaysMatch: { unhandledPromptBehavior: { default: "ignore" as const } } },
+    };
+    let conn: BidiConnection;
+    try {
+      conn = await connectBidiSessionUrl(endpoint, dialOpts); // already retained once
+    } catch (e) {
+      if (e instanceof BidiError && /maximum number of active sessions/i.test(e.message)) {
+        // Firefox still holds a BiDi session our lease didn't account for: a FOREIGN client we don't
+        // coordinate with, OUR OWN orphan from a dead socket (the `owned` re-dial), or a pre-lease
+        // orphan. Recovery: blind Marionette force-clear, then retry the dial EXACTLY once.
+        const cleared = await forceClearOrphanSession(mport);
+        if (!cleared) {
+          await releaseSlotOnDialFailure(endpoint, owned);
+          throw orphanNotClearedError(endpoint);
+        }
+        try {
+          conn = await connectBidiSessionUrl(endpoint, dialOpts); // already retained once
+        } catch (e2) {
+          await releaseSlotOnDialFailure(endpoint, owned);
+          if (e2 instanceof BidiError && /maximum number of active sessions/i.test(e2.message)) {
+            throw orphanNotClearedError(endpoint); // cleared, yet still refused: treat as unrecoverable
+          }
+          throw e2;
+        }
+      } else {
+        // Any other dial failure never established a session: release a slot we took THIS call and
+        // rethrow unchanged (the `owned` re-dial never acquired, so its claim is left intact).
+        await releaseSlotOnDialFailure(endpoint, owned);
+        throw e;
+      }
     }
-    throw e;
-  }); // already retained once
-  connections.set(endpoint, conn);
-  return conn;
+    connections.set(endpoint, conn);
+    return conn;
+  })();
+
+  connecting.set(endpoint, create);
+  try {
+    return await create;
+  } finally {
+    connecting.delete(endpoint);
+  }
 }
 
 /**
@@ -515,7 +669,21 @@ class BidiPageDriver implements PageDriver {
     const sendOpts = opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {};
     try {
       if (reload) {
-        const r = await this.conn.send("browsingContext.reload", { context: this.contextId, ignoreCache: opts.ignoreCache === true, wait }, sendOpts);
+        // Firefox rejects ANY `ignoreCache` argument to browsingContext.reload — even an
+        // explicit `false` — with `Argument "ignoreCache" is not supported yet` (observed on
+        // Firefox 153.0.3): the server-side validator refuses the key unconditionally, it does
+        // not read its value. So the key is OMITTED from the params unless a hard reload was
+        // actually requested, and a requested hard reload fails fast HERE with a
+        // backend-capability error instead of an opaque mid-flight protocol rejection — the
+        // same doctrine as click()'s modifiers on this backend. If a future Firefox starts
+        // honoring the argument, re-verify and lift the guard.
+        if (opts.ignoreCache === true) {
+          throw driverError(
+            "unsupported",
+            "navigate: ignoreCache:true (hard reload) is not supported on the Firefox WebDriver-BiDi backend — Firefox rejects the ignoreCache argument outright. A plain reload (reload:true, no ignoreCache) works.",
+          );
+        }
+        const r = await this.conn.send("browsingContext.reload", { context: this.contextId, wait }, sendOpts);
         return { url: r.url, contextId: this.contextId, reloaded: true, waitedFor: waitUntil };
       }
       if (history) {
@@ -1333,8 +1501,17 @@ class BidiBrowserDriver implements BrowserDriver {
     for (const key of [...bidiBeaconScripts.keys()]) {
       if (key.startsWith(`${this.endpoint}:`)) bidiBeaconScripts.delete(key);
     }
+    // Drop any in-flight create record defensively: dispose() ends this endpoint's life, so a
+    // pending create must not leave a stale promise that a later getConnection would await.
+    connecting.delete(this.endpoint);
     const conn = connections.get(this.endpoint);
-    if (!conn) return;
+    if (!conn) {
+      // No live connection, but we may still hold the session slot lock (e.g. a create acquired the
+      // slot then failed to establish a session on an `owned` re-dial, which leaves the claim). A
+      // clean dispose must free it cross-process so the next process is not falsely told we're LIVE.
+      await this.releaseSessionSlotIfHeld();
+      return;
+    }
     connections.delete(this.endpoint);
     // Cleanly END the BiDi session so Firefox frees its single session slot for the next
     // attach (a launch-mode Firefox is killed anyway; this matters for attach mode where the
@@ -1343,6 +1520,19 @@ class BidiBrowserDriver implements BrowserDriver {
     // Best-effort: harmless if the socket is already gone.
     await conn.send("session.end", {}).catch(() => undefined);
     conn.dispose();
+    // Release OUR cross-process session-slot lock so the next process (this or another) can take the
+    // slot cleanly. releaseSessionSlot only unlinks a record still stamped with our (pid, createdAt),
+    // so it never clobbers a claim a later process made after stealing ours. (session-lease.ts)
+    await this.releaseSessionSlotIfHeld();
+  }
+
+  /** Free this endpoint's session-slot lock if this process holds it, and forget the handle.
+   *  Best-effort and idempotent — safe to call whether or not a live connection existed. */
+  private async releaseSessionSlotIfHeld(): Promise<void> {
+    const h = slotHandles.get(this.endpoint);
+    if (!h) return;
+    await releaseSessionSlot(h).catch(() => undefined);
+    slotHandles.delete(this.endpoint);
   }
 }
 

package/src/bidi/launch.ts

@@ -22,7 +22,7 @@
  * node:os and node:path.
  */
 import { spawn, type ChildProcess } from "node:child_process";
-import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { createConnection, createServer } from "node:net";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -145,6 +145,55 @@ function waitForExit(proc: ChildProcess, deadlineMs: number): Promise<void> {
   });
 }
 
+/**
+ * Automation prefs seeded into every THROWAWAY profile this toolkit launches (#6).
+ *
+ * Firefox's Remote Agent has its own recommended automation-pref bundle, but it is gated on
+ * `remote.prefs.recommended`, which DEFAULTS TO FALSE — a bare `--remote-debugging-port`
+ * launch therefore runs with consumer prefs (verified in mozilla-central's
+ * remote/shared/RecommendedPreferences.sys.mjs; `defineLazyPreferenceGetter(..., "remote.prefs.recommended", false)`).
+ * The two file://-relevant consequences, and the reason each pref is here:
+ *
+ * - `remote.prefs.recommended: true` — flips on Firefox's own bundle at BiDi session start. This
+ *   is the same mechanism geckodriver/Marionette automation leans on.
+ * - `security.fileuri.strict_origin_policy: false` — a member of that bundle, also set explicitly
+ *   so the behavior does not depend on the bundle's exact contents per Firefox version. Without
+ *   it, file:// documents get unique opaque origins and sibling file:// subresources fail.
+ * - `dom.file.createInChild: true` — the bundle's file-API automation companion pref.
+ *
+ * Only the throwaway profile is seeded. An explicit `profilePath` belongs to its owner: writing
+ * prefs into someone's real profile (where `security.fileuri.strict_origin_policy: false` would
+ * persist as a security relaxation of their daily browsing) is not this toolkit's call to make.
+ */
+const LAUNCH_USER_JS_PREFS: Record<string, boolean> = {
+  "remote.prefs.recommended": true,
+  "security.fileuri.strict_origin_policy": false,
+  "dom.file.createInChild": true,
+};
+
+/**
+ * Seed `user.js` in a profile directory with the toolkit's automation prefs. Firefox reads
+ * user.js exactly once, at profile startup — which is why this runs BEFORE spawn, never after.
+ * Merges with an existing user.js (never rewrites a pref that file already sets, never drops a
+ * line it did not write) so the function is idempotent and safe on a reused profile dir.
+ * Exported for unit tests; returns the file's final content.
+ */
+export function writeLaunchUserJs(profilePath: string): string {
+  const target = join(profilePath, "user.js");
+  const existing = existsSync(target) ? readFileSync(target, "utf8") : "";
+  const present = new Set<string>();
+  for (const line of existing.split("\n")) {
+    const m = /^user_pref\("([^"]+)"\s*,/.exec(line.trim());
+    if (m) present.add(m[1]!);
+  }
+  const additions = Object.entries(LAUNCH_USER_JS_PREFS)
+    .filter(([k]) => !present.has(k))
+    .map(([k, v]) => `user_pref("${k}", ${JSON.stringify(v)});`);
+  const content = (existing.length > 0 ? existing.replace(/\n*$/, "\n") : "") + additions.join("\n") + (additions.length > 0 ? "\n" : "");
+  if (content !== existing) writeFileSync(target, content, "utf-8");
+  return content;
+}
+
 /**
  * Launch Firefox with the BiDi debug port enabled and resolve once the port
  * actually accepts a TCP connection. Never resolves on a fixed sleep.
@@ -156,6 +205,7 @@ export async function launchFirefox(opts: LaunchFirefoxOptions = {}): Promise<La
 
   const isThrowaway = opts.profilePath === undefined;
   const profilePath = opts.profilePath ?? mkdtempSync(join(tmpdir(), "cdp-toolkit-ff-profile-"));
+  if (isThrowaway) writeLaunchUserJs(profilePath);
 
   const args = ["--profile", profilePath, "--remote-debugging-port", String(port), "--no-remote"];
   if (opts.headless) args.push("--headless");