imprint-mcp 0.2.1 → 0.3.0

package/src/imprint/redact.ts

@@ -21,6 +21,22 @@ import type { CapturedRequest, Session } from './types.ts';
 const USER_INTERACTION_TYPES = new Set(['click', 'input', 'change', 'submit']);
 const MULTI_VALUE_HEADERS = new Set(['cookie', 'set-cookie']);
 
+/**
+ * Detect a structured RPC envelope (XSSI-guarded or length-prefixed) whose body
+ * is NOT top-level JSON but carries doubly-encoded JSON as string payloads —
+ * e.g. Google `batchexecute` (`)]}'` guard + `<len>\n[...]` frames). Running the
+ * flat-text freeform scanner over such a body injects `[REDACTED]` into bare
+ * numeric IDs/coordinates inside the inner JSON and makes it unparseable, so the
+ * freeform fallback must skip these. The structure-aware key-based redaction
+ * still applies to any clean-JSON bodies; this only gates the flat-text scan.
+ */
+export function looksLikeRpcEnvelope(body: string): boolean {
+  const head = body.slice(0, 64).trimStart();
+  if (head.startsWith(")]}'")) return true; // anti-XSSI guard: )]}' and )]}',
+  if (/^\d{1,9}\r?\n\[/.test(head)) return true; // length-prefixed frame: 219006\n[
+  return false;
+}
+
 /**
  * Detect sensitive headers whose values are page-minted constants — baked
  * into the site's JavaScript, not per-user secrets. The recording starts
@@ -179,7 +195,12 @@ export function redactJsonBody(
             const visited = visit(inner, [...pathSoFar, k]);
             out[k] = JSON.stringify(visited);
           } catch {
-            const r = freeform ? redactFreeformText(v) : { redacted: v, redactionsCount: 0 };
+            // Nested string that isn't parseable JSON: scan it as free text,
+            // unless it's a structured RPC envelope (flat-scanning corrupts it).
+            const r =
+              freeform && !looksLikeRpcEnvelope(v)
+                ? redactFreeformText(v)
+                : { redacted: v, redactionsCount: 0 };
             freeformCount += r.redactionsCount;
             out[k] = r.redacted;
           }
@@ -228,6 +249,9 @@ export function redactBody(
   } catch {
     const formR = redactFormBody(body, formPlaceholders, markerContext);
     if (formR.redactionsCount > 0 || formR.placeholdersInjected > 0 || !freeform) return formR;
+    // A structured RPC envelope (XSSI/length-prefixed) is not flat text —
+    // flat-scanning it would corrupt the doubly-encoded JSON payloads it carries.
+    if (looksLikeRpcEnvelope(body)) return formR;
     const freeformR = redactFreeformText(body);
     return {
       redacted: freeformR.redacted,
@@ -437,7 +461,11 @@ export function redactSession(
           response.mimeType,
           undefined,
           undefined,
-          useFreeform,
+          // Responses are key-based only: never value-pattern (freeform) scan a
+          // server body. Keeps redaction focused on real secrets (post-login
+          // cookies + user-entered PII) and avoids corrupting structured RPC
+          // envelopes whose payloads are doubly-encoded JSON.
+          false,
           markerContext,
         );
         respBody = respBodyR.redacted;

package/src/imprint/replay-capture.ts

@@ -15,6 +15,7 @@ import { join as pathJoin } from 'node:path';
 import type { Browser, BrowserContext, Locator, Page } from 'playwright';
 import { createLog } from './log.ts';
 import type { CapturedReplayRequest } from './session-diff.ts';
+import { getStealthChromium, getStealthExecutablePath } from './stealth-chromium.ts';
 import type { CapturedEvent, Session } from './types.ts';
 
 const log = createLog('replay-capture');
@@ -62,25 +63,17 @@ export async function replayRawSession(opts: RawReplayOptions): Promise<ReplayCa
 
   let chromium: typeof import('playwright').chromium;
   try {
-    const pwExtra = await import('playwright-extra');
-    const stealthMod = await import('puppeteer-extra-plugin-stealth');
-    const stealthFactory =
-      (stealthMod as { default?: () => unknown }).default ??
-      (stealthMod as unknown as () => unknown);
-    pwExtra.chromium.use(stealthFactory() as never);
-    chromium = pwExtra.chromium as unknown as typeof import('playwright').chromium;
-  } catch {
-    try {
-      const pw = await import('playwright');
-      chromium = pw.chromium;
-    } catch (innerErr) {
-      return { ok: false, requests: [], error: `Playwright not available: ${errMsg(innerErr)}` };
-    }
+    chromium = await getStealthChromium();
+  } catch (innerErr) {
+    return { ok: false, requests: [], error: `Playwright not available: ${errMsg(innerErr)}` };
   }
 
   try {
     replayLog(`launching browser (headed=${!!opts.headed})`);
-    browser = await chromium.launch({ headless: !opts.headed });
+    browser = await chromium.launch({
+      headless: !opts.headed,
+      executablePath: getStealthExecutablePath(),
+    });
   } catch (err) {
     replayLog(`browser launch failed: ${errMsg(err)}`);
     return { ok: false, requests: [], error: `Could not launch Chromium: ${errMsg(err)}` };
@@ -215,11 +208,19 @@ export async function replayRawSession(opts: RawReplayOptions): Promise<ReplayCa
       opts.onProgress?.(i + 1, replayableEvents.length, captured.length);
     }
 
-    // Allow final network requests to settle
+    // Allow final network requests to settle, but never block forever: on a
+    // large recording a single hung response-body read can stall allSettled
+    // indefinitely (there is no outer timeout on the replay stage). Cap the
+    // wait and proceed with whatever bodies are ready — replay-diff is
+    // best-effort, so partial captures are acceptable.
+    const SETTLE_TIMEOUT_MS = 15_000;
     replayLog('waiting for networkidle...');
-    await page.waitForLoadState('networkidle').catch(() => {});
+    await page.waitForLoadState('networkidle', { timeout: SETTLE_TIMEOUT_MS }).catch(() => {});
     await page.waitForTimeout(1000);
-    await Promise.allSettled(pendingReads);
+    await Promise.race([
+      Promise.allSettled(pendingReads),
+      new Promise<void>((resolve) => setTimeout(resolve, SETTLE_TIMEOUT_MS)),
+    ]);
     captured.sort((a, b) => a.seq - b.seq);
 
     replayLog(`replay complete: captured ${captured.length} requests total`);

package/src/imprint/runtime.ts

@@ -113,14 +113,23 @@ export async function executeWorkflow<T = unknown>(opts: ExecuteOptions): Promis
     (await loadCredentialStore(opts.workflow.site)) ??
     emptyStore(opts.workflow.site);
 
-  // Validate required parameters are present.
+  // Validate required parameters are present and merge declared defaults
+  // into the working params map. Without the merge, `parameter.default` would
+  // be a presence-sentinel only — the substitution layer at
+  // `resolvePlaceholder` would still throw STATE_MISSING because it reads
+  // from this map directly. The schema declares `default` as a real value
+  // (string | number | boolean), so honor it.
+  const params: Record<string, string | number | boolean> = { ...opts.params };
   for (const p of opts.workflow.parameters) {
-    if (!(p.name in opts.params) && p.default === undefined) {
-      return {
-        ok: false,
-        error: 'UNKNOWN',
-        message: `Missing required parameter: ${p.name} (${p.description})`,
-      };
+    if (!(p.name in params)) {
+      if (p.default === undefined) {
+        return {
+          ok: false,
+          error: 'UNKNOWN',
+          message: `Missing required parameter: ${p.name} (${p.description})`,
+        };
+      }
+      params[p.name] = p.default;
     }
   }
 
@@ -163,7 +172,7 @@ export async function executeWorkflow<T = unknown>(opts: ExecuteOptions): Promis
     if (!req) continue;
 
     const subbedResult = substituteRequest(req, {
-      params: opts.params,
+      params,
       credentials: liveCredentials,
       responseSlots,
       state,
@@ -180,7 +189,7 @@ export async function executeWorkflow<T = unknown>(opts: ExecuteOptions): Promis
           subbed.method,
           subbed.url,
           responseSlots.map((s) => s.raw),
-          opts.params,
+          params,
         );
         if (typeof transformResult === 'string') {
           subbed.url = transformResult;
@@ -312,7 +321,7 @@ export async function executeWorkflow<T = unknown>(opts: ExecuteOptions): Promis
         };
       }
       finalData = mod.extract(finalData, {
-        params: opts.params,
+        params,
         responses: responseSlots.map((s) => s.raw),
       });
     } catch (err) {

package/src/imprint/session-diff.ts

@@ -318,6 +318,17 @@ function suggestStateName(location: string): string {
     .toLowerCase();
 }
 
+/** Whether a value looks like an opaque token/id (vs human text, a city name, a
+ *  date). Gates provenance-tagging of stable values so an incidental constant
+ *  (a UI label, the echoed query) isn't treated as a server-provided token.
+ *  Shared with the build-plan token detector. */
+export function looksLikeToken(v: string): boolean {
+  if (v.length < 12) return false;
+  if (/\s/.test(v)) return false; // multi-word / free text
+  if (/^\d{4}-\d{2}-\d{2}$/.test(v)) return false; // dates
+  return /[:|_-]/.test(v) || /\d/.test(v);
+}
+
 // ─── Main diff ──────────────────────────────────────────────────────────────
 
 export function diffTriagedSessions(
@@ -327,6 +338,12 @@ export function diffTriagedSessions(
   const pairs = alignRequests(original.requests, replay.requests);
   const pairedOrigSeqs = new Set(pairs.map((p) => p.originalSeq));
   const pairedReplaySeqs = new Set(pairs.map((p) => p.replaySeq));
+  // `searchPriorResponses` over the replay returns a producer in REPLAY-seq
+  // space, but `originalSeq` and every downstream consumer (capture hints,
+  // build-plan token detection, the planner) work in ORIGINAL-seq space — so a
+  // replay producer must be translated back via the alignment pairs.
+  const replayToOriginal = new Map(pairs.map((p) => [p.replaySeq, p.originalSeq]));
+  const toOriginalSeq = (replaySeq: number): number => replayToOriginal.get(replaySeq) ?? replaySeq;
 
   const classifications: ClassifiedValue[] = [];
 
@@ -347,17 +364,28 @@ export function diffTriagedSessions(
       if (v2Value === undefined) continue; // field only in run 1
 
       if (v1.value === v2Value) {
+        // Stable across runs. Normally a constant — but an OPAQUE stable value
+        // that also appears in a PRIOR response is a server-PROVIDED token (e.g.
+        // a per-entity id minted by a sibling search tool). The same-flow replay
+        // can't expose it by variance (same entity → same token), so recover its
+        // provenance from the original responses (already original-seq space).
+        // A cross-tool consumer then sources it as a param instead of hardcoding.
+        const provider = looksLikeToken(v1.value)
+          ? searchPriorResponses(v1.value, original.requests, pair.originalSeq)
+          : null;
         classifications.push({
           classification: 'constant',
           location: v1.location,
           originalSeq: pair.originalSeq,
           value1: v1.value,
           value2: v2Value,
+          ...(provider ? { producerSeq: provider.seq, producerPath: provider.path } : {}),
         });
         continue;
       }
 
-      // Value differs — check if it came from a prior response in run 2
+      // Value differs — check if it came from a prior response in run 2,
+      // translating the replay producer back to original-seq space.
       const producer = searchPriorResponses(v2Value, replay.requests, pair.replaySeq);
 
       if (producer) {
@@ -368,7 +396,7 @@ export function diffTriagedSessions(
           originalSeq: pair.originalSeq,
           value1: v1.value,
           value2: v2Value,
-          producerSeq: producer.seq,
+          producerSeq: toOriginalSeq(producer.seq),
           producerPath: producer.path,
           suggestedStateName: name || undefined,
         });
@@ -407,3 +435,52 @@ export function triageByAlignment(
   const aligned = alignRequests(run1TriagedRequests, run2AllRequests);
   return aligned.filter((pair) => pair.confidence >= 0.5).map((pair) => pair.replaySeq);
 }
+
+/**
+ * Severity order — a value seen varying in ANY pass outranks one seen constant.
+ * server_derived (traceable to a response) wins over browser_minted.
+ */
+const CLASSIFICATION_RANK: Record<ValueClassification, number> = {
+  constant: 0,
+  browser_minted: 1,
+  server_derived: 2,
+};
+
+/**
+ * Merge `ClassifiedValue`s from several diff passes that all share the SAME
+ * `original` recording (so `originalSeq` is a stable join key across passes).
+ *
+ * Each pass diffs the original recording against one other run — the automated
+ * browser replay AND every other real recording of the site. Anti-bot edges
+ * (Akamai, DataDome, …) often block the automated replay at the page level, so
+ * the replay reproduces only a fraction of the recording's requests and their
+ * functional values (GraphQL safelisting signatures, persisted-query hashes,
+ * app keys) never get classified. Real recordings come from a trusted browser
+ * and DO carry those requests, so diffing recordings against each other
+ * recovers the missing signal.
+ *
+ * Merge rule per (originalSeq, location):
+ *   - a value that VARIES in any pass is ephemeral — the strongest non-constant
+ *     classification wins (server_derived > browser_minted), preserving its
+ *     producer provenance;
+ *   - a value constant in every pass that observed it is `constant`.
+ * A value the replay never observed (because it was blocked) but that is
+ * identical across time-separated recordings is therefore kept as `constant`,
+ * not silently dropped.
+ */
+export function mergeClassifications(passes: ClassifiedValue[][]): ClassifiedValue[] {
+  const byKey = new Map<string, ClassifiedValue>();
+  for (const pass of passes) {
+    for (const cv of pass) {
+      const key = `${cv.originalSeq}${cv.location}`;
+      const prev = byKey.get(key);
+      if (
+        !prev ||
+        CLASSIFICATION_RANK[cv.classification] > CLASSIFICATION_RANK[prev.classification]
+      ) {
+        byKey.set(key, cv);
+      }
+    }
+  }
+  return [...byKey.values()];
+}

package/src/imprint/session-merge.ts

@@ -7,7 +7,7 @@
  * pipeline consumes unchanged.
  */
 
-import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import { join as pathJoin } from 'node:path';
 import { localSessionsDir } from './paths.ts';
 import { friendlySessionTimestamp } from './teach-state.ts';
@@ -34,10 +34,13 @@ interface SessionInfo {
 }
 
 export function listSiteSessions(site: string): SessionInfo[] {
-  const sessDir = localSessionsDir(site);
-  if (!existsSync(sessDir)) return [];
+  return listSessionsInDir(localSessionsDir(site));
+}
+
+export function listSessionsInDir(dir: string): SessionInfo[] {
+  if (!existsSync(dir)) return [];
 
-  const files = readdirSync(sessDir).filter(
+  const files = readdirSync(dir).filter(
     (f) =>
       f.endsWith('.json') &&
       !f.includes('.redacted') &&
@@ -47,7 +50,7 @@ export function listSiteSessions(site: string): SessionInfo[] {
 
   const infos: SessionInfo[] = [];
   for (const filename of files) {
-    const absPath = pathJoin(sessDir, filename);
+    const absPath = pathJoin(dir, filename);
     try {
       const raw = JSON.parse(readFileSync(absPath, 'utf8'));
       const session = SessionSchema.parse(raw);
@@ -190,6 +193,7 @@ export function mergeSessions(sessions: Session[]): Session {
 
 export function writeCombinedSession(site: string, combined: Session): string {
   const sessDir = localSessionsDir(site);
+  mkdirSync(sessDir, { recursive: true });
   const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
   const filename = `combined-${timestamp}.json`;
   const absPath = pathJoin(sessDir, filename);

package/src/imprint/stealth-chromium.ts

@@ -0,0 +1,81 @@
+import { findChromium } from './chromium.ts';
+
+/**
+ * Shared loader for Playwright's chromium with the stealth plugin applied.
+ *
+ * Stealth patches navigator.webdriver, plugin enumeration, WebGL vendor
+ * strings, and other headless-Chrome telltales that anti-bot services
+ * (Akamai, Cloudflare, PerimeterX) detect. Vanilla headless Playwright
+ * gets tarpitted or 403'd by these services; the stealth-patched chromium
+ * loads the same pages in seconds.
+ *
+ * Falls back to vanilla `playwright` if `playwright-extra` /
+ * `puppeteer-extra-plugin-stealth` are not installed (preserves the
+ * graceful-degrade behavior of the original duplicated loaders in
+ * playbook-runner, replay-capture, and backend-ladder).
+ *
+ * Throws if no Playwright is available at all — callers translate the
+ * thrown error into their own result shape.
+ */
+export async function getStealthChromium(): Promise<typeof import('playwright').chromium> {
+  try {
+    const pwExtra = await import('playwright-extra');
+    const stealthMod = await import('puppeteer-extra-plugin-stealth');
+    const stealthFactory =
+      (stealthMod as { default?: () => unknown }).default ??
+      (stealthMod as unknown as () => unknown);
+    pwExtra.chromium.use(stealthFactory() as never);
+    return pwExtra.chromium as unknown as typeof import('playwright').chromium;
+  } catch {
+    const pw = await import('playwright');
+    return pw.chromium;
+  }
+}
+
+/**
+ * True when the puppeteer-extra stealth plugin is installed and WILL be applied
+ * by getStealthChromium() (i.e. we're not on the vanilla-Playwright fallback).
+ *
+ * Callers use this to avoid stacking a manual `navigator.webdriver` patch on top
+ * of the plugin's: the stealth plugin removes the property the way a real Chrome
+ * does (it simply lacks `webdriver`), whereas a redundant
+ * `Object.defineProperty(navigator,'webdriver',{get:()=>false})` leaves a
+ * non-native property descriptor that is ITSELF a fingerprinting tell. So the
+ * manual patch should only run on the vanilla fallback, where it's the only
+ * protection. Import resolution is cached, so probing here is cheap.
+ */
+export async function isStealthPluginAvailable(): Promise<boolean> {
+  try {
+    await import('playwright-extra');
+    await import('puppeteer-extra-plugin-stealth');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Path to the same Chromium binary `imprint record` uses for the user's
+ * recording session — Playwright's bundled "Google Chrome for Testing"
+ * (full Chrome build), the system Chrome on macOS, or a Linux distro
+ * Chrome/Chromium package, in that order of preference.
+ *
+ * Why this matters: by default Playwright's `chromium.launch({ headless: true })`
+ * picks `chrome-headless-shell` — a separate stripped-down binary that
+ * Akamai / Cloudflare / PerimeterX class anti-bot services detect at the
+ * binary/TLS-fingerprint layer regardless of how thoroughly the JS-level
+ * `navigator.webdriver` etc. are patched by the stealth plugin. The
+ * recording browser uses the FULL Chrome binary and Akamai trusts it; the
+ * replay browser using chrome-headless-shell looks like a bot. Using the
+ * SAME binary for both eliminates the binary asymmetry.
+ *
+ * Returns `undefined` if no Chromium can be located — callers should let
+ * Playwright fall back to whatever default it finds.
+ */
+export function getStealthExecutablePath(): string | undefined {
+  try {
+    return findChromium();
+  } catch {
+    return undefined;
+  }
+}