imprint-mcp 0.4.6 → 0.4.7

package/src/imprint/cdp-browser-fetch.ts

@@ -28,6 +28,7 @@
  * trusted browser session.
  */
 
+import { Buffer } from 'node:buffer';
 import CDP from 'chrome-remote-interface';
 import { launchChromium, proxyUrl } from './chromium.ts';
 import { createLog } from './log.ts';
@@ -52,6 +53,23 @@ export interface MintedJar {
   /** The bootstrap page HTML, so callers can satisfy html_regex captures
    *  (e.g. csrf / csp-nonce scraped from the page) without the browser. */
   html: string;
+  /** Browser-generated requests observed while the bootstrap page loaded. Lets
+   *  workflows capture replay headers minted by page JavaScript for later XHRs. */
+  observedRequests?: Array<{
+    method: string;
+    url: string;
+    headers: Record<string, string>;
+    body?: string;
+    resourceType?: string;
+    source?: 'browser' | 'replay';
+    response?: {
+      status: number;
+      headers: Record<string, string>;
+      body?: string;
+    };
+  }>;
+  /** Exact page URL used to create the page-specific HTML / observed requests. */
+  bootstrapUrl?: string;
   /** Date.now() at mint — the jar's validity is bounded (~2h fixed for Akamai). */
   bootstrapEpoch: number;
   /** The final `_abck` status field at capture (`0` = validated, `-1` = pending).
@@ -73,17 +91,36 @@ export interface MintedJar {
   source?: 'mint' | 'recording';
 }
 
-/** A session is replay-safe when `_abck` is validated (`~0~`) OR the Akamai
- *  validated-session marker `bm_sv` is present (it is only set post-validation,
- *  and survives `_abck` rotating back to `~-1~`). Shared by the cdp mint and the
- *  recording-seed paths so both judge "validated" identically. */
+/** Akamai session jars have recognizable sensor cookies and must pass the
+ *  stricter validation check below before plain-fetch replay is trusted. Other
+ *  sites may still need a cached CDP bootstrap page + ordinary cookies, but
+ *  will never carry `_abck`/`bm_sv`; do not apply Akamai-specific invalidation
+ *  to those generic bootstrap artifacts. */
+export function jarHasAkamaiValidationSignals(
+  cookies: Array<{ name: string; value: string }>,
+): boolean {
+  return cookies.some((c) =>
+    ['_abck', 'bm_sv', 'ak_bmsc', 'bm_sz', 'bm_mi'].includes(c.name.toLowerCase()),
+  );
+}
+
+/** A session is replay-safe when either it is not an Akamai-style jar, or its
+ *  `_abck` is validated (`~0~`) / the Akamai validated-session marker `bm_sv` is
+ *  present. `bm_sv` survives `_abck` rotating back to `~-1~`. Shared by the cdp
+ *  mint and recording-seed paths so both judge "validated" identically. */
 export function jarCookiesValidated(cookies: Array<{ name: string; value: string }>): boolean {
+  if (!jarHasAkamaiValidationSignals(cookies)) return true;
   const abck = cookies.find((c) => c.name === '_abck')?.value;
   if (abck && abck.split('~')[1] === '0') return true;
   return cookies.some((c) => c.name === 'bm_sv');
 }
 
 export interface CdpBrowserFetch {
+  /** Actual page URL used to bootstrap this browser session. */
+  readonly bootstrapUrl: string;
+  /** Reuse the same Chrome process for a new bootstrap page. The next
+   *  ensure/mint call navigates there and refreshes page-observed request state. */
+  setBootstrapUrl(bootstrapUrl: string): void;
   /** typeof fetch — executes the request inside the live trusted Chrome page. */
   readonly fetchImpl: typeof fetch;
   /** Force the bootstrap navigation + `_abck` validation now; returns the
@@ -214,7 +251,7 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
   // and never establishes the sensor session). Fall back to the origin root,
   // which loads a real page and runs the Akamai sensor JS.
   const baseLooksLikeApi = /\.act(\?|$)|\/api\//i.test(opts.baseUrl);
-  const navUrl = opts.bootstrapUrl ?? (baseLooksLikeApi ? `${baseOrigin}/` : opts.baseUrl);
+  let navUrl = opts.bootstrapUrl ?? (baseLooksLikeApi ? `${baseOrigin}/` : opts.baseUrl);
   const abckWaitMs = (opts.abckWaitSeconds ?? 25) * 1000;
   const reqTimeoutMs = opts.requestTimeoutMs ?? 60_000;
   const cdpCommandTimeoutMs = opts.cdpCommandTimeoutMs ?? 20_000;
@@ -224,6 +261,18 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
   let client: CdpClient | null = null;
   let bootstrapped = false;
   let appliedUa: string | undefined;
+  let forceDocumentReset = false;
+  const observedRequests: NonNullable<MintedJar['observedRequests']> = [];
+  const observedByRequestId = new Map<string, NonNullable<MintedJar['observedRequests']>[number]>();
+  const pendingResponseCaptures = new Set<Promise<void>>();
+  let networkObserversAttached = false;
+  const replayRequestKeys = new Set<string>();
+
+  function resetObservedRequests(): void {
+    observedRequests.length = 0;
+    observedByRequestId.clear();
+    pendingResponseCaptures.clear();
+  }
 
   async function close(): Promise<void> {
     const c = client;
@@ -274,6 +323,65 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
       await withTimeout(Runtime.enable(), 'CDP Runtime.enable', cdpCommandTimeoutMs);
       await withTimeout(Network.enable(), 'CDP Network.enable', cdpCommandTimeoutMs);
       await withTimeout(Page.enable(), 'CDP Page.enable', cdpCommandTimeoutMs);
+      if (!networkObserversAttached) {
+        networkObserversAttached = true;
+        Network.requestWillBeSent((params) => {
+          const req = params.request;
+          const headers: Record<string, string> = {};
+          for (const [k, v] of Object.entries(req.headers ?? {})) {
+            if (typeof v === 'string') headers[k] = v;
+          }
+          const postData = (req as { postData?: unknown }).postData;
+          const entry: NonNullable<MintedJar['observedRequests']>[number] = {
+            method: req.method,
+            url: req.url,
+            headers,
+            source: replayRequestKeys.delete(observedRequestKey(req.method, req.url, postData))
+              ? 'replay'
+              : 'browser',
+            ...(typeof postData === 'string' ? { body: postData } : {}),
+            ...(typeof params.type === 'string' ? { resourceType: params.type } : {}),
+          };
+          observedRequests.push(entry);
+          observedByRequestId.set(params.requestId, entry);
+          if (observedRequests.length > 100) observedRequests.shift();
+        });
+        Network.responseReceived((params) => {
+          const entry = observedByRequestId.get(params.requestId);
+          if (!entry) return;
+          const headers: Record<string, string> = {};
+          for (const [k, v] of Object.entries(params.response.headers ?? {})) {
+            if (typeof v === 'string') headers[k] = v;
+          }
+          entry.response = {
+            status: params.response.status,
+            headers,
+          };
+        });
+        Network.loadingFinished((params) => {
+          const entry = observedByRequestId.get(params.requestId);
+          if (!entry || !shouldCaptureObservedBody(entry)) return;
+          const pending = (async () => {
+            try {
+              const bodyResult = await withTimeout(
+                Network.getResponseBody({ requestId: params.requestId }),
+                'CDP Network.getResponseBody',
+                shortCdpTimeoutMs,
+              );
+              if (!entry.response) {
+                entry.response = { status: 200, headers: {} };
+              }
+              entry.response.body = bodyResult.base64Encoded
+                ? Buffer.from(bodyResult.body, 'base64').toString('utf8')
+                : bodyResult.body;
+            } catch {
+              // best-effort — response reuse simply won't match without a body
+            }
+          })();
+          pendingResponseCaptures.add(pending);
+          void pending.finally(() => pendingResponseCaptures.delete(pending));
+        });
+      }
       // Plant the high-trust seed cookies (the recording's validated Akamai jar)
       // BEFORE navigating, so the first request to the protected origin carries the
       // trusted session. A synthetic mint can reach `_abck~0~` yet still get its
@@ -335,6 +443,20 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
       // loads normally. Bound the CDP command and proceed regardless — _abck
       // polling below tolerates a partial load.
       try {
+        if (forceDocumentReset) {
+          forceDocumentReset = false;
+          await withTimeout(
+            Page.navigate({ url: 'about:blank' }),
+            'CDP Page.navigate(about:blank)',
+            Math.max(1, Math.min(abckWaitMs, 5_000)),
+          );
+          await withTimeout(
+            Page.loadEventFired(),
+            'CDP Page.loadEventFired(about:blank)',
+            Math.max(1, Math.min(abckWaitMs, 2_000)),
+          ).catch(() => {});
+          await sleep(250);
+        }
         await withTimeout(
           Page.navigate({ url: navUrl }),
           'CDP Page.navigate',
@@ -363,6 +485,7 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
       const start = Date.now();
       let i = 0;
       let status = '?';
+      let absentAbckChecks = 0;
       let pos = { x: rand(120, 1100), y: rand(120, 600) };
       while (Date.now() - start < abckWaitMs) {
         try {
@@ -425,7 +548,16 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
         }
         await sleep(rand(180, 520)); // non-uniform dwell between interaction bursts
         const abck = await getCookie(client, '_abck');
-        status = abck?.split('~')[1] ?? '?';
+        if (abck === undefined) {
+          absentAbckChecks++;
+          if (absentAbckChecks >= 2) {
+            status = 'absent';
+            break;
+          }
+        } else {
+          absentAbckChecks = 0;
+          status = abck.split('~')[1] ?? '?';
+        }
         if (abckIsValidated(abck)) break;
         i++;
       }
@@ -519,6 +651,7 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
 
     // Execute the fetch INSIDE the trusted page. credentials:'include' so the
     // browser attaches the validated session cookies.
+    replayRequestKeys.add(observedRequestKey(method, fullUrl, body));
     const expr = `(async () => {
       try {
         const ctrl = new AbortController();
@@ -562,6 +695,17 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
   }) as typeof fetch;
 
   return {
+    get bootstrapUrl() {
+      return navUrl;
+    },
+    setBootstrapUrl(nextBootstrapUrl: string): void {
+      if (nextBootstrapUrl === navUrl) return;
+      log(`retargeting pooled Chrome bootstrap (${navUrl} → ${nextBootstrapUrl})`);
+      navUrl = nextBootstrapUrl;
+      bootstrapped = false;
+      forceDocumentReset = true;
+      resetObservedRequests();
+    },
     fetchImpl,
     async ensureBootstrapped() {
       const c = await ensure();
@@ -581,6 +725,7 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
     },
     async mintJar(): Promise<MintedJar> {
       const c = await ensure();
+      await settlePendingResponseCaptures(pendingResponseCaptures, shortCdpTimeoutMs);
       const cookies: MintedJar['cookies'] = [];
       try {
         const res = await withTimeout(
@@ -623,6 +768,8 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
         cookies,
         ua: appliedUa ?? '',
         html,
+        observedRequests: observedRequests.slice(),
+        bootstrapUrl: navUrl,
         bootstrapEpoch: Date.now(),
         abckFlag: abck?.split('~')[1] ?? '?',
         validated: jarCookiesValidated(cookies),
@@ -637,6 +784,29 @@ function sleep(ms: number): Promise<void> {
   return new Promise((r) => setTimeout(r, ms));
 }
 
+function shouldCaptureObservedBody(
+  entry: NonNullable<MintedJar['observedRequests']>[number],
+): boolean {
+  const type = entry.resourceType?.toLowerCase();
+  if (type && type !== 'xhr' && type !== 'fetch') return false;
+  return true;
+}
+
+function observedRequestKey(method: string, url: string, body: unknown): string {
+  return `${method.toUpperCase()} ${url} ${typeof body === 'string' ? body : ''}`;
+}
+
+async function settlePendingResponseCaptures(
+  pending: Set<Promise<void>>,
+  timeoutMs: number,
+): Promise<void> {
+  if (pending.size === 0) return;
+  await Promise.race([
+    Promise.allSettled([...pending]),
+    new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
+  ]);
+}
+
 async function withTimeout<T>(promise: Promise<T>, label: string, timeoutMs: number): Promise<T> {
   if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return promise;
   let timer: ReturnType<typeof setTimeout> | undefined;

package/src/imprint/cdp-jar-cache.ts

@@ -25,7 +25,11 @@ import {
   writeFileSync,
 } from 'node:fs';
 import { join as pathJoin } from 'node:path';
-import { type MintedJar, jarCookiesValidated } from './cdp-browser-fetch.ts';
+import {
+  type MintedJar,
+  jarCookiesValidated,
+  jarHasAkamaiValidationSignals,
+} from './cdp-browser-fetch.ts';
 import { createLog } from './log.ts';
 
 const log = createLog('cdp-jar');
@@ -55,27 +59,32 @@ function jarPath(siteDir: string): string {
  *  The cached `ua` is reused for replay verbatim; a UA drift (Chrome auto-update
  *  mid-window) is rare and self-heals reactively on a replay 403, so we do NOT
  *  launch Chrome just to gate on UA here. */
-export function loadJar(siteDir: string): MintedJar | null {
+export function loadJar(siteDir: string, bootstrapUrl?: string): MintedJar | null {
   const p = jarPath(siteDir);
   if (!existsSync(p)) return null;
   try {
     const raw = JSON.parse(readFileSync(p, 'utf8')) as Partial<MintedJar>;
     if (!raw || !Array.isArray(raw.cookies) || typeof raw.bootstrapEpoch !== 'number') return null;
+    if (bootstrapUrl && raw.bootstrapUrl && raw.bootstrapUrl !== bootstrapUrl) {
+      log('cached jar bootstrap URL differs from current workflow page — re-mint');
+      return null;
+    }
     const ageSeconds = (Date.now() - raw.bootstrapEpoch) / 1000;
     const maxAge = jarMaxAgeSeconds();
     if (ageSeconds >= maxAge) {
       log(`cached jar in ${siteDir} is ${Math.round(ageSeconds)}s old (>= ${maxAge}s) — re-mint`);
       return null;
     }
-    // Validated = `_abck~0~` OR `bm_sv` present (the latter survives `_abck`
-    // rotating back to `~-1~`). Fall back to the abckFlag check for caches
-    // written before the `validated` field existed.
-    const validated = raw.validated ?? raw.abckFlag === '0';
-    if (!validated) {
+    // Akamai jars must be validated (`_abck~0~` OR `bm_sv`). Non-Akamai jars
+    // may still be valid generic bootstrap artifacts (HTML captures + ordinary
+    // cookies), so do not discard them just because they lack Akamai markers.
+    // Fall back to cookie inspection for caches written before `validated`.
+    const validated = raw.validated ?? jarCookiesValidated(raw.cookies);
+    if (!validated && jarHasAkamaiValidationSignals(raw.cookies)) {
       log(`cached jar not validated (_abck~${raw.abckFlag}~, no bm_sv) — re-mint`);
       return null;
     }
-    return raw as MintedJar;
+    return stripDurableObservedResponseBodies(raw as MintedJar);
   } catch {
     return null;
   }
@@ -87,13 +96,30 @@ export function saveJar(siteDir: string, jar: MintedJar): void {
     mkdirSync(siteDir, { recursive: true });
     const p = jarPath(siteDir);
     const tmp = `${p}.${process.pid}.tmp`;
-    writeFileSync(tmp, `${JSON.stringify(jar)}\n`, 'utf8');
+    writeFileSync(tmp, `${JSON.stringify(stripDurableObservedResponseBodies(jar))}\n`, 'utf8');
     renameSync(tmp, p);
   } catch (err) {
     log(`failed to persist jar to ${siteDir}: ${err instanceof Error ? err.message : String(err)}`);
   }
 }
 
+function stripDurableObservedResponseBodies(jar: MintedJar): MintedJar {
+  if (!jar.observedRequests) return jar;
+  return {
+    ...jar,
+    observedRequests: jar.observedRequests.map((req) => {
+      if (!req.response || req.response.body === undefined) return req;
+      return {
+        ...req,
+        response: {
+          status: req.response.status,
+          headers: req.response.headers,
+        },
+      };
+    }),
+  };
+}
+
 /** Remove a cached jar (best-effort) — call on a replay 401/403/428/429 so the
  *  next call re-mints (reactive self-heal), or when a site's teach run ends. */
 export function clearJar(siteDir: string): void {
@@ -203,6 +229,7 @@ export function seedJarFromRecording(
   }));
   const abck = cookies.find((c) => c.name === '_abck')?.value;
   const abckFlag = abck?.split('~')[1] ?? '?';
+  const hasAkamaiValidationSignals = jarHasAkamaiValidationSignals(cookies);
   // Validated = `_abck~0~` OR a `bm_sv` cookie (Akamai's validated-session
   // marker). `_abck` rotates back to `~-1~` after clearing a request, so a real
   // working recording often ends with `_abck~-1~` + `bm_sv` — that jar replays
@@ -241,21 +268,89 @@ export function seedJarFromRecording(
   // captured HTML. Without this (the old `html: ''`), any workflow whose
   // requests reference `${state.X}` from an html_regex capture STATE_MISSINGs.
   const html = pickBootstrapHtml(session.requests ?? [], bootstrapUrl);
+  const observedRequests = collectObservedRequests(session.requests ?? []);
   saveJar(siteDir, {
     cookies,
     ua,
     html,
+    observedRequests,
+    ...(bootstrapUrl ? { bootstrapUrl } : {}),
     bootstrapEpoch: Math.round(newestMtime),
     abckFlag,
     validated: true, // gated above on jarCookiesValidated
     source: 'recording',
   });
+  const validationLabel = hasAkamaiValidationSignals ? 'bm_sv-validated' : 'generic-bootstrap';
   log(
-    `seeded jar from recording ${newest} (${cookies.length} cookies, _abck~${abckFlag}~, bm_sv-validated, ua=${ua ? `${ua.slice(0, 40)}…` : '(none)'}, html=${html.length}b)`,
+    `seeded jar from recording ${newest} (${cookies.length} cookies, _abck~${abckFlag}~, ${validationLabel}, ua=${ua ? `${ua.slice(0, 40)}…` : '(none)'}, html=${html.length}b)`,
   );
   return true;
 }
 
+function collectObservedRequests(
+  requests: Array<{
+    requestHeaders?: unknown;
+    headers?: unknown;
+    body?: unknown;
+    url?: string;
+    method?: string;
+    resourceType?: string;
+    response?: {
+      status?: number;
+      headers?: Record<string, string>;
+      body?: string;
+    };
+  }>,
+): NonNullable<MintedJar['observedRequests']> {
+  const observed: NonNullable<MintedJar['observedRequests']> = [];
+  for (const r of requests) {
+    if (!r.url) continue;
+    const headers = normalizeHeaders(r.requestHeaders ?? r.headers);
+    observed.push({
+      method: (r.method ?? 'GET').toUpperCase(),
+      url: r.url,
+      headers,
+      source: 'browser',
+      ...(typeof r.body === 'string' ? { body: r.body } : {}),
+      ...(typeof r.resourceType === 'string' ? { resourceType: r.resourceType } : {}),
+      ...(typeof r.response?.status === 'number'
+        ? {
+            response: {
+              status: r.response.status,
+              headers: r.response.headers ?? {},
+            },
+          }
+        : {}),
+    });
+    if (observed.length > 100) observed.shift();
+  }
+  return observed;
+}
+
+function normalizeHeaders(raw: unknown): Record<string, string> {
+  if (Array.isArray(raw)) {
+    const entries = raw
+      .map((x) =>
+        x &&
+        typeof x === 'object' &&
+        'name' in x &&
+        'value' in x &&
+        typeof x.name === 'string' &&
+        typeof x.value === 'string'
+          ? [x.name, x.value]
+          : null,
+      )
+      .filter((x): x is [string, string] => x !== null);
+    return Object.fromEntries(entries);
+  }
+  if (!raw || typeof raw !== 'object') return {};
+  const out: Record<string, string> = {};
+  for (const [k, v] of Object.entries(raw)) {
+    if (typeof v === 'string') out[k] = v;
+  }
+  return out;
+}
+
 /**
  * Choose the recorded HTML to seed into jar.html for html_regex bootstrap
  * captures. Preference order: (1) the recorded response for the exact bootstrap

package/src/imprint/compile-tools.ts

@@ -1261,7 +1261,7 @@ export async function typecheckArtifacts(
 
   try {
     const result = await runCommand(
-      'bunx tsc --noEmit -p .imprint-typecheck.tsconfig.json',
+      'bun x tsc --noEmit -p .imprint-typecheck.tsconfig.json',
       dir,
       120000,
     );
@@ -2937,7 +2937,7 @@ export async function externalVerification(
     const output = await typecheckArtifacts(toolDir, ['parser.ts', 'request-transform.ts']);
     if (output.exitCode !== 0 || output.timedOut) {
       failures.push(
-        `generated TypeScript artifacts failed typecheck (bunx tsc --noEmit -p .imprint-typecheck.tsconfig.json) exited ${output.exitCode}${output.timedOut ? ' after timing out' : ''}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`,
+        `generated TypeScript artifacts failed typecheck (bun x tsc --noEmit -p .imprint-typecheck.tsconfig.json) exited ${output.exitCode}${output.timedOut ? ' after timing out' : ''}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`,
       );
     }
   }