imprint-mcp 0.4.7 → 0.4.9

package/src/imprint/cdp-browser-fetch.ts

@@ -28,7 +28,6 @@
  * trusted browser session.
  */
 
-import { Buffer } from 'node:buffer';
 import CDP from 'chrome-remote-interface';
 import { launchChromium, proxyUrl } from './chromium.ts';
 import { createLog } from './log.ts';
@@ -53,23 +52,6 @@ export interface MintedJar {
   /** The bootstrap page HTML, so callers can satisfy html_regex captures
    *  (e.g. csrf / csp-nonce scraped from the page) without the browser. */
   html: string;
-  /** Browser-generated requests observed while the bootstrap page loaded. Lets
-   *  workflows capture replay headers minted by page JavaScript for later XHRs. */
-  observedRequests?: Array<{
-    method: string;
-    url: string;
-    headers: Record<string, string>;
-    body?: string;
-    resourceType?: string;
-    source?: 'browser' | 'replay';
-    response?: {
-      status: number;
-      headers: Record<string, string>;
-      body?: string;
-    };
-  }>;
-  /** Exact page URL used to create the page-specific HTML / observed requests. */
-  bootstrapUrl?: string;
   /** Date.now() at mint — the jar's validity is bounded (~2h fixed for Akamai). */
   bootstrapEpoch: number;
   /** The final `_abck` status field at capture (`0` = validated, `-1` = pending).
@@ -91,36 +73,17 @@ export interface MintedJar {
   source?: 'mint' | 'recording';
 }
 
-/** Akamai session jars have recognizable sensor cookies and must pass the
- *  stricter validation check below before plain-fetch replay is trusted. Other
- *  sites may still need a cached CDP bootstrap page + ordinary cookies, but
- *  will never carry `_abck`/`bm_sv`; do not apply Akamai-specific invalidation
- *  to those generic bootstrap artifacts. */
-export function jarHasAkamaiValidationSignals(
-  cookies: Array<{ name: string; value: string }>,
-): boolean {
-  return cookies.some((c) =>
-    ['_abck', 'bm_sv', 'ak_bmsc', 'bm_sz', 'bm_mi'].includes(c.name.toLowerCase()),
-  );
-}
-
-/** A session is replay-safe when either it is not an Akamai-style jar, or its
- *  `_abck` is validated (`~0~`) / the Akamai validated-session marker `bm_sv` is
- *  present. `bm_sv` survives `_abck` rotating back to `~-1~`. Shared by the cdp
- *  mint and recording-seed paths so both judge "validated" identically. */
+/** A session is replay-safe when `_abck` is validated (`~0~`) OR the Akamai
+ *  validated-session marker `bm_sv` is present (it is only set post-validation,
+ *  and survives `_abck` rotating back to `~-1~`). Shared by the cdp mint and the
+ *  recording-seed paths so both judge "validated" identically. */
 export function jarCookiesValidated(cookies: Array<{ name: string; value: string }>): boolean {
-  if (!jarHasAkamaiValidationSignals(cookies)) return true;
   const abck = cookies.find((c) => c.name === '_abck')?.value;
   if (abck && abck.split('~')[1] === '0') return true;
   return cookies.some((c) => c.name === 'bm_sv');
 }
 
 export interface CdpBrowserFetch {
-  /** Actual page URL used to bootstrap this browser session. */
-  readonly bootstrapUrl: string;
-  /** Reuse the same Chrome process for a new bootstrap page. The next
-   *  ensure/mint call navigates there and refreshes page-observed request state. */
-  setBootstrapUrl(bootstrapUrl: string): void;
   /** typeof fetch — executes the request inside the live trusted Chrome page. */
   readonly fetchImpl: typeof fetch;
   /** Force the bootstrap navigation + `_abck` validation now; returns the
@@ -251,7 +214,7 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
   // and never establishes the sensor session). Fall back to the origin root,
   // which loads a real page and runs the Akamai sensor JS.
   const baseLooksLikeApi = /\.act(\?|$)|\/api\//i.test(opts.baseUrl);
-  let navUrl = opts.bootstrapUrl ?? (baseLooksLikeApi ? `${baseOrigin}/` : opts.baseUrl);
+  const navUrl = opts.bootstrapUrl ?? (baseLooksLikeApi ? `${baseOrigin}/` : opts.baseUrl);
   const abckWaitMs = (opts.abckWaitSeconds ?? 25) * 1000;
   const reqTimeoutMs = opts.requestTimeoutMs ?? 60_000;
   const cdpCommandTimeoutMs = opts.cdpCommandTimeoutMs ?? 20_000;
@@ -261,18 +224,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
   let client: CdpClient | null = null;
   let bootstrapped = false;
   let appliedUa: string | undefined;
-  let forceDocumentReset = false;
-  const observedRequests: NonNullable<MintedJar['observedRequests']> = [];
-  const observedByRequestId = new Map<string, NonNullable<MintedJar['observedRequests']>[number]>();
-  const pendingResponseCaptures = new Set<Promise<void>>();
-  let networkObserversAttached = false;
-  const replayRequestKeys = new Set<string>();
-
-  function resetObservedRequests(): void {
-    observedRequests.length = 0;
-    observedByRequestId.clear();
-    pendingResponseCaptures.clear();
-  }
 
   async function close(): Promise<void> {
     const c = client;
@@ -323,65 +274,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
       await withTimeout(Runtime.enable(), 'CDP Runtime.enable', cdpCommandTimeoutMs);
       await withTimeout(Network.enable(), 'CDP Network.enable', cdpCommandTimeoutMs);
       await withTimeout(Page.enable(), 'CDP Page.enable', cdpCommandTimeoutMs);
-      if (!networkObserversAttached) {
-        networkObserversAttached = true;
-        Network.requestWillBeSent((params) => {
-          const req = params.request;
-          const headers: Record<string, string> = {};
-          for (const [k, v] of Object.entries(req.headers ?? {})) {
-            if (typeof v === 'string') headers[k] = v;
-          }
-          const postData = (req as { postData?: unknown }).postData;
-          const entry: NonNullable<MintedJar['observedRequests']>[number] = {
-            method: req.method,
-            url: req.url,
-            headers,
-            source: replayRequestKeys.delete(observedRequestKey(req.method, req.url, postData))
-              ? 'replay'
-              : 'browser',
-            ...(typeof postData === 'string' ? { body: postData } : {}),
-            ...(typeof params.type === 'string' ? { resourceType: params.type } : {}),
-          };
-          observedRequests.push(entry);
-          observedByRequestId.set(params.requestId, entry);
-          if (observedRequests.length > 100) observedRequests.shift();
-        });
-        Network.responseReceived((params) => {
-          const entry = observedByRequestId.get(params.requestId);
-          if (!entry) return;
-          const headers: Record<string, string> = {};
-          for (const [k, v] of Object.entries(params.response.headers ?? {})) {
-            if (typeof v === 'string') headers[k] = v;
-          }
-          entry.response = {
-            status: params.response.status,
-            headers,
-          };
-        });
-        Network.loadingFinished((params) => {
-          const entry = observedByRequestId.get(params.requestId);
-          if (!entry || !shouldCaptureObservedBody(entry)) return;
-          const pending = (async () => {
-            try {
-              const bodyResult = await withTimeout(
-                Network.getResponseBody({ requestId: params.requestId }),
-                'CDP Network.getResponseBody',
-                shortCdpTimeoutMs,
-              );
-              if (!entry.response) {
-                entry.response = { status: 200, headers: {} };
-              }
-              entry.response.body = bodyResult.base64Encoded
-                ? Buffer.from(bodyResult.body, 'base64').toString('utf8')
-                : bodyResult.body;
-            } catch {
-              // best-effort — response reuse simply won't match without a body
-            }
-          })();
-          pendingResponseCaptures.add(pending);
-          void pending.finally(() => pendingResponseCaptures.delete(pending));
-        });
-      }
       // Plant the high-trust seed cookies (the recording's validated Akamai jar)
       // BEFORE navigating, so the first request to the protected origin carries the
       // trusted session. A synthetic mint can reach `_abck~0~` yet still get its
@@ -443,20 +335,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
       // loads normally. Bound the CDP command and proceed regardless — _abck
       // polling below tolerates a partial load.
       try {
-        if (forceDocumentReset) {
-          forceDocumentReset = false;
-          await withTimeout(
-            Page.navigate({ url: 'about:blank' }),
-            'CDP Page.navigate(about:blank)',
-            Math.max(1, Math.min(abckWaitMs, 5_000)),
-          );
-          await withTimeout(
-            Page.loadEventFired(),
-            'CDP Page.loadEventFired(about:blank)',
-            Math.max(1, Math.min(abckWaitMs, 2_000)),
-          ).catch(() => {});
-          await sleep(250);
-        }
         await withTimeout(
           Page.navigate({ url: navUrl }),
           'CDP Page.navigate',
@@ -485,7 +363,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
       const start = Date.now();
       let i = 0;
       let status = '?';
-      let absentAbckChecks = 0;
       let pos = { x: rand(120, 1100), y: rand(120, 600) };
       while (Date.now() - start < abckWaitMs) {
         try {
@@ -548,16 +425,7 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
         }
         await sleep(rand(180, 520)); // non-uniform dwell between interaction bursts
         const abck = await getCookie(client, '_abck');
-        if (abck === undefined) {
-          absentAbckChecks++;
-          if (absentAbckChecks >= 2) {
-            status = 'absent';
-            break;
-          }
-        } else {
-          absentAbckChecks = 0;
-          status = abck.split('~')[1] ?? '?';
-        }
+        status = abck?.split('~')[1] ?? '?';
         if (abckIsValidated(abck)) break;
         i++;
       }
@@ -651,7 +519,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
 
     // Execute the fetch INSIDE the trusted page. credentials:'include' so the
     // browser attaches the validated session cookies.
-    replayRequestKeys.add(observedRequestKey(method, fullUrl, body));
     const expr = `(async () => {
       try {
         const ctrl = new AbortController();
@@ -695,17 +562,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
   }) as typeof fetch;
 
   return {
-    get bootstrapUrl() {
-      return navUrl;
-    },
-    setBootstrapUrl(nextBootstrapUrl: string): void {
-      if (nextBootstrapUrl === navUrl) return;
-      log(`retargeting pooled Chrome bootstrap (${navUrl} → ${nextBootstrapUrl})`);
-      navUrl = nextBootstrapUrl;
-      bootstrapped = false;
-      forceDocumentReset = true;
-      resetObservedRequests();
-    },
     fetchImpl,
     async ensureBootstrapped() {
       const c = await ensure();
@@ -725,7 +581,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
     },
     async mintJar(): Promise<MintedJar> {
       const c = await ensure();
-      await settlePendingResponseCaptures(pendingResponseCaptures, shortCdpTimeoutMs);
       const cookies: MintedJar['cookies'] = [];
       try {
         const res = await withTimeout(
@@ -768,8 +623,6 @@ export function createCdpBrowserFetch(opts: CdpBrowserFetchOptions): CdpBrowserF
         cookies,
         ua: appliedUa ?? '',
         html,
-        observedRequests: observedRequests.slice(),
-        bootstrapUrl: navUrl,
         bootstrapEpoch: Date.now(),
         abckFlag: abck?.split('~')[1] ?? '?',
         validated: jarCookiesValidated(cookies),
@@ -784,29 +637,6 @@ function sleep(ms: number): Promise<void> {
   return new Promise((r) => setTimeout(r, ms));
 }
 
-function shouldCaptureObservedBody(
-  entry: NonNullable<MintedJar['observedRequests']>[number],
-): boolean {
-  const type = entry.resourceType?.toLowerCase();
-  if (type && type !== 'xhr' && type !== 'fetch') return false;
-  return true;
-}
-
-function observedRequestKey(method: string, url: string, body: unknown): string {
-  return `${method.toUpperCase()} ${url} ${typeof body === 'string' ? body : ''}`;
-}
-
-async function settlePendingResponseCaptures(
-  pending: Set<Promise<void>>,
-  timeoutMs: number,
-): Promise<void> {
-  if (pending.size === 0) return;
-  await Promise.race([
-    Promise.allSettled([...pending]),
-    new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
-  ]);
-}
-
 async function withTimeout<T>(promise: Promise<T>, label: string, timeoutMs: number): Promise<T> {
   if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return promise;
   let timer: ReturnType<typeof setTimeout> | undefined;

package/src/imprint/cdp-jar-cache.ts

@@ -25,11 +25,7 @@ import {
   writeFileSync,
 } from 'node:fs';
 import { join as pathJoin } from 'node:path';
-import {
-  type MintedJar,
-  jarCookiesValidated,
-  jarHasAkamaiValidationSignals,
-} from './cdp-browser-fetch.ts';
+import { type MintedJar, jarCookiesValidated } from './cdp-browser-fetch.ts';
 import { createLog } from './log.ts';
 
 const log = createLog('cdp-jar');
@@ -59,32 +55,27 @@ function jarPath(siteDir: string): string {
  *  The cached `ua` is reused for replay verbatim; a UA drift (Chrome auto-update
  *  mid-window) is rare and self-heals reactively on a replay 403, so we do NOT
  *  launch Chrome just to gate on UA here. */
-export function loadJar(siteDir: string, bootstrapUrl?: string): MintedJar | null {
+export function loadJar(siteDir: string): MintedJar | null {
   const p = jarPath(siteDir);
   if (!existsSync(p)) return null;
   try {
     const raw = JSON.parse(readFileSync(p, 'utf8')) as Partial<MintedJar>;
     if (!raw || !Array.isArray(raw.cookies) || typeof raw.bootstrapEpoch !== 'number') return null;
-    if (bootstrapUrl && raw.bootstrapUrl && raw.bootstrapUrl !== bootstrapUrl) {
-      log('cached jar bootstrap URL differs from current workflow page — re-mint');
-      return null;
-    }
     const ageSeconds = (Date.now() - raw.bootstrapEpoch) / 1000;
     const maxAge = jarMaxAgeSeconds();
     if (ageSeconds >= maxAge) {
       log(`cached jar in ${siteDir} is ${Math.round(ageSeconds)}s old (>= ${maxAge}s) — re-mint`);
       return null;
     }
-    // Akamai jars must be validated (`_abck~0~` OR `bm_sv`). Non-Akamai jars
-    // may still be valid generic bootstrap artifacts (HTML captures + ordinary
-    // cookies), so do not discard them just because they lack Akamai markers.
-    // Fall back to cookie inspection for caches written before `validated`.
-    const validated = raw.validated ?? jarCookiesValidated(raw.cookies);
-    if (!validated && jarHasAkamaiValidationSignals(raw.cookies)) {
+    // Validated = `_abck~0~` OR `bm_sv` present (the latter survives `_abck`
+    // rotating back to `~-1~`). Fall back to the abckFlag check for caches
+    // written before the `validated` field existed.
+    const validated = raw.validated ?? raw.abckFlag === '0';
+    if (!validated) {
       log(`cached jar not validated (_abck~${raw.abckFlag}~, no bm_sv) — re-mint`);
       return null;
     }
-    return stripDurableObservedResponseBodies(raw as MintedJar);
+    return raw as MintedJar;
   } catch {
     return null;
   }
@@ -96,30 +87,13 @@ export function saveJar(siteDir: string, jar: MintedJar): void {
     mkdirSync(siteDir, { recursive: true });
     const p = jarPath(siteDir);
     const tmp = `${p}.${process.pid}.tmp`;
-    writeFileSync(tmp, `${JSON.stringify(stripDurableObservedResponseBodies(jar))}\n`, 'utf8');
+    writeFileSync(tmp, `${JSON.stringify(jar)}\n`, 'utf8');
     renameSync(tmp, p);
   } catch (err) {
     log(`failed to persist jar to ${siteDir}: ${err instanceof Error ? err.message : String(err)}`);
   }
 }
 
-function stripDurableObservedResponseBodies(jar: MintedJar): MintedJar {
-  if (!jar.observedRequests) return jar;
-  return {
-    ...jar,
-    observedRequests: jar.observedRequests.map((req) => {
-      if (!req.response || req.response.body === undefined) return req;
-      return {
-        ...req,
-        response: {
-          status: req.response.status,
-          headers: req.response.headers,
-        },
-      };
-    }),
-  };
-}
-
 /** Remove a cached jar (best-effort) — call on a replay 401/403/428/429 so the
  *  next call re-mints (reactive self-heal), or when a site's teach run ends. */
 export function clearJar(siteDir: string): void {
@@ -229,7 +203,6 @@ export function seedJarFromRecording(
   }));
   const abck = cookies.find((c) => c.name === '_abck')?.value;
   const abckFlag = abck?.split('~')[1] ?? '?';
-  const hasAkamaiValidationSignals = jarHasAkamaiValidationSignals(cookies);
   // Validated = `_abck~0~` OR a `bm_sv` cookie (Akamai's validated-session
   // marker). `_abck` rotates back to `~-1~` after clearing a request, so a real
   // working recording often ends with `_abck~-1~` + `bm_sv` — that jar replays
@@ -268,89 +241,21 @@ export function seedJarFromRecording(
   // captured HTML. Without this (the old `html: ''`), any workflow whose
   // requests reference `${state.X}` from an html_regex capture STATE_MISSINGs.
   const html = pickBootstrapHtml(session.requests ?? [], bootstrapUrl);
-  const observedRequests = collectObservedRequests(session.requests ?? []);
   saveJar(siteDir, {
     cookies,
     ua,
     html,
-    observedRequests,
-    ...(bootstrapUrl ? { bootstrapUrl } : {}),
     bootstrapEpoch: Math.round(newestMtime),
     abckFlag,
     validated: true, // gated above on jarCookiesValidated
     source: 'recording',
   });
-  const validationLabel = hasAkamaiValidationSignals ? 'bm_sv-validated' : 'generic-bootstrap';
   log(
-    `seeded jar from recording ${newest} (${cookies.length} cookies, _abck~${abckFlag}~, ${validationLabel}, ua=${ua ? `${ua.slice(0, 40)}…` : '(none)'}, html=${html.length}b)`,
+    `seeded jar from recording ${newest} (${cookies.length} cookies, _abck~${abckFlag}~, bm_sv-validated, ua=${ua ? `${ua.slice(0, 40)}…` : '(none)'}, html=${html.length}b)`,
   );
   return true;
 }
 
-function collectObservedRequests(
-  requests: Array<{
-    requestHeaders?: unknown;
-    headers?: unknown;
-    body?: unknown;
-    url?: string;
-    method?: string;
-    resourceType?: string;
-    response?: {
-      status?: number;
-      headers?: Record<string, string>;
-      body?: string;
-    };
-  }>,
-): NonNullable<MintedJar['observedRequests']> {
-  const observed: NonNullable<MintedJar['observedRequests']> = [];
-  for (const r of requests) {
-    if (!r.url) continue;
-    const headers = normalizeHeaders(r.requestHeaders ?? r.headers);
-    observed.push({
-      method: (r.method ?? 'GET').toUpperCase(),
-      url: r.url,
-      headers,
-      source: 'browser',
-      ...(typeof r.body === 'string' ? { body: r.body } : {}),
-      ...(typeof r.resourceType === 'string' ? { resourceType: r.resourceType } : {}),
-      ...(typeof r.response?.status === 'number'
-        ? {
-            response: {
-              status: r.response.status,
-              headers: r.response.headers ?? {},
-            },
-          }
-        : {}),
-    });
-    if (observed.length > 100) observed.shift();
-  }
-  return observed;
-}
-
-function normalizeHeaders(raw: unknown): Record<string, string> {
-  if (Array.isArray(raw)) {
-    const entries = raw
-      .map((x) =>
-        x &&
-        typeof x === 'object' &&
-        'name' in x &&
-        'value' in x &&
-        typeof x.name === 'string' &&
-        typeof x.value === 'string'
-          ? [x.name, x.value]
-          : null,
-      )
-      .filter((x): x is [string, string] => x !== null);
-    return Object.fromEntries(entries);
-  }
-  if (!raw || typeof raw !== 'object') return {};
-  const out: Record<string, string> = {};
-  for (const [k, v] of Object.entries(raw)) {
-    if (typeof v === 'string') out[k] = v;
-  }
-  return out;
-}
-
 /**
  * Choose the recorded HTML to seed into jar.html for html_regex bootstrap
  * captures. Preference order: (1) the recorded response for the exact bootstrap

package/src/imprint/compile-tools.ts

@@ -1261,7 +1261,7 @@ export async function typecheckArtifacts(
 
   try {
     const result = await runCommand(
-      'bun x tsc --noEmit -p .imprint-typecheck.tsconfig.json',
+      'bunx tsc --noEmit -p .imprint-typecheck.tsconfig.json',
       dir,
       120000,
     );
@@ -2937,7 +2937,7 @@ export async function externalVerification(
     const output = await typecheckArtifacts(toolDir, ['parser.ts', 'request-transform.ts']);
     if (output.exitCode !== 0 || output.timedOut) {
       failures.push(
-        `generated TypeScript artifacts failed typecheck (bun x tsc --noEmit -p .imprint-typecheck.tsconfig.json) exited ${output.exitCode}${output.timedOut ? ' after timing out' : ''}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`,
+        `generated TypeScript artifacts failed typecheck (bunx tsc --noEmit -p .imprint-typecheck.tsconfig.json) exited ${output.exitCode}${output.timedOut ? ' after timing out' : ''}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`,
       );
     }
   }