imprint-mcp 0.2.1 → 0.3.0

package/src/imprint/compile.ts

@@ -19,6 +19,7 @@ import {
 import { dirname, join as pathJoin } from 'node:path';
 import type { OnDeadlineReached } from './agent.ts';
 import { inferAppApiHosts } from './app-api-hosts.ts';
+import type { SharedModuleManifestEntry } from './build-plan.ts';
 import { type CompileAgentProgress, compileAgent } from './compile-agent.ts';
 import { isSameRegistrableDomain, registrableDomain } from './etld.ts';
 import { type LLMOptions, extractJsonArray, resolveProvider } from './llm.ts';
@@ -82,6 +83,14 @@ interface GenerateOptions extends CompileOptions {
   classifications?: ClassifiedValue[];
   /** Credential values extracted during teach, passed to integration tests via env var. */
   teachCredentials?: { site: string; values: Record<string, string> };
+  /** Absolute path to the multi-tool build plan sidecar (.build-plan.json). */
+  buildPlanPath?: string;
+  /** Shared-module build manifest for this site (verified flags). */
+  sharedModules?: SharedModuleManifestEntry[];
+  /** Per-tool implementation plan (param→field mapping, request construction,
+   *  response parsing, shared-module imports). Injected into the agent's initial
+   *  message so the compile follows it. */
+  toolPlan?: string;
 }
 
 interface GenerateResult {
@@ -122,6 +131,9 @@ export async function generate(opts: GenerateOptions): Promise<GenerateResult> {
         sharedContext: opts.sharedContext,
         classifications: opts.classifications,
         teachCredentials: opts.teachCredentials,
+        buildPlanPath: opts.buildPlanPath,
+        sharedModules: opts.sharedModules,
+        toolPlan: opts.toolPlan,
       });
 
       setSpanAttributes(span, {
@@ -274,7 +286,7 @@ const TRIAGE_RESOURCE_TYPES = new Set(['XHR', 'Fetch', 'Document']);
 const HEADER_TRUNCATE_LIMIT = 200;
 // Per-request body cap for triage. Triage only needs enough body to distinguish
 // data-bearing POSTs (search/booking) from telemetry; full bodies on a busy
-// site can total >1MB and blow the 200K-token cap on `claude-opus-4-7`.
+// site can total >1MB and blow the 200K-token cap on `claude-opus-4-8`.
 const TRIAGE_BODY_LIMIT = 500;
 
 export interface TriageResult {

package/src/imprint/concurrency.ts

@@ -0,0 +1,87 @@
+/**
+ * Bounded-concurrency fan-out helpers shared across the teach pipeline.
+ *
+ * Lives in its own module (rather than teach.ts) so leaf modules like
+ * teach-plan.ts can reuse it without importing teach.ts, which would create an
+ * import cycle (teach.ts → teach-plan.ts → teach.ts). teach.ts re-exports both
+ * for backwards compatibility with existing callers + tests.
+ */
+
+/** Run `fn` over `items` with at most `concurrency` in flight, preserving input
+ *  order in the result. Throws the first error encountered (after in-flight work
+ *  settles); use mapLimitSettled when you need per-item success/failure. */
+export async function mapLimit<T, R>(
+  items: T[],
+  concurrency: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  let firstError: unknown;
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (next < items.length && firstError === undefined) {
+      const index = next++;
+      const item = items[index];
+      if (item === undefined) continue;
+      try {
+        results[index] = await fn(item);
+      } catch (err) {
+        firstError ??= err;
+      }
+    }
+  });
+  await Promise.allSettled(workers);
+  if (firstError !== undefined) throw firstError;
+  return results;
+}
+
+type SettledResult<R> = { ok: true; value: R } | { ok: false; error: unknown };
+
+/** Like mapLimit, but never throws: each item resolves to a tagged
+ *  success/failure entry, preserving input order. */
+export async function mapLimitSettled<T, R>(
+  items: T[],
+  concurrency: number,
+  fn: (item: T) => Promise<R>,
+): Promise<SettledResult<R>[]> {
+  const results = new Array<SettledResult<R>>(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (next < items.length) {
+      const index = next++;
+      const item = items[index];
+      if (item === undefined) continue;
+      try {
+        results[index] = { ok: true, value: await fn(item) };
+      } catch (err) {
+        results[index] = { ok: false, error: err };
+      }
+    }
+  });
+  await Promise.allSettled(workers);
+  return results;
+}
+
+/** Error thrown by withTimeout when the deadline elapses before the work settles.
+ *  A distinct class lets callers tell a timeout apart from a genuine failure. */
+export class TimeoutError extends Error {
+  constructor(label: string, ms: number) {
+    super(`${label} exceeded ${Math.round(ms / 1000)}s timeout`);
+    this.name = 'TimeoutError';
+  }
+}
+
+/** Race a promise against a timeout. The underlying work (e.g. a CLI child) is
+ *  NOT cancelled — the caller just stops awaiting it and decides how to degrade.
+ *  Throws TimeoutError on timeout. */
+export async function withTimeout<T>(work: Promise<T>, ms: number, label: string): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new TimeoutError(label, ms)), ms);
+  });
+  try {
+    return await Promise.race([work, timeout]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}

package/src/imprint/cron.ts

@@ -242,6 +242,7 @@ async function runCronImpl(opts: RunCronOptions): Promise<void> {
   if (
     ladder.includes('fetch') ||
     ladder.includes('fetch-bootstrap') ||
+    ladder.includes('cdp-replay') ||
     ladder.includes('stealth-fetch')
   ) {
     const validator = buildZodValidator(tool.workflow.parameters);

package/src/imprint/doctor.ts

@@ -1,6 +1,7 @@
 /** `imprint doctor` — check that the environment can actually run imprint.
  *  Reports pass/fail per prerequisite plus a one-line fix when failed. */
 
+import { spawnSync } from 'node:child_process';
 import { existsSync, readFileSync, readdirSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { join as pathJoin } from 'node:path';
@@ -20,6 +21,7 @@ export function doctor(): CheckResult[] {
     checkBun(),
     checkChromium(),
     checkPlaywrightChromium(),
+    checkVirtualDisplay(),
     checkLLMProvider(),
     checkPushOptional(),
     checkClaudeCode(),
@@ -87,6 +89,43 @@ function checkPlaywrightChromium(): CheckResult {
   };
 }
 
+function hasXvfbBinary(): boolean {
+  try {
+    return spawnSync('sh', ['-c', 'command -v Xvfb'], { stdio: 'ignore' }).status === 0;
+  } catch {
+    return false;
+  }
+}
+
+/** The trusted-browser replay (playbook rung's cdp-browser transport) runs Chrome
+ *  HEADLESS by default and needs NO display — the `HeadlessChrome` UA token is
+ *  stripped so anti-bot services don't edge-block it. A display only matters as a
+ *  fallback on a GPU-less Linux host, where headless WebGL reports SwiftShader and
+ *  the replay must run HEADED under Xvfb (launchChromium auto-starts it when a
+ *  headed launch finds no `$DISPLAY`). macOS/Windows need nothing. Advisory only. */
+function checkVirtualDisplay(): CheckResult {
+  const name = 'Display (headed replay)';
+  if (process.platform !== 'linux') {
+    return { name, ok: true, detail: `${process.platform}: native window server (no Xvfb needed)` };
+  }
+  const display = process.env.DISPLAY;
+  if (display) return { name, ok: true, detail: `$DISPLAY=${display}` };
+  if (hasXvfbBinary()) {
+    return {
+      name,
+      ok: true,
+      detail: 'no $DISPLAY; Xvfb present — headed-replay fallback available for GPU-less hosts',
+    };
+  }
+  return {
+    name,
+    ok: true, // advisory — default replay is headless; Xvfb is only a GPU-less fallback
+    detail:
+      'Linux, no $DISPLAY and no Xvfb — default replay is headless (fine); install Xvfb only if a GPU-less host gets bot-flagged',
+    fix: 'GPU-less host bot-flagged? install the headed-replay fallback: apt-get install xvfb (or export DISPLAY=:0)',
+  };
+}
+
 function checkLLMProvider(): CheckResult {
   const statuses = getProviderStatuses();
   const detected = statuses.filter((s) => s.detected);

package/src/imprint/freeform-redact.ts

@@ -73,10 +73,11 @@ const FREEFORM_POLICIES: PolicyName[] = [
   Policies.PGP_PRIVATE_KEY,
   Policies.PASSWORD_ASSIGNMENT,
   Policies.ENVIRONMENT_VARIABLE_SECRET,
-  Policies.GENERIC_PASSWORD,
-  Policies.GENERIC_TOKEN,
-  Policies.GENERIC_CREDENTIAL,
-  Policies.GENERIC_SECRET,
+  // NOTE: the GENERIC_* catch-alls (GENERIC_PASSWORD/TOKEN/CREDENTIAL/SECRET) are
+  // intentionally omitted — they match on value shape alone and fire on benign
+  // data (e.g. `id=1234567890`), corrupting/over-redacting structured payloads.
+  // Real secrets are still covered by the keyword-anchored and specific policies
+  // above and below (PASSWORD_ASSIGNMENT, OAUTH_*, private keys, cloud tokens, PII).
   Policies.OAUTH_CLIENT_SECRET,
   Policies.OAUTH_REFRESH_TOKEN,
   Policies.OAUTH_ACCESS_TOKEN,

package/src/imprint/integrations.ts

@@ -87,7 +87,7 @@ export function generatePasteSnippet(opts: {
 
   switch (platform) {
     case 'claude-code':
-      return `Add the ${toolName} tool: run \`${shellCmd}\` to register ${descLower}. Parameters: ${paramList}. The backend ladder handles browser/API state and bot detection automatically (fetch → gated fetch-bootstrap → stealth-fetch → playbook).`;
+      return `Add the ${toolName} tool: run \`${shellCmd}\` to register ${descLower}. Parameters: ${paramList}. The backend ladder handles browser/API state and bot detection automatically (fetch → gated fetch-bootstrap → cdp-replay → stealth-fetch → playbook).`;
 
     case 'codex':
       return `Add the ${toolName} tool: run \`${shellCmd}\` to register ${descLower}. Parameters: ${paramList}.`;
@@ -352,7 +352,7 @@ ${yamlStringify(p, { lineWidth: 0 }).trim()}
   // Backend ladder explanation.
   const backendBlock = `## Backend Ladder
 
-The MCP server automatically escalates from fetch API replay to gated fetch-bootstrap when browser-minted state is declared, then stealth-fetch for bot-defense state, then playbook for full DOM replay.
+The MCP server automatically escalates from fetch API replay to gated fetch-bootstrap when browser-minted state is declared, then cdp-replay (API requests run inside a live trusted Chrome so a protected POST refreshes its anti-bot token between calls), then stealth-fetch for bot-defense state, then playbook for full DOM replay.
 Bot detection is handled transparently.`;
 
   // Scheduling block (optional).

package/src/imprint/llm.ts

@@ -6,6 +6,7 @@ import {
   llmSpanAttributes,
   resolveTraceTokenCount,
   setSpanAttributes,
+  totalPromptTokens,
   traceLlmIoEnabled,
   traceLlmMessages,
   traced,
@@ -17,6 +18,16 @@ interface AnalyzeResult {
   text: string;
   inputTokens: number | null;
   outputTokens: number | null;
+  /**
+   * Prompt-cache token counts, when the provider reports them. `inputTokens` is
+   * the *uncached* input only (the Anthropic/CLI `usage.input_tokens`); the bulk
+   * of a cache-hit call lives here. Threaded through so `llm.analyze` cost is
+   * cache-aware (cache reads bill at 0.1×, writes at 1.25×) instead of charging
+   * the whole prompt at the full input rate. Null/undefined for providers that
+   * don't expose usage (codex-cli, cursor-cli).
+   */
+  cacheReadInputTokens?: number | null;
+  cacheCreationInputTokens?: number | null;
   durationMs: number;
   stopReason: string | null;
 }
@@ -129,6 +140,8 @@ class AnthropicApiProvider implements LLMProvider {
           text,
           inputTokens: response.usage.input_tokens,
           outputTokens: response.usage.output_tokens,
+          cacheReadInputTokens: response.usage.cache_read_input_tokens ?? null,
+          cacheCreationInputTokens: response.usage.cache_creation_input_tokens ?? null,
           durationMs: Date.now() - t0,
           stopReason: response.stop_reason ?? null,
         };
@@ -256,7 +269,15 @@ class ClaudeCliProvider implements LLMProvider {
           );
         }
 
-        let parsed: { result?: string; usage?: { input_tokens?: number; output_tokens?: number } };
+        let parsed: {
+          result?: string;
+          usage?: {
+            input_tokens?: number;
+            output_tokens?: number;
+            cache_read_input_tokens?: number;
+            cache_creation_input_tokens?: number;
+          };
+        };
         try {
           parsed = JSON.parse(stdout);
         } catch (parseErr) {
@@ -273,6 +294,8 @@ class ClaudeCliProvider implements LLMProvider {
           text: parsed.result,
           inputTokens: parsed.usage?.input_tokens ?? null,
           outputTokens: parsed.usage?.output_tokens ?? null,
+          cacheReadInputTokens: parsed.usage?.cache_read_input_tokens ?? null,
+          cacheCreationInputTokens: parsed.usage?.cache_creation_input_tokens ?? null,
           durationMs: Date.now() - t0,
           stopReason: null,
         };
@@ -437,7 +460,20 @@ async function traceAnalyze(
     },
     async (span) => {
       const result = await fn();
-      const inputTokens = resolveTraceTokenCount(result.inputTokens, details?.inputText);
+      // Providers report `inputTokens` as the *uncached* input only; the cached
+      // portion lives in the cache fields. `llmCostAttributes` expects the TOTAL
+      // prompt tokens (it derives uncached = total − cacheRead − cacheWrite), so
+      // sum them here. A real total is also large enough to clear the
+      // resolveTraceTokenCount sanity check, so cache-hit calls stop falling back
+      // to the chars/4 estimate.
+      const cacheReadTokens = result.cacheReadInputTokens ?? undefined;
+      const cacheWriteTokens = result.cacheCreationInputTokens ?? undefined;
+      const totalInputTokens = totalPromptTokens(
+        result.inputTokens,
+        cacheReadTokens,
+        cacheWriteTokens,
+      );
+      const inputTokens = resolveTraceTokenCount(totalInputTokens, details?.inputText);
       const outputTokens = resolveTraceTokenCount(result.outputTokens, result.text);
       setSpanAttributes(span, {
         ...llmSpanAttributes({
@@ -445,6 +481,8 @@ async function traceAnalyze(
           model,
           inputTokens: inputTokens.tokens,
           outputTokens: outputTokens.tokens,
+          cacheReadTokens,
+          cacheWriteTokens,
           tokenCountsEstimated:
             inputTokens.source === 'estimated' || outputTokens.source === 'estimated',
           inputTokenSource: inputTokens.source,
@@ -512,12 +550,20 @@ async function traceMessageWithTools(
           return `[${b.type}]`;
         })
         .join('\n');
+      const cacheReadTokens = response.usage.cache_read_input_tokens ?? undefined;
+      const cacheWriteTokens = response.usage.cache_creation_input_tokens ?? undefined;
       setSpanAttributes(span, {
         ...llmSpanAttributes({
           provider,
           model,
-          inputTokens: response.usage.input_tokens,
+          inputTokens: totalPromptTokens(
+            response.usage.input_tokens,
+            cacheReadTokens,
+            cacheWriteTokens,
+          ),
           outputTokens: response.usage.output_tokens,
+          cacheReadTokens,
+          cacheWriteTokens,
           stopReason: response.stop_reason,
           outputMessages: captureIo
             ? traceLlmMessages([{ role: 'assistant', content: outputText }])
@@ -824,7 +870,7 @@ export function detectTeachProvider(): ProviderName {
 }
 
 function createProvider(name: ProviderName, opts: LLMOptions = {}): LLMProvider {
-  const model = opts.model ?? process.env.ANTHROPIC_MODEL ?? 'claude-opus-4-7';
+  const model = opts.model ?? process.env.ANTHROPIC_MODEL ?? 'claude-opus-4-8';
   const temperature = opts.temperature ?? 0;
   const maxTokens = opts.maxTokens ?? 8192;
 
@@ -865,11 +911,11 @@ export function preferredAgentModel(provider: ProviderName): string {
   switch (provider) {
     case 'anthropic-api':
     case 'claude-cli':
-      return 'claude-opus-4-7';
+      return 'claude-opus-4-8';
     case 'codex-cli':
       return 'gpt-5.5';
     case 'cursor-cli':
-      return 'claude-opus-4-7'; // best-effort; cursor passes through
+      return 'claude-opus-4-8'; // best-effort; cursor passes through
   }
 }
 
@@ -883,7 +929,8 @@ export function availableModelsForProvider(provider: ProviderName): ModelOption[
     case 'anthropic-api':
     case 'claude-cli':
       return [
-        { model: 'claude-opus-4-7', isDefault: true },
+        { model: 'claude-opus-4-8', isDefault: true },
+        { model: 'claude-opus-4-7', isDefault: false },
         { model: 'claude-sonnet-4-6', isDefault: false },
         { model: 'claude-haiku-4-5', isDefault: false },
         { model: 'claude-opus-4-6', isDefault: false },
@@ -908,7 +955,8 @@ export function availableModelsForProvider(provider: ProviderName): ModelOption[
       ];
     case 'cursor-cli':
       return [
-        { model: 'claude-opus-4-7', isDefault: true },
+        { model: 'claude-opus-4-8', isDefault: true },
+        { model: 'claude-opus-4-7', isDefault: false },
         { model: 'claude-sonnet-4-6', isDefault: false },
         { model: 'claude-haiku-4-5', isDefault: false },
         { model: 'gpt-5.5', isDefault: false },

package/src/imprint/mcp-compile-server.ts

@@ -24,7 +24,13 @@ import {
   ListToolsRequestSchema,
   type Tool,
 } from '@modelcontextprotocol/sdk/types.js';
-import { buildCompileTools, externalVerification } from './compile-tools.ts';
+import { type SharedModuleManifestEntry, resolvePlanSliceFromFile } from './build-plan.ts';
+import {
+  applyLiveVerification,
+  applyParamVerification,
+  buildCompileTools,
+  externalVerification,
+} from './compile-tools.ts';
 import { loadJsonFile } from './load-json.ts';
 import { createLog } from './log.ts';
 import { redactSession } from './redact.ts';
@@ -43,6 +49,10 @@ interface RunCompileMcpServerOptions {
   maxVerificationCycles?: number;
   candidate?: ToolCandidate;
   sharedContext?: SharedCompileContext;
+  /** Absolute path to the multi-tool build plan sidecar (.build-plan.json). */
+  buildPlanPath?: string;
+  /** Shared-module build manifest for this site (verified flags). */
+  sharedModules?: SharedModuleManifestEntry[];
 }
 
 const DONE_SENTINEL = '.compile-done.json';
@@ -67,12 +77,24 @@ export async function runCompileMcpServer(opts: RunCompileMcpServerOptions): Pro
     session = redactSession(session).session;
   }
 
-  // Build the 8 read/write tools (same as the in-process loop).
+  // Build the read/write tools (same as the in-process loop). When a build
+  // plan is present, buildCompileTools also exposes read_build_plan.
   const compileTools = buildCompileTools(session, opts.toolDir, opts.sessionPath, {
     candidate: opts.candidate,
     sharedContext: opts.sharedContext,
+    buildPlanPath: opts.buildPlanPath,
+    sharedModules: opts.sharedModules,
   });
 
+  // Resolve the shared modules + producer→consumer token contracts the plan
+  // assigned this tool, so verification can assert modules are imported and
+  // require a chained test for each producer-sourced token param.
+  const { assignedSharedModules, tokenParams, emittedTokens } = resolvePlanSliceFromFile(
+    opts.buildPlanPath,
+    opts.candidate?.toolName,
+    opts.sharedModules,
+  );
+
   // The custom done/give_up tools live alongside in MCP space.
   const doneTool: Tool = {
     name: 'done',
@@ -136,25 +158,36 @@ export async function runCompileMcpServer(opts: RunCompileMcpServerOptions): Pro
     if (name === 'done') {
       const summary = (args as { summary?: string }).summary ?? 'Task completed';
       log(`done() called: ${summary}`);
-      const { failures, warnings } = await externalVerification(
-        opts.toolDir,
-        session,
-        opts.sessionPath,
-        {
+      const { failures, warnings, paramVerification, liveVerification } =
+        await externalVerification(opts.toolDir, session, opts.sessionPath, {
           expectedToolName: opts.candidate?.toolName,
           likelyParams: opts.candidate?.likelyParams,
           candidateRequestSeqs: opts.candidate?.requestSeqs,
-        },
-      );
+          // Widen Fix B's variation pool to dependency requests so a token that
+          // varies only across them and is frozen as a literal in the tool's
+          // request is caught (the cross-request session-token leak case).
+          dependencyRequestSeqs: opts.candidate?.dependencySeqs,
+          assignedSharedModules,
+          tokenParams,
+          emittedTokens,
+        });
       if (warnings.length > 0) {
         log(`verification warnings (non-blocking):\n${warnings.join('\n')}`);
       }
       if (failures.length === 0) {
+        // Persist per-parameter verified flags + the live-verification stamp
+        // onto workflow.json. Audit and teach read the stamp.
+        applyLiveVerification(opts.toolDir, liveVerification);
+        const paramWarnings = applyParamVerification(opts.toolDir, paramVerification);
+        if (paramWarnings.length > 0) {
+          log(`parameter verification:\n${paramWarnings.join('\n')}`);
+        }
+        const allWarnings = [...warnings, ...paramWarnings];
         const sentinel = pathJoin(opts.toolDir, DONE_SENTINEL);
         writeFileSync(
           sentinel,
           JSON.stringify(
-            { summary, verification: 'passed', warnings, timestamp: Date.now() },
+            { summary, verification: 'passed', warnings: allWarnings, timestamp: Date.now() },
             null,
             2,
           ),