imprint-mcp 0.6.2 → 0.6.4

package/src/imprint/audit.ts

@@ -17,11 +17,16 @@ import { type ChildProcess, spawn } from 'node:child_process';
 import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { dirname, join as pathJoin } from 'node:path';
 import { z } from 'zod';
+import {
+  type AuthVerificationReceiptStatus,
+  readAuthVerificationReceiptStatus,
+} from './auth-compile-tools.ts';
 import { type ProviderName, preferredAgentModel } from './llm.ts';
 import { createLog } from './log.ts';
 import { imprintHomeDir } from './paths.ts';
-import { discoverTools } from './tool-loader.ts';
-import { llmSpanAttributes, setSpanAttributes, totalPromptTokens, traced } from './tracing.ts';
+import { type TeachState, loadTeachState } from './teach-state.ts';
+import { type ResolvedTool, discoverTools } from './tool-loader.ts';
+import { recordLlmUsageSpan, setSpanAttributes, totalPromptTokens, traced } from './tracing.ts';
 
 const log = createLog('audit');
 
@@ -110,6 +115,123 @@ interface AuditEvaluation {
   missingParameters: MissingAuditedParameter[];
 }
 
+interface SkippedInteractiveAuth {
+  name: string;
+  reason: string;
+  verificationStatus: AuthVerificationReceiptStatus['status'];
+  verificationReason: string;
+  verificationAction?: string;
+  verificationBackend?: string;
+  verificationTimestamp?: number;
+}
+
+interface AuditToolPartition {
+  eligible: ResolvedTool[];
+  nonInteractiveAuthNames: string[];
+  skippedInteractiveAuth: SkippedInteractiveAuth[];
+}
+
+interface DetectedButUnemittedTool {
+  name: string;
+  reason?: string;
+  lastCompletedStep?: string;
+}
+
+/** Report candidates from the latest retained recording that teach detected but
+ * did not emit. They are diagnostic metadata, not part of the audit score: the
+ * audit agent can only call emitted MCP tools. */
+export function detectedButUnemittedTools(
+  state: TeachState,
+  emittedToolNames: readonly string[],
+): DetectedButUnemittedTool[] {
+  const candidates = Object.values(state.workflows).filter((workflow) => workflow.candidate);
+  if (candidates.length === 0) return [];
+
+  const latest = candidates.reduce((current, workflow) =>
+    Date.parse(workflow.updatedAt) > Date.parse(current.updatedAt) ? workflow : current,
+  );
+  const emitted = new Set(emittedToolNames);
+  const byName = new Map<string, DetectedButUnemittedTool>();
+  for (const workflow of candidates) {
+    if (workflow.sessionPath !== latest.sessionPath || !workflow.candidate) continue;
+    const name = workflow.candidate.toolName;
+    // A same-named artifact may belong to an older successful teach. Only
+    // suppress the diagnostic when the latest recording itself reached emit
+    // successfully and the emitted artifact still exists.
+    if (emitted.has(name) && workflow.completedSteps.includes('emit') && !workflow.error) continue;
+    byName.set(name, {
+      name,
+      ...(workflow.error ? { reason: workflow.error.split('\n')[0] } : {}),
+      ...(workflow.completedSteps.at(-1)
+        ? { lastCompletedStep: workflow.completedSteps.at(-1) }
+        : {}),
+    });
+  }
+  return [...byName.values()].sort((a, b) => a.name.localeCompare(b.name));
+}
+
+export function countDetectedToolNames(
+  emittedToolNames: readonly string[],
+  detectedButUnemitted: readonly DetectedButUnemittedTool[],
+): number {
+  return new Set([...emittedToolNames, ...detectedButUnemitted.map((candidate) => candidate.name)])
+    .size;
+}
+
+/** Partition generated tools for a headless audit. Authentication is eligible
+ * only when every declared action completes directly: any pause means a human
+ * continuation (OTP, CAPTCHA, device approval, email link, etc.) may be needed. */
+export function partitionAuditTools(tools: readonly ResolvedTool[]): AuditToolPartition {
+  const eligible: ResolvedTool[] = [];
+  const nonInteractiveAuthNames: string[] = [];
+  const skippedInteractiveAuth: SkippedInteractiveAuth[] = [];
+
+  for (const tool of tools) {
+    // authConfig is the authoritative capability marker for legacy workflows
+    // where the newer optional toolKind field may be absent.
+    const auth = tool.workflow.authConfig;
+    if (tool.workflow.toolKind === 'authenticate' && !auth) {
+      const receipt = readAuthVerificationReceiptStatus(tool.dir, tool.workflow);
+      skippedInteractiveAuth.push({
+        name: tool.workflow.toolName,
+        reason: 'authentication configuration is missing, so unattended completion is not proven',
+        verificationStatus: receipt.status,
+        verificationReason: receipt.reason,
+        ...(receipt.action ? { verificationAction: receipt.action } : {}),
+        ...(receipt.backend ? { verificationBackend: receipt.backend } : {}),
+        ...(receipt.timestamp ? { verificationTimestamp: receipt.timestamp } : {}),
+      });
+      continue;
+    }
+    const pauses = auth
+      ? Object.entries(auth.actions).filter(([, action]) => action.outcome.type === 'pause')
+      : [];
+    if (pauses.length === 0) {
+      eligible.push(tool);
+      if (auth) nonInteractiveAuthNames.push(tool.workflow.toolName);
+      continue;
+    }
+
+    const receipt = readAuthVerificationReceiptStatus(tool.dir, tool.workflow);
+    const pauseSummary = pauses
+      .map(([name, action]) =>
+        action.outcome.type === 'pause' ? `${name}: ${action.outcome.message}` : name,
+      )
+      .join('; ');
+    skippedInteractiveAuth.push({
+      name: tool.workflow.toolName,
+      reason: `requires human continuation (${pauseSummary})`,
+      verificationStatus: receipt.status,
+      verificationReason: receipt.reason,
+      ...(receipt.action ? { verificationAction: receipt.action } : {}),
+      ...(receipt.backend ? { verificationBackend: receipt.backend } : {}),
+      ...(receipt.timestamp ? { verificationTimestamp: receipt.timestamp } : {}),
+    });
+  }
+
+  return { eligible, nonInteractiveAuthNames, skippedInteractiveAuth };
+}
+
 /**
  * Restrict an agent-authored report to the connected workflows before it is
  * scored. Agent output is not a trusted inventory: it can repeat a tool or a
@@ -405,6 +527,24 @@ export function buildAuditMcpEnvironment(assetRoot = imprintHomeDir()): Record<s
   };
 }
 
+export function buildAuditMcpCommandArgs(site: string, toolNames: readonly string[]): string[] {
+  return ['run', CLI_PATH, 'mcp-server', site, '--include-tools', toolNames.join(',')];
+}
+
+/** Normalize provider usage to OpenInference's total prompt-token contract. */
+export function auditPromptTokens(
+  provider: ProviderName,
+  inputTokens: number,
+  cacheReadTokens: number,
+  cacheWriteTokens: number,
+): number {
+  // Codex reports input_tokens as the total and cached_input_tokens as a subset.
+  // Anthropic reports input_tokens as the uncached delta, with cache buckets
+  // alongside it, so only Anthropic usage needs summing.
+  if (provider === 'codex-cli') return inputTokens;
+  return totalPromptTokens(inputTokens, cacheReadTokens, cacheWriteTokens) ?? inputTokens;
+}
+
 export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
   return await traced(
     'audit.session',
@@ -415,13 +555,27 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
     },
     async (span) => {
       const assetRoot = imprintHomeDir();
-      const tools = await discoverTools(assetRoot, opts.site, '[imprint audit]');
-      const toolCount = tools.length;
+      const discoveredTools = await discoverTools(assetRoot, opts.site, '[imprint audit]');
+      const toolCount = discoveredTools.length;
       if (toolCount === 0) {
         throw new Error(
           `No generated tool found for site "${opts.site}" — run \`imprint teach ${opts.site}\` first, then audit it.`,
         );
       }
+      const {
+        eligible: tools,
+        nonInteractiveAuthNames,
+        skippedInteractiveAuth,
+      } = partitionAuditTools(discoveredTools);
+      const auditedToolCount = tools.length;
+      const detectedButUnemitted = detectedButUnemittedTools(
+        loadTeachState(opts.site),
+        discoveredTools.map((tool) => tool.workflow.toolName),
+      );
+      const detectedToolCount = countDetectedToolNames(
+        discoveredTools.map((tool) => tool.workflow.toolName),
+        detectedButUnemitted,
+      );
 
       const provider = opts.provider ?? 'claude-cli';
       const model = opts.model ?? preferredAgentModel(provider);
@@ -433,18 +587,29 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
         );
       }
 
+      const nonInteractiveAuthSet = new Set(nonInteractiveAuthNames);
       const expectedTools = tools.map((tool) => ({
         name: tool.workflow.toolName,
-        parameters: tool.workflow.parameters.map((parameter) => parameter.name),
+        parameters: nonInteractiveAuthSet.has(tool.workflow.toolName)
+          ? []
+          : tool.workflow.parameters.map((parameter) => parameter.name),
       }));
       const toolNames = expectedTools.map((tool) => tool.name);
-      log(`auditing ${toolCount} tool(s) for site "${opts.site}": ${toolNames.join(', ')}`);
+      log(
+        `auditing ${auditedToolCount}/${toolCount} eligible tool(s) for site "${opts.site}": ${toolNames.join(', ') || '(none)'}`,
+      );
+      for (const skipped of skippedInteractiveAuth) {
+        log(
+          `skipping interactive auth ${skipped.name}: ${skipped.reason}; receipt=${skipped.verificationStatus}`,
+        );
+      }
 
       // Parameters that shipped live-unverified at compile time (Fix D). Tell the
       // auditor to probe them especially — these are the most likely to be broken
       // (the compile-time differential could not confirm their effect).
       const unverifiedParams: Array<{ tool: string; params: string[] }> = [];
       for (const t of tools) {
+        if (nonInteractiveAuthSet.has(t.workflow.toolName)) continue;
         const params = (t.workflow.parameters ?? [])
           .filter((p) => p.verified === false)
           .map((p) => p.name);
@@ -468,16 +633,45 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
         }
       }
 
-      const drive = await driveAudit({
-        site: opts.site,
-        provider,
-        model,
-        timeoutMs,
-        systemPromptPath,
-        toolNames,
-        unverifiedParams,
-        tokenDeps,
-      });
+      const drive =
+        auditedToolCount === 0
+          ? emptyDriveAuditResult()
+          : await driveAudit({
+              site: opts.site,
+              provider,
+              model,
+              timeoutMs,
+              systemPromptPath,
+              toolNames,
+              nonInteractiveAuthNames,
+              unverifiedParams,
+              tokenDeps,
+            });
+      if (auditedToolCount === 0) {
+        log('all discovered tools require human-interactive authentication; auditor not spawned');
+      }
+      recordLlmUsageSpan(
+        'audit.llm_usage',
+        {
+          provider,
+          model,
+          inputTokens: auditPromptTokens(
+            provider,
+            drive.inputTokens,
+            drive.cacheReadInputTokens,
+            drive.cacheCreationInputTokens,
+          ),
+          outputTokens: drive.outputTokens,
+          cacheReadTokens: drive.cacheReadInputTokens,
+          cacheWriteTokens: drive.cacheCreationInputTokens,
+        },
+        {
+          'imprint.audit.turns': drive.turns,
+          ...(drive.totalCostUsd != null
+            ? { 'imprint.audit.provider_reported_cost_usd': drive.totalCostUsd }
+            : {}),
+        },
+      );
 
       // Cross-reference compile-time live verification with the audit grade.
       // The downgrade rule's purpose is to surface "flying blind" runs —
@@ -537,15 +731,6 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
         }
       }
 
-      // TOTAL prompt (uncached + cache) for the cost calc; the cache split is
-      // passed to llmSpanAttributes separately. Always a number here
-      // (drive.inputTokens is non-null), so the cost-suppression happens via the
-      // `|| undefined` at the call site below.
-      const totalInputTokens = totalPromptTokens(
-        drive.inputTokens,
-        drive.cacheReadInputTokens,
-        drive.cacheCreationInputTokens,
-      );
       setSpanAttributes(span, {
         'imprint.audit.score': score.score,
         'imprint.audit.correct': score.correct,
@@ -558,6 +743,10 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
         'imprint.audit.params_broken': score.paramsBroken,
         'imprint.audit.params_untestable': score.paramsUntestable,
         'imprint.audit.tool_count': toolCount,
+        'imprint.audit.audited_tool_count': auditedToolCount,
+        'imprint.audit.detected_tool_count': detectedToolCount,
+        'imprint.audit.detected_but_unemitted_count': detectedButUnemitted.length,
+        'imprint.audit.skipped_interactive_auth_count': skippedInteractiveAuth.length,
         'imprint.audit.verdict': score.verdict,
         'imprint.audit.unverified_and_ungradeable_count': unverifiedAndUngradeable.length,
         'imprint.audit.missing_tool_count': missingToolNames.length,
@@ -565,16 +754,6 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
         'imprint.audit.timed_out': drive.timedOut,
         'imprint.audit.turns': drive.turns,
         ...(drive.totalCostUsd != null ? { 'imprint.audit.cost_usd': drive.totalCostUsd } : {}),
-        ...llmSpanAttributes({
-          provider,
-          model,
-          // `|| undefined`: when no usage was captured (e.g. spawn failure → 0
-          // tokens), suppress a bogus $0 cost instead of emitting it.
-          inputTokens: totalInputTokens || undefined,
-          outputTokens: drive.outputTokens || undefined,
-          cacheReadTokens: drive.cacheReadInputTokens || undefined,
-          cacheWriteTokens: drive.cacheCreationInputTokens || undefined,
-        }),
       });
 
       // Persist the full result (deterministic score + the raw model report).
@@ -583,6 +762,10 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
         report: drive.report,
         site: opts.site,
         toolCount,
+        auditedToolCount,
+        detectedToolCount,
+        detectedButUnemittedTools: detectedButUnemitted,
+        skippedInteractiveAuth,
         ungradeableTools: ungradeableNames,
         /** Connected tools without an invocation record. A partial audit
          *  cannot pass merely because the auditor named an uncalled tool. */
@@ -623,6 +806,10 @@ export async function runAudit(opts: RunAuditOptions): Promise<AuditScore> {
           missingTools: missingToolNames,
           missingParameters,
           report: drive.report,
+          auditedToolCount,
+          skippedInteractiveAuth,
+          detectedToolCount,
+          detectedButUnemittedTools: detectedButUnemitted,
         });
       }
 
@@ -661,6 +848,9 @@ interface DriveAuditOptions {
   timeoutMs: number;
   systemPromptPath: string;
   toolNames: string[];
+  /** Eligible password-only/direct-success auth tools. Interactive auth tools
+   * are filtered before the MCP session is created and never appear here. */
+  nonInteractiveAuthNames?: string[];
   /** Per-tool params that shipped live-unverified at compile time. */
   unverifiedParams: Array<{ tool: string; params: string[] }>;
   /** Producer→consumer token contracts (param.sourcedFrom) so the auditor chains. */
@@ -723,17 +913,18 @@ async function driveAuditWithClaude(opts: DriveAuditOptions): Promise<DriveAudit
   // with zero tools. The `imprint-audit-` prefix keeps the inline server unique.
   const serverName = `imprint-audit-${opts.site}`;
   const bunPath = process.execPath;
+  const mcpEnv = buildAuditMcpEnvironment();
   const mcpConfig = {
     mcpServers: {
       [serverName]: {
         command: bunPath,
-        args: ['run', CLI_PATH, 'mcp-server', opts.site],
+        args: buildAuditMcpCommandArgs(opts.site, opts.toolNames),
         // Pace every audit tool call: the auditor now differentially probes
         // bot-defended idempotent reads (search/calendar) instead of bailing
         // after one call, so a deliberate inter-call delay keeps the probing
         // steady enough that the per-IP anti-bot defense isn't tripped. Only
         // the audit sets this; production mcp-server runs unpaced.
-        env: { IMPRINT_AUDIT_PACING_MS: '5000', IMPRINT_AUDIT_TOOL_DEADLINE_MS: '120000' },
+        env: mcpEnv,
       },
     },
   };
@@ -829,7 +1020,7 @@ async function driveAuditWithClaude(opts: DriveAuditOptions): Promise<DriveAudit
 async function driveAuditWithCodex(opts: DriveAuditOptions): Promise<DriveAuditResult> {
   const serverName = `imprint-audit-${opts.site}`;
   const bunPath = process.execPath;
-  const mcpArgs = ['run', CLI_PATH, 'mcp-server', opts.site];
+  const mcpArgs = buildAuditMcpCommandArgs(opts.site, opts.toolNames);
   const mcpEnv = buildAuditMcpEnvironment();
 
   const unverifiedNote = buildUnverifiedParamNote(opts.unverifiedParams);
@@ -919,7 +1110,7 @@ ${buildAuditInitialPrompt(opts, unverifiedNote)}`;
     transcript: session.transcript,
     inputTokens: session.inputTokens,
     outputTokens: session.outputTokens,
-    cacheReadInputTokens: 0,
+    cacheReadInputTokens: session.cacheReadInputTokens,
     cacheCreationInputTokens: 0,
     totalCostUsd: null,
   };
@@ -937,6 +1128,10 @@ export function buildUnverifiedParamNote(
 }
 
 export function buildAuditInitialPrompt(opts: DriveAuditOptions, unverifiedNote: string): string {
+  const nonInteractiveAuthNames = opts.nonInteractiveAuthNames ?? [];
+  const authNote = nonInteractiveAuthNames.length
+    ? `\n\nThe following authentication tool(s) are eligible because every action completes without a human pause: ${nonInteractiveAuthNames.join(', ')}. Call each exactly once with one safe baseline. Never differentially probe or retry authentication actions. If any DATA tool returns AUTH_EXPIRED, classify that invocation as infra/inconclusive evidence; do not call an authentication tool in response.`
+    : '\n\nNo authentication tool is eligible in this audit. If a DATA tool returns AUTH_EXPIRED, classify that invocation as infra/inconclusive evidence; do not attempt to authenticate.';
   return `Audit every MCP tool connected to you for the site "${opts.site}".
 
 There are ${opts.toolNames.length} connected tool(s). For each one: read its description and input schema, invoke it with a realistic parameter set, judge the result, and classify each invocation as correct | tool_broken | infra | bad_params per your system prompt.
@@ -945,7 +1140,7 @@ Parameter coverage is mandatory for idempotent read tools. A cold cdp-replay/bro
 
 Promo/coupon/voucher/discount-code parameters require a known valid code to prove a visible effect. If neither the schema nor another tool output gives you a valid code, classify that parameter as untestable; do not invent a random code and call an unchanged response no_op.
 
-IMPORTANT: Call tools strictly sequentially — issue exactly one tool call, wait for its result, then issue the next. Never issue tool calls in parallel or batch them in one turn. Many target sites share an anti-bot defense across endpoints, so a parallel burst trips a site-wide rate-limit (HTTP 429) that then poisons every later call. If a call returns a 429 / rate-limit / anti-bot result, classify it \`infra\` and pause before the next call.${unverifiedNote}${buildTokenDepNote(opts.tokenDeps)}
+IMPORTANT: Call tools strictly sequentially — issue exactly one tool call, wait for its result, then issue the next. Never issue tool calls in parallel or batch them in one turn. Many target sites share an anti-bot defense across endpoints, so a parallel burst trips a site-wide rate-limit (HTTP 429) that then poisons every later call. If a call returns a 429 / rate-limit / anti-bot result, classify it \`infra\` and pause before the next call.${authNote}${unverifiedNote}${buildTokenDepNote(opts.tokenDeps)}
 
 When you are done, end your final message with exactly one fenced \`\`\`json block containing the full report and nothing after it.`;
 }
@@ -1130,6 +1325,7 @@ async function collectCodexAssistantText(
   let turns = 0;
   let inputTokens = 0;
   let outputTokens = 0;
+  let cacheReadInputTokens = 0;
   const t0 = Date.now();
   const elapsedStr = (): string => {
     const s = Math.floor((Date.now() - t0) / 1000);
@@ -1179,6 +1375,7 @@ async function collectCodexAssistantText(
       if (evt.type === 'turn.completed') {
         inputTokens += evt.usage?.input_tokens ?? 0;
         outputTokens += evt.usage?.output_tokens ?? 0;
+        cacheReadInputTokens += evt.usage?.cached_input_tokens ?? 0;
         log(`[${elapsedStr()}] turn ${turns} completed`);
         continue;
       }
@@ -1233,7 +1430,7 @@ async function collectCodexAssistantText(
     turns,
     inputTokens,
     outputTokens,
-    cacheReadInputTokens: 0,
+    cacheReadInputTokens,
     cacheCreationInputTokens: 0,
     totalCostUsd: null,
   };
@@ -1283,6 +1480,8 @@ interface CodexStreamJsonEvent {
   usage?: {
     input_tokens?: number;
     output_tokens?: number;
+    cached_input_tokens?: number;
+    reasoning_output_tokens?: number;
   };
   message?: string;
   error?: { message?: string };
@@ -1391,6 +1590,10 @@ function printSummary(
     missingTools: string[];
     missingParameters: MissingAuditedParameter[];
     report: AuditReport;
+    auditedToolCount: number;
+    skippedInteractiveAuth: SkippedInteractiveAuth[];
+    detectedToolCount: number;
+    detectedButUnemittedTools: DetectedButUnemittedTool[];
   },
 ): void {
   const pct = score.graded === 0 ? 'n/a' : `${score.score.toFixed(1)}%`;
@@ -1404,8 +1607,18 @@ function printSummary(
   const invGraded = score.graded - paramsTested;
   const invTotal = invGraded + score.infra + score.badParams;
   console.log(
-    `[imprint]   graded ${score.graded} unit(s) = ${invGraded}/${invTotal} invocation(s) + ${paramsTested} parameter(s) across ${toolCount} tool(s) — excluded: ${score.infra} infra, ${score.badParams} bad_params, ${score.paramsUntestable} untestable param(s)`,
+    `[imprint]   graded ${score.graded} unit(s) = ${invGraded}/${invTotal} invocation(s) + ${paramsTested} parameter(s) across ${extra.auditedToolCount}/${toolCount} eligible tool(s) — excluded: ${score.infra} infra, ${score.badParams} bad_params, ${score.paramsUntestable} untestable param(s)`,
   );
+  for (const skipped of extra.skippedInteractiveAuth) {
+    console.log(
+      `[imprint]   skipped interactive auth ${skipped.name} — ${skipped.reason}; verification receipt: ${skipped.verificationStatus}`,
+    );
+  }
+  if (extra.detectedButUnemittedTools.length > 0) {
+    console.log(
+      `[imprint]   ${extra.detectedButUnemittedTools.length}/${extra.detectedToolCount} detected tool(s) were not emitted and are excluded from scoring: ${extra.detectedButUnemittedTools.map((tool) => tool.name).join(', ')}`,
+    );
+  }
   if (paramsTested + score.paramsUntestable > 0) {
     console.log(
       `[imprint]   parameters: ${score.paramsWorking}/${paramsTested} working — ${score.paramsNoOp} no-op, ${score.paramsBroken} broken, ${score.paramsUntestable} untestable`,

package/src/imprint/auth-compile-tools.ts

@@ -362,30 +362,81 @@ export function authWorkflowHash(workflow: Workflow): string {
   return createHash('sha256').update(JSON.stringify(workflow)).digest('hex');
 }
 
-function liveAttemptFailures(toolDir: string, workflow: Workflow): string[] {
+export interface AuthVerificationReceiptStatus {
+  status: 'verified' | 'stale' | 'failed' | 'missing';
+  reason: string;
+  action?: string;
+  backend?: string;
+  timestamp?: number;
+}
+
+/** Read the latest auth verification receipt without initiating authentication.
+ * The workflow hash is authoritative: a successful receipt for older generated
+ * code is diagnostic history, not current verification evidence. */
+export function readAuthVerificationReceiptStatus(
+  toolDir: string,
+  workflow: Workflow,
+): AuthVerificationReceiptStatus {
   const path = pathJoin(toolDir, AUTH_VERIFICATION_ATTEMPT_SENTINEL);
   if (!existsSync(path)) {
-    return ['no live auth verification attempt was recorded; call run_verification before done'];
+    return {
+      status: 'missing',
+      reason: 'no live auth verification attempt was recorded',
+    };
   }
 
-  let attempt: { action?: unknown; ok?: unknown; workflowHash?: unknown };
+  let attempt: {
+    action?: unknown;
+    ok?: unknown;
+    workflowHash?: unknown;
+    backend?: unknown;
+    timestamp?: unknown;
+  };
   try {
     attempt = JSON.parse(readFileSync(path, 'utf8')) as typeof attempt;
   } catch (err) {
-    return [
-      `live auth verification record is invalid: ${err instanceof Error ? err.message : String(err)}`,
-    ];
+    return {
+      status: 'failed',
+      reason: `live auth verification record is invalid: ${err instanceof Error ? err.message : String(err)}`,
+    };
   }
+  const details = {
+    ...(typeof attempt.action === 'string' ? { action: attempt.action } : {}),
+    ...(typeof attempt.backend === 'string' ? { backend: attempt.backend } : {}),
+    ...(typeof attempt.timestamp === 'number' ? { timestamp: attempt.timestamp } : {}),
+  };
   if (attempt.ok !== true || typeof attempt.action !== 'string') {
-    return ['the most recent live auth verification did not succeed'];
+    return {
+      status: 'failed',
+      reason: 'the most recent live auth verification did not succeed',
+      ...details,
+    };
   }
   if (attempt.workflowHash !== authWorkflowHash(workflow)) {
-    return ['workflow.json changed after the most recent successful live auth verification'];
+    return {
+      status: 'stale',
+      reason: 'workflow.json changed after the most recent successful live auth verification',
+      ...details,
+    };
   }
   const action = workflow.authConfig?.actions[attempt.action];
-  return action?.outcome.type === 'success'
-    ? []
-    : [
-        `the most recent successful live action ${JSON.stringify(attempt.action)} does not declare a success outcome`,
-      ];
+  if (action?.outcome.type !== 'success') {
+    return {
+      status: 'failed',
+      reason: `the most recent successful live action ${JSON.stringify(attempt.action)} does not declare a success outcome`,
+      ...details,
+    };
+  }
+  return {
+    status: 'verified',
+    reason: `live auth verification succeeded with action ${JSON.stringify(attempt.action)}`,
+    ...details,
+  };
+}
+
+function liveAttemptFailures(toolDir: string, workflow: Workflow): string[] {
+  const receipt = readAuthVerificationReceiptStatus(toolDir, workflow);
+  if (receipt.status === 'verified') return [];
+  const suffix = receipt.status === 'missing' ? '; call run_verification before done' : '';
+  return [`${receipt.reason}${suffix}`];
 }

package/src/imprint/claude-cli-compile.ts

@@ -43,7 +43,7 @@ import { COMPILE_SENTINELS, compileDeadlineAfterVerification } from './mcp-compi
 import type { SharedCompileContext, ToolCandidate } from './tool-candidates.ts';
 import {
   endTraceSpan,
-  llmSpanAttributes,
+  recordLlmUsageSpan,
   setSpanAttributes,
   startTraceSpan,
   totalPromptTokens,
@@ -88,6 +88,28 @@ const USAGE_POLICY_REFUSAL =
 /** Total attempts (1 initial + retries) when a usage-policy refusal is hit. */
 const MAX_USAGE_POLICY_ATTEMPTS = 3;
 
+interface CompileUsageTotals {
+  turns: number;
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadInputTokens: number;
+  cacheCreationInputTokens: number;
+}
+
+/** Add one paid CLI attempt to the aggregate usage reported for the compile. */
+export function addCompileUsageTotals(
+  totals: CompileUsageTotals,
+  attempt: CompileUsageTotals,
+): CompileUsageTotals {
+  return {
+    turns: totals.turns + attempt.turns,
+    inputTokens: totals.inputTokens + attempt.inputTokens,
+    outputTokens: totals.outputTokens + attempt.outputTokens,
+    cacheReadInputTokens: totals.cacheReadInputTokens + attempt.cacheReadInputTokens,
+    cacheCreationInputTokens: totals.cacheCreationInputTokens + attempt.cacheCreationInputTokens,
+  };
+}
+
 /** Exponential backoff with jitter between refusal retries. Spacing matters:
  *  bursts of near-identical requests raise the safety-filter trip rate. */
 function usagePolicyBackoffMs(attempt: number): number {
@@ -212,6 +234,7 @@ export async function compileViaClaudeCli(
     },
     async (span) => {
       const result = await compileViaClaudeCliImpl(opts);
+      const model = opts.model ?? preferredAgentModel('claude-cli');
       setSpanAttributes(span, {
         'imprint.compile.outcome': result.outcome,
         'imprint.compile.turns': result.turns,
@@ -220,9 +243,12 @@ export async function compileViaClaudeCli(
         'imprint.compile.output_tokens': result.outputTokens,
         'imprint.compile.cache_read_input_tokens': result.cacheReadInputTokens,
         'imprint.compile.cache_creation_input_tokens': result.cacheCreationInputTokens,
-        ...llmSpanAttributes({
+      });
+      recordLlmUsageSpan(
+        'compile.claude_cli_usage',
+        {
           provider: 'claude-cli',
-          model: opts.model ?? preferredAgentModel('claude-cli'),
+          model,
           // TOTAL prompt (uncached + cache); the cache split is passed separately
           // for cost. `result.inputTokens` alone is the uncached delta (often a
           // few hundred), which would mislabel `llm.token_count.prompt`.
@@ -234,8 +260,9 @@ export async function compileViaClaudeCli(
           outputTokens: result.outputTokens,
           cacheReadTokens: result.cacheReadInputTokens,
           cacheWriteTokens: result.cacheCreationInputTokens,
-        }),
-      });
+        },
+        { 'imprint.compile.turns': result.turns },
+      );
       return result;
     },
   );
@@ -251,11 +278,20 @@ async function compileViaClaudeCliImpl(
   opts: CompileViaClaudeCliOptions,
 ): Promise<CompileAgentResult> {
   let lastResult: CompileAgentResult | undefined;
+  let usageTotals: CompileUsageTotals = {
+    turns: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadInputTokens: 0,
+    cacheCreationInputTokens: 0,
+  };
   for (let attempt = 1; attempt <= MAX_USAGE_POLICY_ATTEMPTS; attempt++) {
     const result = await runClaudeCliAttempt(opts);
+    usageTotals = addCompileUsageTotals(usageTotals, result);
+    const resultWithAggregateUsage = { ...result, ...usageTotals };
     const isRefusal = !result.success && USAGE_POLICY_REFUSAL.test(result.message ?? '');
-    if (!isRefusal) return result;
-    lastResult = result;
+    if (!isRefusal) return resultWithAggregateUsage;
+    lastResult = resultWithAggregateUsage;
     if (attempt < MAX_USAGE_POLICY_ATTEMPTS) {
       const backoffMs = usagePolicyBackoffMs(attempt);
       log(