@ngockhoale/ukit 3.4.47 → 3.4.49

package/scripts/bench/data-foundation.mjs

@@ -1,929 +0,0 @@
-#!/usr/bin/env node
-// data-foundation bench harness (C54 / DF-00)
-//
-//   --fixture    Regenerate tests/fixtures/observability/corpus.json deterministically (seed=42).
-//   --recorder   Emit the corpus through createRecorder into a temp segment store.
-//   --analytics  Rebuild the derived index from retained segments and print summaries.
-//   --evaluate   Replay the golden corpus (tests/fixtures/observability/golden/cases.json)
-//                under both pre-registered policies and print paired scorecards +
-//                a fixed-seed variance report (TASK-014).
-//   --perturb    Inject seeded synthetic mutations into corpus summaries and measure
-//                detection through computeFingerprint/detectOpportunities (TASK-002).
-//   --anomalies  Replay the golden cases, summarize each run, and report anomaly
-//                detection counts per case (TASK-015). Degrades honestly to
-//                `detector: 'unavailable'` while analytics/anomalies.js
-//                (TASK-011) does not exist — never a stub detector.
-//
-// Flags run in argument order; the exit code is the first non-zero code.
-// --evaluate exits 1 by design while the corpus carries expected-negative
-// strata (a fail verdict is the honest scorecard), so a combined invocation
-// ending in --evaluate exits 1.
-//
-// The corpus is a synthetic, seeded fixture: no real user content. Records are
-// envelope-shaped per SPEC §7 (DF-FR01) but intentionally NOT schema-validated —
-// TASK-003 owns validation. Unknown host measurements are marked UNKNOWN, never 0.
-
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { SEMANTIC_REGISTRY } from '../../src/core/observability/schema/registry.js';
-import { PRIVACY_CLASSES } from '../../src/core/observability/schema/constants.js';
-
-const SEED = 42;
-const METRIC_VERSION = 'df-m1';
-const SCHEMA_VERSION = 1;
-const BOOT_ID = 'boot-fixture-0001';
-const WRITER_ID = 'writer-fixture-0001';
-const SESSION_ID = 'session-fixture-0001';
-const PROJECT_REF = 'project-fixture-0001';
-const WALL_EPOCH_MS = Date.UTC(2026, 0, 1, 0, 0, 0); // fixed epoch → deterministic wall_time_utc
-
-const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
-const corpusPath = path.join(repoRoot, 'tests/fixtures/observability/corpus.json');
-
-// Records may raise privacy above the registry floor but never lower it —
-// validateSemanticRecord (TASK-003) rejects below-floor classes, so the
-// fixture clamps each record's declared class up to the registry floor.
-function clampToRegistryFloor(semanticName, privacyClass) {
-  const floor = SEMANTIC_REGISTRY[semanticName]?.privacy_class;
-  if (!floor) return privacyClass;
-  return PRIVACY_CLASSES.indexOf(privacyClass) < PRIVACY_CLASSES.indexOf(floor) ? floor : privacyClass;
-}
-
-// Deterministic PRNG (mulberry32) — used only for token counts; IDs are counters.
-function mulberry32(seed) {
-  let a = seed >>> 0;
-  return function next() {
-    a |= 0;
-    a = (a + 0x6d2b79f5) | 0;
-    let t = Math.imul(a ^ (a >>> 15), 1 | a);
-    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
-    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-  };
-}
-
-function makeCtx() {
-  const rand = mulberry32(SEED);
-  let seq = 0;
-  let rec = 0;
-  let span = 0;
-  let trace = 0;
-  let clockMs = 0;
-  return {
-    nextSeq: () => ++seq,
-    recordId: () => `rec-${String(++rec).padStart(4, '0')}`,
-    spanId: () => `span-${String(++span).padStart(4, '0')}`,
-    traceId: () => `trace-${String(++trace).padStart(4, '0')}`,
-    // advance the deterministic clock by `ms` and return { wall_time_utc, monotonic_ns }
-    tick: (ms) => {
-      clockMs += ms;
-      return {
-        wall_time_utc: new Date(WALL_EPOCH_MS + clockMs).toISOString(),
-        monotonic_ns: clockMs * 1e6,
-      };
-      },
-    tokens: (lo, hi) => lo + Math.floor(rand() * (hi - lo + 1)),
-  };
-}
-function record(ctx, { semanticName, traceId, spanId, parentSpanId, executionId, agentId, importance, privacyClass, payload, dtMs }) {
-  const t = ctx.tick(dtMs);
-  const rec = {
-    record_type: 'fact',
-    semantic_name: semanticName,
-    schema_version: SCHEMA_VERSION,
-    record_id: ctx.recordId(),
-    trace_id: traceId,
-    span_id: spanId,
-    parent_span_id: parentSpanId,
-    execution_id: executionId,
-    session_id: SESSION_ID,
-    project_ref: PROJECT_REF,
-    boot_id: BOOT_ID,
-    writer_id: WRITER_ID,
-    sequence: ctx.nextSeq(),
-    wall_time_utc: t.wall_time_utc,
-    monotonic_ns: t.monotonic_ns,
-    importance,
-    privacy_class: clampToRegistryFloor(semanticName, privacyClass),
-    payload,
-  };
-  // DF2-FR01 v1.1: agent_id is the optional multi-agent correlation field —
-  // emitted only when the scenario declares one.
-  if (typeof agentId === 'string' && agentId.length > 0) rec.agent_id = agentId;
-  return rec;
-}
-
-function execSpan(ctx, traceId, executionId, privacyClass, agentId) {
-  const spanId = ctx.spanId();
-  return {
-    spanId,
-    started: record(ctx, {
-      semanticName: 'execution.started',
-      traceId,
-      spanId,
-      parentSpanId: null,
-      executionId,
-      agentId,
-      importance: 'normal',
-      privacyClass,
-      payload: { operation: 'task', duration_ms: null },
-      dtMs: 10,
-    }),
-  };
-}
-
-function modelAttempt(ctx, { traceId, parentSpanId, executionId, agentId, attemptIndex, status, resource, privacyClass, extra = {} }) {
-  const spanId = ctx.spanId();
-  const started = record(ctx, {
-    semanticName: 'model.started',
-    traceId,
-    spanId,
-    parentSpanId,
-    executionId,
-    agentId,
-    importance: 'normal',
-    privacyClass,
-    payload: { attempt_index: attemptIndex, ...extra },
-    dtMs: 5,
-  });
-  const completed = record(ctx, {
-    semanticName: status === 'ok' ? 'model.completed' : 'model.failed',
-    traceId,
-    spanId,
-    parentSpanId,
-    executionId,
-    agentId,
-    importance: 'normal',
-    privacyClass,
-    payload: {
-      attempt_index: attemptIndex,
-      duration_ms: 120 + attemptIndex * 30,
-      resource,
-      ...(status === 'ok' ? {} : { error_code: extra.error_code || 'MODEL_ERROR' }),
-      ...extra,
-    },
-    dtMs: 120 + attemptIndex * 30,
-  });
-  return [started, completed];
-}
-function execEnd(ctx, { traceId, spanId, executionId, agentId, status, privacyClass, extra = {} }) {
-  return record(ctx, {
-    semanticName: `execution.${status}`,
-    traceId,
-    spanId,
-    parentSpanId: null,
-    executionId,
-    agentId,
-    importance: 'normal',
-    privacyClass,
-    payload: { duration_ms: 0, ...extra },
-    dtMs: 10,
-  });
-}
-
-function sumUsage(attempts) {
-  const values = attempts
-    .map((r) => r.payload.resource)
-    .filter((res) => res && typeof res.value === 'number')
-    .map((res) => res.value);
-  return values.length ? values.reduce((a, b) => a + b, 0) : null;
-}
-function buildTrace(ctx, { scenario, privacyClass, agentId, build }) {
-  const traceId = ctx.traceId();
-  const executionId = `exec-${traceId}`;
-  const exec = execSpan(ctx, traceId, executionId, privacyClass, agentId);
-  const { records, attempts, succeeded } = build({ traceId, executionId, execSpanId: exec.spanId });
-  const all = [exec.started, ...records];
-  return {
-    trace_id: traceId,
-    scenario,
-    privacy_class: privacyClass,
-    records: all,
-    metrics: {
-      metric_version: METRIC_VERSION,
-      tokens_to_success: succeeded ? sumUsage(attempts) : null,
-      correction_observed: scenario === 'retry',
-    },
-  };
-}
-
-function buildCorpus() {
-  const ctx = makeCtx();
-  const traces = [];
-
-  // success: single attempt, provider-reported usage.
-  traces.push(
-    buildTrace(ctx, {
-      scenario: 'success',
-      privacyClass: 'public',
-      build: ({ traceId, executionId, execSpanId }) => {
-        const resource = { value: ctx.tokens(800, 1200), source: 'PROVIDER' };
-        const attempts = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          attemptIndex: 1, status: 'ok', resource, privacyClass: 'public',
-        });
-        const end = execEnd(ctx, {
-          traceId, spanId: execSpanId, executionId, status: 'completed',
-          privacyClass: 'public', extra: { outcome: 'success' },
-        });
-        return { records: [...attempts, end], attempts, succeeded: true };
-      },
-    }),
-  );
-
-  // retry: user correction observed → second attempt; tokens_to_success sums both.
-  traces.push(
-    buildTrace(ctx, {
-      scenario: 'retry',
-      privacyClass: 'public',
-      build: ({ traceId, executionId, execSpanId }) => {
-        const res1 = { value: ctx.tokens(400, 700), source: 'PROVIDER' };
-        const res2 = { value: ctx.tokens(500, 900), source: 'PROVIDER' };
-        const a1 = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          attemptIndex: 1, status: 'ok', resource: res1, privacyClass: 'public',
-        });
-        const correction = record(ctx, {
-          semanticName: 'context.item.injected',
-          traceId, spanId: ctx.spanId(), parentSpanId: execSpanId, executionId,
-          importance: 'high', privacyClass: 'public',
-          payload: { kind: 'user_correction', correction_observed: true },
-          dtMs: 15,
-        });
-        const a2 = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          attemptIndex: 2, status: 'ok', resource: res2, privacyClass: 'public',
-        });
-        const end = execEnd(ctx, {
-          traceId, spanId: execSpanId, executionId, status: 'completed',
-          privacyClass: 'public', extra: { outcome: 'success', attempts: 2 },
-        });
-        return { records: [...a1, correction, ...a2, end], attempts: [...a1, ...a2], succeeded: true };
-      },
-    }),
-  );
-
-  // failure: host-blind — usage UNKNOWN (never 0), execution failed.
-  traces.push(
-    buildTrace(ctx, {
-      scenario: 'failure',
-      privacyClass: 'public',
-      build: ({ traceId, executionId, execSpanId }) => {
-        const resource = { value: null, source: 'UNKNOWN' };
-        const attempts = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          attemptIndex: 1, status: 'error', resource, privacyClass: 'public',
-          extra: { error_code: 'TOOL_TIMEOUT' },
-        });
-        const end = execEnd(ctx, {
-          traceId, spanId: execSpanId, executionId, status: 'failed',
-          privacyClass: 'public', extra: { outcome: 'failure', error_code: 'TOOL_TIMEOUT' },
-        });
-        return { records: [...attempts, end], attempts, succeeded: false };
-      },
-    }),
-  );
-
-  // dropped: queue overflow — telemetry.dropped counted, telemetry_complete=false.
-  traces.push(
-    buildTrace(ctx, {
-      scenario: 'dropped',
-      privacyClass: 'public',
-      build: ({ traceId, executionId, execSpanId }) => {
-        const resource = { value: ctx.tokens(300, 600), source: 'PROVIDER' };
-        const attempts = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          attemptIndex: 1, status: 'ok', resource, privacyClass: 'public',
-        });
-        const dropped = record(ctx, {
-          semanticName: 'telemetry.dropped',
-          traceId, spanId: ctx.spanId(), parentSpanId: execSpanId, executionId,
-          importance: 'high', privacyClass: 'public',
-          payload: { reason_code: 'QUEUE_OVERFLOW', dropped_count: 3, telemetry_complete: false },
-          dtMs: 5,
-        });
-        const end = execEnd(ctx, {
-          traceId, spanId: execSpanId, executionId, status: 'completed',
-          privacyClass: 'public', extra: { outcome: 'success', telemetry_complete: false },
-        });
-        return { records: [...attempts, dropped, end], attempts, succeeded: true };
-      },
-    }),
-  );
-
-  // secret: payload carries a synthetic fixture secret (never a real credential);
-  // host-blind usage → UNKNOWN. TASK-004 leak tests scan for the marker string.
-  traces.push(
-    buildTrace(ctx, {
-      scenario: 'secret',
-      privacyClass: 'sensitive',
-      build: ({ traceId, executionId, execSpanId }) => {
-        const resource = { value: null, source: 'UNKNOWN' };
-        const attempts = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          attemptIndex: 1, status: 'error', resource, privacyClass: 'sensitive',
-          extra: {
-            error_code: 'AUTH_FAILED',
-            error_detail: 'credential rejected: FIXTURE_SECRET_DO_NOT_LEAK_7f3a9c',
-          },
-        });
-        const end = execEnd(ctx, {
-          traceId, spanId: execSpanId, executionId, status: 'failed',
-          privacyClass: 'sensitive', extra: { outcome: 'failure', error_code: 'AUTH_FAILED' },
-        });
-        return { records: [...attempts, end], attempts, succeeded: false };
-      },
-    }),
-  );
-
-  // memory_outcome: retrieval.query → memory.retrieved hits → model →
-  // memory.write → execution.completed → outcome.observed — metadata-only
-  // payloads (counts, kinds, opaque refs), never content.
-  traces.push(
-    buildTrace(ctx, {
-      scenario: 'memory_outcome',
-      privacyClass: 'sensitive',
-      build: ({ traceId, executionId, execSpanId }) => {
-        const query = record(ctx, {
-          semanticName: 'retrieval.query',
-          traceId, spanId: ctx.spanId(), parentSpanId: execSpanId, executionId,
-          importance: 'normal', privacyClass: 'internal',
-          payload: {
-            lane: 'v2', scope: 'all', indexed: true,
-            candidate_count: ctx.tokens(10, 40), returned_count: 3,
-            duration_ms: ctx.tokens(5, 20), freshness: {},
-          },
-          dtMs: 8,
-        });
-        const retrieved = [
-          { kind: 'fact' }, { kind: 'decision' }, { kind: 'decision' },
-        ].map((hit) => record(ctx, {
-          semanticName: 'memory.retrieved',
-          traceId, spanId: ctx.spanId(), parentSpanId: execSpanId, executionId,
-          importance: 'normal', privacyClass: 'sensitive',
-          payload: { kind: hit.kind, record_ref: `memref-${ctx.tokens(1000, 9999)}` },
-          dtMs: 4,
-        }));
-        const resource = { value: ctx.tokens(800, 1200), source: 'PROVIDER' };
-        const attempts = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          attemptIndex: 1, status: 'ok', resource, privacyClass: 'public',
-        });
-        const write = record(ctx, {
-          semanticName: 'memory.write',
-          traceId, spanId: ctx.spanId(), parentSpanId: execSpanId, executionId,
-          importance: 'normal', privacyClass: 'sensitive',
-          payload: {
-            op: 'write', record_count: ctx.tokens(1, 5), changed_count: 1,
-            kinds: { fact: 1 }, record_refs: [`memref-${ctx.tokens(1000, 9999)}`],
-            tombstone_count: 0,
-          },
-          dtMs: 6,
-        });
-        const end = execEnd(ctx, {
-          traceId, spanId: execSpanId, executionId, status: 'completed',
-          privacyClass: 'public', extra: { outcome: 'success' },
-        });
-        const outcome = record(ctx, {
-          semanticName: 'outcome.observed',
-          traceId, spanId: ctx.spanId(), parentSpanId: execSpanId, executionId,
-          importance: 'high', privacyClass: 'internal',
-          payload: {
-            kind: 'feedback', verdict: 'accepted', source: 'user',
-            evidence_refs: [end.record_id],
-          },
-          dtMs: 30,
-        });
-        return {
-          records: [query, ...retrieved, ...attempts, write, end, outcome],
-          attempts,
-          succeeded: true,
-        };
-      },
-    }),
-  );
-
-  // multi_agent: one trace spanning two agents — orchestrator runs the
-  // retrieval query and owns the execution span, worker runs the model
-  // attempt. agent_id is the v1.1 correlation field; trace_id stays single.
-  traces.push(
-    buildTrace(ctx, {
-      scenario: 'multi_agent',
-      privacyClass: 'public',
-      agentId: 'agent-orchestrator',
-      build: ({ traceId, executionId, execSpanId }) => {
-        const query = record(ctx, {
-          semanticName: 'retrieval.query',
-          traceId, spanId: ctx.spanId(), parentSpanId: execSpanId, executionId,
-          agentId: 'agent-orchestrator',
-          importance: 'normal', privacyClass: 'internal',
-          payload: {
-            lane: 'v2', scope: 'all', indexed: true,
-            candidate_count: ctx.tokens(10, 40), returned_count: 2,
-            duration_ms: ctx.tokens(5, 20), freshness: {},
-          },
-          dtMs: 8,
-        });
-        const resource = { value: ctx.tokens(800, 1200), source: 'PROVIDER' };
-        const attempts = modelAttempt(ctx, {
-          traceId, parentSpanId: execSpanId, executionId,
-          agentId: 'agent-worker',
-          attemptIndex: 1, status: 'ok', resource, privacyClass: 'public',
-        });
-        const end = execEnd(ctx, {
-          traceId, spanId: execSpanId, executionId, agentId: 'agent-orchestrator',
-          status: 'completed', privacyClass: 'public', extra: { outcome: 'success' },
-        });
-        return { records: [query, ...attempts, end], attempts, succeeded: true };
-      },
-    }),
-  );
-
-  return {
-    metric_version: METRIC_VERSION,
-    seed: SEED,
-    generated_by: 'scripts/bench/data-foundation.mjs --fixture',
-    metric_definitions: {
-      tokens_to_success: {
-        metric_version: METRIC_VERSION,
-        unit: 'tokens',
-        definition:
-          'Sum of numeric resource.value across all model attempts in the episode; null when no attempt reported a numeric value or the episode did not succeed.',
-      },
-      correction_observed: {
-        metric_version: METRIC_VERSION,
-        unit: 'boolean',
-        definition:
-          'True when a user correction was injected mid-episode and the episode required ≥2 model attempts.',
-      },
-    },
-    traces,
-  };
-}
-
-function writeFixture() {
-  const corpus = buildCorpus();
-  fs.mkdirSync(path.dirname(corpusPath), { recursive: true });
-  fs.writeFileSync(corpusPath, `${JSON.stringify(corpus, null, 2)}\n`);
-  process.stdout.write(`wrote ${path.relative(repoRoot, corpusPath)} (seed=${SEED}, traces=${corpus.traces.length})\n`);
-  return 0;
-}
-
-function percentile(sorted, p) {
-  if (sorted.length === 0) return 0;
-  const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
-  return sorted[Math.max(0, idx)];
-}
-
-// --recorder: emit the whole corpus through createRecorder into a temp
-// segment root, measure per-emit latency distribution + flush outcome, then
-// read the store back to prove the round-trip. Exit 0 on success, 1 when the
-// pipeline loses or corrupts records.
-async function runRecorderBench() {
-  const { createRecorder } = await import('../../src/core/observability/emit/recorder.js');
-  const { readSegments } = await import('../../src/core/observability/segments/readSegments.js');
-
-  const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
-  const inputs = corpus.traces.flatMap((t) => t.records);
-
-  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ukit-bench-rec-'));
-  try {
-    const recorder = createRecorder({
-      root,
-      config: { observability: { stage: 'default' } },
-    });
-
-    const latenciesMs = [];
-    let accepted = 0;
-    let dropped = 0;
-    for (const record of inputs) {
-      const t0 = process.hrtime.bigint();
-      const r = recorder.emit(record);
-      latenciesMs.push(Number(process.hrtime.bigint() - t0) / 1e6);
-      if (r.status === 'accepted') accepted += 1;
-      else dropped += 1;
-    }
-
-    const flushStart = process.hrtime.bigint();
-    const flush = await recorder.flush({ deadlineMs: 60_000 });
-    const flushMs = Number(process.hrtime.bigint() - flushStart) / 1e6;
-
-    let readBack = 0;
-    const it = readSegments(root);
-    for await (const _record of it) readBack += 1;
-
-    latenciesMs.sort((a, b) => a - b);
-    const summary = {
-      metric_version: METRIC_VERSION,
-      bench: 'recorder',
-      records: inputs.length,
-      accepted,
-      dropped,
-      written: flush.written,
-      read_back: readBack,
-      flush_status: flush.status,
-      flush_ms: Number(flushMs.toFixed(3)),
-      emit_latency_ms: {
-        p50: Number(percentile(latenciesMs, 50).toFixed(4)),
-        p95: Number(percentile(latenciesMs, 95).toFixed(4)),
-        p99: Number(percentile(latenciesMs, 99).toFixed(4)),
-        max: Number((latenciesMs[latenciesMs.length - 1] || 0).toFixed(4)),
-      },
-      health: recorder.health(),
-    };
-    process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
-    const okRun = flush.status === 'ok' && readBack === accepted && dropped === 0;
-    return okRun ? 0 : 1;
-  } finally {
-    fs.rmSync(root, { recursive: true, force: true });
-  }
-}
-
-// --analytics: emit the corpus through createRecorder into a temp segment
-// root, rebuild the derived index from retained facts, and print the
-// per-trace summaries + coverage denominators. Exit 0 when the rebuild is
-// complete and equivalent to direct summarization of the corpus records.
-async function runAnalyticsBench() {
-  const { createRecorder } = await import('../../src/core/observability/emit/recorder.js');
-  const { rebuildIndex } = await import('../../src/core/observability/analytics/rebuild.js');
-  const { summarizeTrace } = await import('../../src/core/observability/analytics/summary.js');
-
-  const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
-  const inputs = corpus.traces.flatMap((t) => t.records);
-
-  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ukit-bench-analytics-'));
-  try {
-    const recorder = createRecorder({
-      root,
-      config: { observability: { stage: 'default' } },
-    });
-    for (const record of inputs) recorder.emit(record);
-    const flush = await recorder.flush({ deadlineMs: 60_000 });
-
-    const started = process.hrtime.bigint();
-    const index = await rebuildIndex(root);
-    const rebuildMs = Number(process.hrtime.bigint() - started) / 1e6;
-
-    // Equivalence: rebuild-from-retained-facts must reproduce the same
-    // summaries as direct analysis of the corpus records (sanitization only
-    // rewrites sensitive free-text, never metric fields).
-    const direct = corpus.traces
-      .map((t) => summarizeTrace(t.records))
-      .sort((a, b) => a.trace_id.localeCompare(b.trace_id));
-    const equivalent = JSON.stringify(index.summaries) === JSON.stringify(direct);
-
-    const report = {
-      metric_version: METRIC_VERSION,
-      bench: 'analytics',
-      records: inputs.length,
-      flush_status: flush.status,
-      rebuild_ms: Number(rebuildMs.toFixed(3)),
-      equivalent_to_direct: equivalent,
-      coverage: index.coverage,
-      summaries: index.summaries,
-    };
-    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
-    const okRun = flush.status === 'ok' && index.ok && equivalent
-      && index.summaries.length === corpus.traces.length;
-    return okRun ? 0 : 1;
-  } finally {
-    fs.rmSync(root, { recursive: true, force: true });
-  }
-}
-
-// --evaluate: replay the golden case set under every pre-registered policy,
-// print each Scorecard plus the paired comparison (fixed-seed bootstrap
-// variance). Exit 0 when no cohort verdict is 'fail'; 1 on fail verdict or
-// an unloadable corpus. Evaluation is offline — no model calls, no writes.
-async function runEvaluateBench() {
-  const { evaluateVariant, compareScorecards } = await import(
-    '../../src/core/observability/evaluation/scorecard.js'
-  );
-
-  const goldenPath = path.join(repoRoot, 'tests/fixtures/observability/golden/cases.json');
-  let corpus;
-  try {
-    corpus = JSON.parse(fs.readFileSync(goldenPath, 'utf8'));
-  } catch (err) {
-    process.stderr.write(`--evaluate: cannot load ${goldenPath}: ${err && err.message}\n`);
-    return 1;
-  }
-
-  const policies = Object.keys(
-    (corpus.pre_registered && corpus.pre_registered.policies) || {},
-  ).sort();
-  if (policies.length === 0) {
-    process.stderr.write('--evaluate: corpus declares no pre_registered.policies\n');
-    return 1;
-  }
-
-  const scorecards = {};
-  for (const policyId of policies) {
-    scorecards[policyId] = evaluateVariant(corpus, policyId);
-  }
-  const comparison = policies.length >= 2
-    ? compareScorecards(scorecards[policies[0]], scorecards[policies[1]], { seed: SEED })
-    : null;
-
-  const report = {
-    metric_version: METRIC_VERSION,
-    bench: 'evaluate',
-    seed: SEED,
-    corpus: path.relative(repoRoot, goldenPath),
-    policies,
-    scorecards,
-    comparison,
-  };
-  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
-  const failed = Object.values(scorecards).some((sc) => sc.verdict === 'fail');
-  return failed ? 1 : 0;
-}
-
-// --perturb: summarize the corpus, inject seeded synthetic mutations via
-// mutateSummaries, then measure detection through the real
-// computeFingerprint/detectOpportunities path. Exit 0 when every injected
-// mutation is detected; 1 when missed[] is non-empty or the corpus is
-// unloadable. Deterministic at SEED; evaluation is offline — no model calls.
-export async function runPerturb({ quiet = false } = {}) {
-  const { summarizeTrace } = await import('../../src/core/observability/analytics/summary.js');
-  const { mutateSummaries, detectPerturbations } = await import(
-    '../../src/core/observability/evaluation/perturbation.js'
-  );
-
-  let corpus;
-  try {
-    corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
-  } catch (err) {
-    process.stderr.write(`--perturb: cannot load ${corpusPath}: ${err && err.message}\n`);
-    return { code: 1, report: null };
-  }
-
-  const baseline = corpus.traces.map((t) => summarizeTrace(t.records));
-  const { summaries: mutated, manifest } = mutateSummaries(baseline, { seed: SEED });
-  const outcome = detectPerturbations(mutated, baseline, manifest);
-
-  const report = {
-    metric_version: METRIC_VERSION,
-    bench: 'perturb',
-    seed: SEED,
-    corpus: path.relative(repoRoot, corpusPath),
-    injected: outcome.injected,
-    detected: outcome.detected,
-    missed: outcome.missed,
-    manifest,
-  };
-  if (!quiet) process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
-  return { code: outcome.missed.length > 0 ? 1 : 0, report };
-}
-
-// --anomalies: replay every golden case under each pre-registered policy,
-// summarize each materialized run, and report per-case anomaly detection
-// counts keyed by detector kind. The detector is imported defensively:
-// while src/core/observability/analytics/anomalies.js (TASK-011) does not
-// exist the report degrades to `detector: 'unavailable'` — never a stub.
-// When a detector IS available, per-case declarations in
-// `expect.anomalies.expected_kinds` act as a regression gate: unmet or
-// over-fired kinds exit 1. Evaluation is offline — no model calls.
-async function loadAnomalyDetector() {
-  try {
-    const mod = await import('../../src/core/observability/analytics/anomalies.js');
-    return typeof mod.detectAnomalies === 'function' ? mod.detectAnomalies : null;
-  } catch {
-    return null;
-  }
-}
-
-function anomalyKind(record) {
-  const kind = record && record.payload && record.payload.kind !== undefined
-    ? record.payload.kind
-    : record && record.kind;
-  return typeof kind === 'string' && kind.length > 0 ? kind : 'unknown';
-}
-
-async function runAnomaliesBench() {
-  const { replayCase } = await import('../../src/core/observability/evaluation/replay.js');
-  const { summarizeTrace } = await import('../../src/core/observability/analytics/summary.js');
-  const detectAnomalies = await loadAnomalyDetector();
-
-  const goldenPath = path.join(repoRoot, 'tests/fixtures/observability/golden/cases.json');
-  let corpus;
-  try {
-    corpus = JSON.parse(fs.readFileSync(goldenPath, 'utf8'));
-  } catch (err) {
-    process.stderr.write(`--anomalies: cannot load ${goldenPath}: ${err && err.message}\n`);
-    return 1;
-  }
-  const policies = Object.keys(
-    (corpus.pre_registered && corpus.pre_registered.policies) || {},
-  ).sort();
-  if (policies.length === 0) {
-    process.stderr.write('--anomalies: corpus declares no pre_registered.policies\n');
-    return 1;
-  }
-
-  const runs = [];
-  for (const goldenCase of corpus.cases || []) {
-    const expectedKinds = goldenCase && goldenCase.expect && goldenCase.expect.anomalies
-      && Array.isArray(goldenCase.expect.anomalies.expected_kinds)
-      ? goldenCase.expect.anomalies.expected_kinds
-      : null;
-    const run = { case_id: goldenCase && goldenCase.case_id, expected_kinds: expectedKinds, policies: {} };
-    for (const policyId of policies) {
-      const replay = replayCase(goldenCase, { policyId });
-      if (!replay.ok) {
-        run.policies[policyId] = { ok: false, reason: replay.reason };
-        continue;
-      }
-      const summary = summarizeTrace(replay.records);
-      run.policies[policyId] = {
-        ok: true,
-        trace_id: summary.trace_id,
-        summary,
-        record_ids: replay.records.map((r) => r.record_id),
-      };
-    }
-    runs.push(run);
-  }
-
-  // Detector wiring: one call over ALL materialized summaries (detectors are
-  // cohort-relative), then attribution back to cases by trace_id or any
-  // evidence_ref that resolves to a materialized record_id.
-  const summaries = runs.flatMap((run) => policies
-    .map((policyId) => run.policies[policyId])
-    .filter((entry) => entry && entry.ok)
-    .map((entry) => entry.summary));
-
-  const byCase = new Map(runs.map((run) => [run.case_id, new Map()]));
-  const byTrace = new Map();
-  const byRecordId = new Map();
-  for (const run of runs) {
-    for (const policyId of policies) {
-      const entry = run.policies[policyId];
-      if (!entry || !entry.ok) continue;
-      if (typeof entry.trace_id === 'string') byTrace.set(entry.trace_id, [run.case_id, policyId]);
-      for (const recordId of entry.record_ids) byRecordId.set(recordId, [run.case_id, policyId]);
-    }
-  }
-
-  let detection = null;
-  if (detectAnomalies) {
-    const anomalies = detectAnomalies(summaries) || [];
-    const unattributed = [];
-    for (const anomaly of anomalies) {
-      const refs = anomaly && anomaly.payload && Array.isArray(anomaly.payload.evidence_refs)
-        ? anomaly.payload.evidence_refs
-        : [];
-      const traceRef = anomaly && anomaly.payload && typeof anomaly.payload.trace_ref === 'string'
-        ? anomaly.payload.trace_ref
-        : (anomaly && typeof anomaly.trace_id === 'string' ? anomaly.trace_id : null);
-      let target = traceRef ? byTrace.get(traceRef) : null;
-      if (!target) {
-        for (const ref of refs) {
-          target = byRecordId.get(ref);
-          if (target) break;
-        }
-      }
-      if (!target) { unattributed.push(anomaly); continue; }
-      const [caseId, policyId] = target;
-      const policyMap = byCase.get(caseId);
-      if (!policyMap.has(policyId)) policyMap.set(policyId, {});
-      const counts = policyMap.get(policyId);
-      const kind = anomalyKind(anomaly);
-      counts[kind] = (counts[kind] || 0) + 1;
-    }
-    detection = { anomalies, unattributed };
-  }
-  // Per-case signals — the summary fields the detectors consume (TASK-011):
-  // attempt/retry counters, cache posture, drops, critical path. Reported
-  // per policy run so a detector regression is attributable to a case.
-  function signalsOf(entry) {
-    const s = entry && entry.summary;
-    if (!s) return null;
-    return {
-      model_attempts: s.retries.model_attempts,
-      retries: s.retries.retries,
-      failed_spans: s.retries.failed_spans,
-      cache_misses: s.cache.misses,
-      cache_hits: s.cache.hits,
-      hit_rate: s.cache.hit_rate,
-      dropped_events: s.drops.events,
-      dropped_count: s.drops.dropped_count,
-      critical_path_ms: s.critical_path_ms,
-      telemetry_complete: s.telemetry_complete,
-      spans: s.coverage.spans,
-    };
-  }
-
-  const casesReport = {};
-  for (const run of runs) {
-    const signals = {};
-    for (const policyId of policies) {
-      signals[policyId] = signalsOf(run.policies[policyId]);
-    }
-    let anomalies = null;
-    if (detection) {
-      anomalies = 0;
-      for (const counts of byCase.get(run.case_id)?.values() || []) {
-        for (const n of Object.values(counts)) anomalies += n;
-      }
-    }
-    casesReport[run.case_id] = { signals, anomalies };
-  }
-
-  // Regression gate — only meaningful when a real detector ran. A case that
-  // declares expect.anomalies.expected_kinds must see those kinds fire; a
-  // case declaring an empty list must stay clean.
-  const unmet = [];
-  if (detection) {
-    for (const run of runs) {
-      if (!Array.isArray(run.expected_kinds)) continue;
-      for (const [policyId, counts] of byCase.get(run.case_id) || []) {
-        for (const kind of run.expected_kinds) {
-          if (!counts[kind]) {
-            unmet.push({ case_id: run.case_id, policy: policyId, kind, expected: '>=1', got: 0 });
-          }
-        }
-        for (const kind of Object.keys(counts)) {
-          if (!run.expected_kinds.includes(kind)) {
-            unmet.push({ case_id: run.case_id, policy: policyId, kind, expected: 'absent', got: counts[kind] });
-          }
-        }
-      }
-    }
-    unmet.sort((a, b) => String(a.case_id).localeCompare(String(b.case_id))
-      || a.policy.localeCompare(b.policy) || a.kind.localeCompare(b.kind));
-  }
-
-  const report = {
-    metric_version: METRIC_VERSION,
-    bench: 'anomalies',
-    seed: SEED,
-    corpus: path.relative(repoRoot, goldenPath),
-    policies,
-    detector: detectAnomalies ? 'detectAnomalies' : 'unavailable',
-    summaries: summaries.length,
-    anomalies_total: detection ? detection.anomalies.length : null,
-    unattributed: detection ? detection.unattributed.length : null,
-    unmet_expectations: detection ? unmet : null,
-    cases: casesReport,
-  };
-  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
-  if (!detection) return 0; // degraded — nothing to gate on
-  return unmet.length > 0 ? 1 : 0;
-}
-
-const BENCH_FLAGS = Object.freeze([
-  '--fixture',
-  '--recorder',
-  '--analytics',
-  '--evaluate',
-  '--perturb',
-  '--anomalies',
-]);
-
-async function runFlag(flag) {
-  switch (flag) {
-    case '--fixture': return writeFixture();
-    case '--recorder': return runRecorderBench();
-    case '--analytics': return runAnalyticsBench();
-    case '--evaluate': return runEvaluateBench();
-    case '--perturb': return runPerturb().then(({ code }) => code);
-    case '--anomalies': return runAnomaliesBench();
-    default:
-      process.stderr.write(`unknown flag: ${flag}\n`);
-      return 2;
-  }
-}
-
-// Flags run in argument order; the exit code is the first non-zero result.
-// (PLAN §5 verification chains --recorder --analytics --evaluate.)
-async function main(argv) {
-  const flags = argv.slice(2).filter((arg) => arg.startsWith('--'));
-  if (flags.length === 0) {
-    process.stderr.write(
-      `usage: node scripts/bench/data-foundation.mjs ${BENCH_FLAGS.join('|')}\n`,
-    );
-    return 2;
-  }
-  let firstFailure = 0;
-  for (const flag of flags) {
-    try {
-      const code = await runFlag(flag);
-      if (code !== 0 && firstFailure === 0) firstFailure = code;
-    } catch (err) {
-      process.stderr.write(`${flag} failed: ${err && err.message}\n`);
-      if (firstFailure === 0) firstFailure = 1;
-    }
-  }
-  return firstFailure;
-}
-const invokedAs = process.argv[1] ? path.resolve(process.argv[1]) : '';
-if (invokedAs === fileURLToPath(import.meta.url)) {
-  // Set the code then let the loop drain stdout — process.exit() hard-kills
-  // pending pipe writes, which truncates reports past the 64 KiB buffer.
-  Promise.resolve(main(process.argv)).then((code) => {
-    process.exitCode = code;
-  });
-}
-