@remnic/bench 1.0.1 → 9.3.515

package/dist/index.d.ts

@@ -44,6 +44,8 @@ interface ExtractedPage {
     frontmatter: Record<string, unknown>;
     hasExecSummary: boolean;
     hasTimeline: boolean;
+    /** Source corpus file references that support this page's generated claims. */
+    sourceRefs?: string[];
     seeAlso: string[];
     content: string;
 }
@@ -72,6 +74,8 @@ declare const REQUIRED_FRONTMATTER_FIELDS: readonly ["title", "type", "state", "
 interface Message {
     role: "user" | "assistant" | "system";
     content: string;
+    /** Optional source timestamp for benchmarks with historical query times. */
+    timestamp?: string;
 }
 interface SearchResult {
     turnIndex: number;
@@ -84,6 +88,7 @@ interface MemoryStats {
     totalMessages: number;
     totalSummaryNodes: number;
     maxDepth: number;
+    maxTurnIndex?: number;
 }
 interface BenchResponse {
     text: string;
@@ -94,8 +99,11 @@ interface BenchResponse {
     latencyMs: number;
     model: string;
 }
+interface BenchPhaseControl {
+    signal?: AbortSignal;
+}
 interface BenchResponder {
-    respond(question: string, recalledText: string): Promise<BenchResponse>;
+    respond(question: string, recalledText: string, control?: BenchPhaseControl): Promise<BenchResponse>;
 }
 interface BenchJudgeResult {
     score: number;
@@ -107,19 +115,32 @@ interface BenchJudgeResult {
     model?: string;
 }
 interface BenchJudge {
-    score(question: string, predicted: string, expected: string): Promise<number>;
-    scoreWithMetrics?(question: string, predicted: string, expected: string): Promise<BenchJudgeResult>;
+    score(question: string, predicted: string, expected: string, control?: BenchPhaseControl): Promise<number>;
+    scoreWithMetrics?(question: string, predicted: string, expected: string, control?: BenchPhaseControl): Promise<BenchJudgeResult>;
+    /**
+     * Run a benchmark-supplied yes/no judging prompt directly and return a
+     * normalized 0/1 score. Published benchmarks such as LongMemEval define
+     * their own evaluator prompt; routing those through the scalar generic
+     * judge prompt would change the metric contract.
+     */
+    scoreBinaryPrompt?(prompt: string, control?: BenchPhaseControl): Promise<BenchJudgeResult>;
 }
 interface BenchMemoryAdapter {
-    store(sessionId: string, messages: Message[]): Promise<void>;
-    recall(sessionId: string, query: string, budgetChars?: number): Promise<string>;
-    search(query: string, limit: number, sessionId?: string): Promise<SearchResult[]>;
-    reset(sessionId?: string): Promise<void>;
-    getStats(sessionId?: string): Promise<MemoryStats>;
+    store(sessionId: string, messages: Message[], control?: BenchPhaseControl): Promise<void>;
+    recall(sessionId: string, query: string, budgetChars?: number, options?: BenchRecallOptions, control?: BenchPhaseControl): Promise<string>;
+    search(query: string, limit: number, sessionId?: string, control?: BenchPhaseControl): Promise<SearchResult[]>;
+    reset(sessionId?: string, control?: BenchPhaseControl): Promise<void>;
+    getStats(sessionId?: string, control?: BenchPhaseControl): Promise<MemoryStats>;
+    /** Wait for background summarization (e.g. LCM) to finish after store(). */
+    drain?(control?: BenchPhaseControl): Promise<void>;
     destroy(): Promise<void>;
     responder?: BenchResponder;
     judge?: BenchJudge;
 }
+interface BenchRecallOptions {
+    /** Optional historical recall timestamp for benchmarks that expose query time. */
+    asOf?: string;
+}
 type LlmJudge = BenchJudge;
 type MemorySystem = BenchMemoryAdapter;
 
@@ -189,11 +210,37 @@ type BenchmarkTier = "published" | "remnic" | "custom";
 type BenchmarkStatus = "ready" | "planned";
 type BenchmarkCategory = "agentic" | "retrieval" | "conversational" | "ingestion";
 type BenchRuntimeProfile = "baseline" | "real" | "openclaw-chain";
-type BuiltInProvider = "openai" | "anthropic" | "ollama" | "litellm";
+type AmaBenchJudgeProtocol = "default" | "recommended";
+/**
+ * Built-in LLM providers supported by the bench harness.
+ *
+ * `local-llm` targets a user-hosted OpenAI-compatible endpoint
+ * (llama.cpp, vLLM, LM Studio, etc.) via `--base-url`. It mirrors
+ * the `localLlm*` plugin config on the Remnic core side so that
+ * `remnic bench published --provider local-llm` actually exercises
+ * the same transport path as the running plugin. Issue #566 slice 5.
+ *
+ * `codex-cli` shells out to `codex exec` as an isolated benchmark-only
+ * responder/judge target. It is intentionally not routed through Remnic
+ * memory or OpenClaw gateway state.
+ */
+type BuiltInProvider = "openai" | "anthropic" | "ollama" | "litellm" | "local-llm" | "codex-cli";
+type BenchReasoningEffort = "low" | "medium" | "high" | "xhigh";
 interface ProviderConfig {
     provider: BuiltInProvider;
     model: string;
     baseUrl?: string;
+    apiKey?: string;
+    retryOptions?: {
+        maxAttempts?: number;
+        baseBackoffMs?: number;
+        timeoutMs?: number;
+        max429WaitMs?: number;
+    };
+    disableThinking?: boolean;
+    reasoningEffort?: BenchReasoningEffort;
+    responderContextBudgetChars?: number;
+    responderPromptBudgetChars?: number;
 }
 interface TaskTokenUsage {
     input: number;
@@ -278,13 +325,19 @@ interface BenchmarkResult {
          * Must stay below the benchmark's canary floor.
          */
         canaryScore?: number;
+        /** "partial" if the benchmark was interrupted; absent or "complete" otherwise. */
+        status?: "complete" | "partial";
+        /** If partial, the error that caused interruption. */
+        failureReason?: string;
     };
     config: {
         runtimeProfile?: BenchRuntimeProfile | null;
         systemProvider: ProviderConfig | null;
         judgeProvider: ProviderConfig | null;
+        internalProvider?: ProviderConfig | null;
         adapterMode: string;
         remnicConfig: Record<string, unknown>;
+        benchmarkOptions?: Record<string, unknown>;
     };
     cost: {
         totalTokens: number;
@@ -339,7 +392,15 @@ interface RunBenchmarkOptions {
     ingestionAdapter?: IngestionBenchAdapter;
     systemProvider?: ProviderConfig | null;
     judgeProvider?: ProviderConfig | null;
+    internalProvider?: ProviderConfig | null;
     remnicConfig?: Record<string, unknown>;
+    benchmarkOptions?: Record<string, unknown>;
+    drainTimeoutMs?: number;
+    amaBenchJudgeProtocol?: AmaBenchJudgeProtocol;
+    amaBenchCrossJudge?: BenchJudge;
+    amaBenchCrossJudgeProvider?: ProviderConfig | null;
+    /** Called after each task completes for progress logging and partial result tracking. */
+    onTaskComplete?: (task: TaskResult, completedCount: number, totalCount?: number) => void;
 }
 interface ResolvedRunBenchmarkOptions extends RunBenchmarkOptions {
     mode: BenchmarkMode;
@@ -457,13 +518,70 @@ interface FixtureGenerator {
 
 interface RemnicAdapterOptions {
     configOverrides?: Record<string, unknown>;
+    memoryDir?: string;
     preserveRuntimeDefaults?: boolean;
     responder?: BenchResponder;
     judge?: BenchJudge;
+    drainTimeoutMs?: number;
+    replayExtractionMode?: "await" | "background" | "skip";
+    replaySourceValidAtMode?: "historical" | "batch";
+    sandboxDir?: string;
 }
 declare const createLightweightAdapter: (options?: RemnicAdapterOptions) => Promise<BenchMemoryAdapter>;
 declare const createRemnicAdapter: (options?: RemnicAdapterOptions) => Promise<BenchMemoryAdapter>;
 
+interface TimeoutGuardOptions {
+    benchmarkId: string;
+    timeoutMs?: number;
+    drainTimeoutMs?: number;
+    logProgress?: boolean;
+    log?: (message: string) => void;
+    onTimeout?: (phase: string) => void | Promise<void>;
+}
+interface TimeoutGuardConfig {
+    remnicConfig?: Record<string, unknown>;
+    systemProvider?: ProviderConfig | null;
+    judgeProvider?: ProviderConfig | null;
+}
+declare function resolveBenchmarkPhaseTimeoutMs(config: TimeoutGuardConfig): number | undefined;
+declare function resolveBenchmarkProgressLogging(remnicConfig?: Record<string, unknown>): boolean;
+declare function createTimeoutGuardedAdapter(adapter: BenchMemoryAdapter, options: TimeoutGuardOptions): BenchMemoryAdapter;
+
+interface SyntheticEmailIngestionAdapterOptions {
+    system?: BenchMemoryAdapter;
+}
+/**
+ * Isolated ingestion adapter for the synthetic email fixture benchmarks.
+ *
+ * It writes the raw source corpus through the benchmark's Remnic memory
+ * adapter when one is supplied, then exposes the extracted fixture graph in
+ * the IngestionBenchAdapter shape expected by the scoring tier. This keeps
+ * the ingestion benchmarks runnable in isolated benchmark jobs without
+ * touching a production Remnic instance.
+ */
+declare function createSyntheticEmailIngestionAdapter(options?: SyntheticEmailIngestionAdapterOptions): IngestionBenchAdapter;
+
+declare const MEMORY_EVAL_PUBLIC_LINE: "Agent memory without evals is vibes with a database.";
+type MemoryEvalDimensionId = "repeated_context_reduction" | "unnecessary_clarification_reduction" | "retrieval_correctness" | "stale_memory_harm" | "scope_respect" | "ask_when_needed" | "act_when_enough_context" | "personalization_quality";
+type MemoryEvalCategory = "context-efficiency" | "retrieval-quality" | "boundary-respect" | "action-confidence" | "personalization";
+interface MemoryEvalMetric {
+    name: string;
+    higherIsBetter: boolean;
+    description: string;
+}
+interface MemoryEvalDimension {
+    id: MemoryEvalDimensionId;
+    question: string;
+    category: MemoryEvalCategory;
+    metrics: readonly MemoryEvalMetric[];
+    quickBenchmarkIds: readonly string[];
+    fullModeGuidance: string;
+}
+declare const MEMORY_EVAL_DIMENSIONS: readonly MemoryEvalDimension[];
+declare function listMemoryEvalDimensions(): readonly MemoryEvalDimension[];
+declare function getMemoryEvalDimension(id: MemoryEvalDimensionId): MemoryEvalDimension;
+declare function listMemoryEvalBenchmarkIds(): string[];
+
 /**
  * Minimal LLM provider contract for the bench engine.
  */
@@ -473,6 +591,7 @@ interface CompletionOpts {
     temperature?: number;
     maxTokens?: number;
     headers?: Record<string, string>;
+    signal?: AbortSignal;
 }
 interface CompletionResult {
     text: string;
@@ -496,6 +615,26 @@ interface ProviderBaseConfig {
     baseUrl?: string;
     apiKey?: string;
     headers?: Record<string, string>;
+    retryOptions?: {
+        maxAttempts?: number;
+        baseBackoffMs?: number;
+        timeoutMs?: number;
+        max429WaitMs?: number;
+    };
+    /** Suppress thinking/reasoning tokens for thinking-capable models (Qwen 3.5, Gemma 4, DeepSeek). */
+    disableThinking?: boolean;
+    /**
+     * Optional answering-only memory-context budget. Benchmark artifacts keep the
+     * full recalled text, but provider-backed responders may receive this compact
+     * deterministic view to avoid transport-specific prompt stalls.
+     */
+    responderContextBudgetChars?: number;
+    /**
+     * Optional answering-only question/protocol budget. This keeps the original
+     * benchmark question and artifact unchanged while shortening repeated harness
+     * instructions for slow transport-backed responders such as Codex CLI.
+     */
+    responderPromptBudgetChars?: number;
 }
 interface OpenAiCompatibleProviderConfig extends ProviderBaseConfig {
     provider?: "openai" | "litellm";
@@ -507,12 +646,46 @@ interface AnthropicProviderConfig extends ProviderBaseConfig {
 interface OllamaProviderConfig extends ProviderBaseConfig {
     provider?: "ollama";
 }
+/**
+ * `local-llm` targets a user-hosted OpenAI-compatible endpoint
+ * (llama.cpp, vLLM, LM Studio, etc.). `baseUrl` is required at the
+ * CLI layer — it mirrors the plugin's `localLlmUrl` config and is
+ * what tells the bench which local server to talk to. The transport
+ * is intentionally OpenAI-compatible: `/v1/chat/completions` +
+ * `/v1/models`. Issue #566 slice 5.
+ */
+interface LocalLlmProviderConfig extends ProviderBaseConfig {
+    provider?: "local-llm";
+    baseUrl: string;
+}
+interface CodexCliProviderConfig extends ProviderBaseConfig {
+    provider?: "codex-cli";
+    /** Codex CLI model reasoning effort. Bench CLI defaults this to xhigh. */
+    reasoningEffort?: BenchReasoningEffort;
+    /** Optional executable override for tests or non-standard Codex CLI installs. */
+    executable?: string;
+    /**
+     * Optional diagnostics artifact directory. When set, the provider writes
+     * per-call metadata that helps debug slow benchmark completions without
+     * depending on transient temp workspaces.
+     */
+    diagnosticsDir?: string;
+    /**
+     * `metadata` stores hashes/counts only. `full` additionally stores the full
+     * benchmark prompt and should only be used for isolated benchmark datasets.
+     */
+    diagnosticsMode?: "metadata" | "full";
+}
 type ProviderFactoryConfig = (OpenAiCompatibleProviderConfig & {
     provider: "openai" | "litellm";
 }) | (AnthropicProviderConfig & {
     provider: "anthropic";
 }) | (OllamaProviderConfig & {
     provider: "ollama";
+}) | (LocalLlmProviderConfig & {
+    provider: "local-llm";
+}) | (CodexCliProviderConfig & {
+    provider: "codex-cli";
 });
 interface ProviderDiscoveryResult {
     provider: BuiltInProvider;
@@ -629,6 +802,15 @@ declare const BENCHMARK_RESULT_SCHEMA: {
                             readonly baseUrl: {
                                 readonly type: "string";
                             };
+                            readonly reasoningEffort: {
+                                readonly type: "string";
+                            };
+                            readonly responderContextBudgetChars: {
+                                readonly type: "number";
+                            };
+                            readonly responderPromptBudgetChars: {
+                                readonly type: "number";
+                            };
                         };
                     }];
                 };
@@ -648,6 +830,31 @@ declare const BENCHMARK_RESULT_SCHEMA: {
                             readonly baseUrl: {
                                 readonly type: "string";
                             };
+                            readonly reasoningEffort: {
+                                readonly type: "string";
+                            };
+                        };
+                    }];
+                };
+                readonly internalProvider: {
+                    readonly anyOf: readonly [{
+                        readonly type: "null";
+                    }, {
+                        readonly type: "object";
+                        readonly required: readonly ["provider", "model"];
+                        readonly properties: {
+                            readonly provider: {
+                                readonly type: "string";
+                            };
+                            readonly model: {
+                                readonly type: "string";
+                            };
+                            readonly baseUrl: {
+                                readonly type: "string";
+                            };
+                            readonly reasoningEffort: {
+                                readonly type: "string";
+                            };
                         };
                     }];
                 };
@@ -752,11 +959,315 @@ declare const BENCHMARK_RESULT_SCHEMA: {
     };
 };
 
+declare const BENCHMARK_REPRO_MANIFEST_FILENAME = "MANIFEST.json";
+declare const BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION = 1;
+interface BenchmarkReproManifestFile {
+    path: string;
+    kind: "file" | "symlink";
+    sizeBytes: number;
+    sha256: string;
+    target?: string;
+}
+interface BenchmarkReproManifestDataset {
+    benchmark: string;
+    status: "not-provided" | "missing" | "hashed";
+    path?: string;
+    realpath?: string;
+    fileCount: number;
+    totalBytes: number;
+    sha256?: string;
+    files: BenchmarkReproManifestFile[];
+}
+interface BenchmarkReproManifestResult {
+    path: string;
+    sha256: string;
+    sizeBytes: number;
+    resultId: string;
+    benchmark: string;
+    mode: BenchmarkMode;
+    gitSha: string;
+    runCount: number;
+    seeds: number[];
+    taskCount: number;
+    configHash: string;
+}
+interface BenchmarkReproManifest {
+    schemaVersion: number;
+    generatedAt: string;
+    run: {
+        id: string;
+        mode?: BenchmarkMode;
+        selectedBenchmarks: string[];
+        runtimeProfiles: string[];
+        selectedWorkItems: Array<{
+            benchmark: string;
+            runtimeProfile: string;
+        }>;
+        limit?: number;
+        seed?: number;
+    };
+    git: {
+        commit: string;
+        shortCommit: string;
+        dirty: boolean;
+        dirtyEntryCount: number;
+    };
+    command: {
+        cwd: string;
+        argv: string[];
+        envKeys: string[];
+    };
+    environment: {
+        platform: NodeJS.Platform;
+        arch: string;
+        nodeVersion: string;
+        hostname: string;
+        packageManager?: string;
+    };
+    qmd?: {
+        configDir?: string;
+        cacheDir?: string;
+        collections: string[];
+    };
+    configFiles: Array<{
+        label: string;
+        path: string;
+        sha256?: string;
+        sizeBytes?: number;
+        missing?: boolean;
+    }>;
+    datasets: BenchmarkReproManifestDataset[];
+    results: BenchmarkReproManifestResult[];
+    artifactHash: string;
+}
+interface BuildBenchmarkReproManifestOptions {
+    resultPaths?: string[];
+    runId?: string;
+    selectedBenchmarks?: string[];
+    runtimeProfiles?: string[];
+    selectedWorkItems?: Array<{
+        benchmark: string;
+        runtimeProfile: string;
+    }>;
+    mode?: BenchmarkMode;
+    limit?: number;
+    seed?: number;
+    datasetDirs?: Record<string, string | undefined>;
+    command?: {
+        cwd?: string;
+        argv?: string[];
+        env?: NodeJS.ProcessEnv;
+        envKeys?: string[];
+    };
+    configFiles?: Array<{
+        label: string;
+        path?: string;
+    }>;
+    qmd?: {
+        configDir?: string;
+        cacheDir?: string;
+        collections?: string[];
+    };
+}
+declare function buildBenchmarkReproManifest(resultsDir: string, options?: BuildBenchmarkReproManifestOptions): Promise<BenchmarkReproManifest>;
+declare function writeBenchmarkReproManifest(resultsDir: string, options?: BuildBenchmarkReproManifestOptions): Promise<string>;
+
+/**
+ * Public leaderboard artifact schema for published benchmarks.
+ *
+ * `BenchmarkArtifact` is deliberately flatter and more opinionated than
+ * the internal `BenchmarkResult`. The goal is a stable, versioned payload
+ * that Remnic.ai and third-party leaderboard consumers can rely on
+ * without digging into every per-task field the internal runner captures.
+ *
+ * One artifact is written per run to
+ *   docs/benchmarks/results/<iso-date>-<benchmark>-<model>-<gitShaShort>.json
+ * (gitignored during development; promoted per-release by slice 6).
+ *
+ * Any breaking change to the artifact shape requires a `schemaVersion`
+ * bump. The companion `buildBenchmarkArtifact()` and
+ * `writeBenchmarkArtifact()` functions in this file emit the current
+ * version; `parseBenchmarkArtifact()` rejects unknown versions.
+ */
+
+/**
+ * Current artifact schema version. Bump when the serialized shape
+ * changes in a way that breaks existing leaderboard consumers.
+ *
+ * History:
+ *   1 — initial schema (issue #566).
+ */
+declare const BENCHMARK_ARTIFACT_SCHEMA_VERSION: 1;
+/** Identifiers of published-benchmark runners that can emit public artifacts. */
+declare const PUBLISHED_BENCHMARK_ARTIFACT_IDS: readonly ["ama-bench", "memory-arena", "amemgym", "longmemeval", "locomo", "beam", "personamem", "memoryagentbench", "membench"];
+/** Identifier of a published-benchmark runner. */
+type PublishedBenchmarkId = (typeof PUBLISHED_BENCHMARK_ARTIFACT_IDS)[number];
+interface BenchmarkArtifactSystem {
+    /** Short product name, e.g. "remnic". */
+    name: string;
+    /** Semver of `@remnic/core` at run time. */
+    version: string;
+    /** Short git SHA of the repository producing the artifact. */
+    gitSha: string;
+}
+interface BenchmarkArtifactEnvironment {
+    /** Node.js version reported by `process.version` at run time. */
+    node: string;
+    /** `process.platform` at run time (linux/darwin/win32/...). */
+    os: string;
+    /** Optional CPU architecture (arm64/x64/...). */
+    arch?: string;
+}
+interface BenchmarkArtifactPerTaskScore {
+    /** Runner-assigned task ID (stable across reruns). */
+    taskId: string;
+    /** Task-level scores keyed by metric name (e.g. f1, llm_judge). */
+    scores: Record<string, number>;
+    /** Optional task category / bucket for group-by reports. */
+    category?: string;
+}
+interface BenchmarkArtifact {
+    /** Artifact schema version. See `BENCHMARK_ARTIFACT_SCHEMA_VERSION`. */
+    schemaVersion: typeof BENCHMARK_ARTIFACT_SCHEMA_VERSION;
+    /** Benchmark identifier, e.g. "longmemeval" or "locomo". */
+    benchmarkId: PublishedBenchmarkId;
+    /**
+     * Dataset version the runner evaluated against. Free-form string so
+     * runners can record the HuggingFace revision, filename, or
+     * upstream dataset tag.
+     */
+    datasetVersion: string;
+    system: BenchmarkArtifactSystem;
+    /** Evaluator model ID (e.g. "gpt-4o-mini"). */
+    model: string;
+    /** RNG / selection seed used for this run. */
+    seed: number;
+    /** Aggregate metric means keyed by metric name. */
+    metrics: Record<string, number>;
+    /** Per-task score breakdown. Arbitrary-length; safe to truncate for public pages. */
+    perTaskScores: BenchmarkArtifactPerTaskScore[];
+    /** ISO-8601 timestamp of run start. */
+    startedAt: string;
+    /** ISO-8601 timestamp of run finish. */
+    finishedAt: string;
+    /** Total wall-clock duration in milliseconds. */
+    durationMs: number;
+    env: BenchmarkArtifactEnvironment;
+    /** Optional explanatory note (e.g. "--limit 100"). Never contains PII. */
+    note?: string;
+}
+/** Input to `buildBenchmarkArtifact()` beyond what `BenchmarkResult` already carries. */
+interface BuildBenchmarkArtifactInput {
+    benchmarkId: PublishedBenchmarkId;
+    datasetVersion: string;
+    model: string;
+    seed: number;
+    startedAt: string;
+    finishedAt: string;
+    result: BenchmarkResult;
+    /** Optional category extractor for `perTaskScores[].category`. */
+    categoryFor?: (task: TaskResult) => string | undefined;
+    /** Optional free-form note (e.g. `"--limit 100"`). */
+    note?: string;
+}
+/**
+ * Build a `BenchmarkArtifact` from a runner's `BenchmarkResult`.
+ * Aggregates metrics to their `.mean` for public consumption; preserves
+ * per-task scores verbatim. The result is sort-stable: metric keys are
+ * emitted in sorted order and perTaskScores preserves runner order.
+ */
+declare function buildBenchmarkArtifact(input: BuildBenchmarkArtifactInput): BenchmarkArtifact;
+/**
+ * Build the canonical on-disk filename for an artifact. Filename shape:
+ *   <iso-date>-<benchmark>-<model>-<gitShaShort>.json
+ * where iso-date is the startedAt date (YYYY-MM-DD) and gitShaShort is
+ * the first 7 chars of system.gitSha (or "unknown" if absent).
+ *
+ * Every segment that contributes to the filename is sanitized through
+ * `sanitizeSegment()` so it cannot contain `/`, `..`, NUL, or any other
+ * path-separator characters — preventing a malicious artifact input
+ * from directing `writeBenchmarkArtifact()` outside of `outputDir`.
+ */
+declare function buildBenchmarkArtifactFilename(artifact: BenchmarkArtifact): string;
+/** Serialize an artifact to deterministic JSON (sorted keys, indented). */
+declare function serializeBenchmarkArtifact(artifact: BenchmarkArtifact): string;
+/** Compute SHA-256 of the canonical JSON serialization of the artifact. */
+declare function hashBenchmarkArtifact(artifact: BenchmarkArtifact): string;
+interface WriteBenchmarkArtifactResult {
+    path: string;
+    filename: string;
+    sha256: string;
+    bytes: number;
+}
+/**
+ * Write the artifact to `<outputDir>/<filename>` and return the resulting
+ * path, filename, SHA-256 of the canonical serialization, and byte count.
+ * Creates `outputDir` recursively if needed.
+ *
+ * Belt-and-suspenders: even though `buildBenchmarkArtifactFilename()`
+ * sanitizes every segment, this function also verifies the resolved
+ * target stays inside `outputDir`. Any path-traversal attempt throws
+ * before the write occurs.
+ */
+declare function writeBenchmarkArtifact(artifact: BenchmarkArtifact, outputDir: string): Promise<WriteBenchmarkArtifactResult>;
+/**
+ * Parse + validate a BenchmarkArtifact from raw JSON. Throws on version
+ * mismatch, missing required fields, or structural errors. Keep this in
+ * sync with the `BenchmarkArtifact` interface — every new required
+ * field needs a matching check here and a `schemaVersion` bump.
+ */
+declare function parseBenchmarkArtifact(raw: string): BenchmarkArtifact;
+/** Read + parse + re-hash an artifact file. Handy for `verify-artifact` CLI. */
+declare function loadBenchmarkArtifact(filePath: string): Promise<{
+    artifact: BenchmarkArtifact;
+    sha256: string;
+    bytes: number;
+}>;
+
 declare function createAnthropicProvider(config: AnthropicProviderConfig): LlmProvider;
 
+interface CodexCliRunRequest {
+    executable: string;
+    args: string[];
+    input: string;
+    outputPath: string;
+    workspacePath: string;
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    env: NodeJS.ProcessEnv;
+}
+interface CodexCliRunResult {
+    status: number | null;
+    signal: NodeJS.Signals | null;
+    stdout: string;
+    stderr: string;
+    outputText: string;
+}
+interface CodexCliProviderDeps {
+    runCodexCli?: (request: CodexCliRunRequest) => Promise<CodexCliRunResult>;
+    runCodexVersion?: (executable: string, env: NodeJS.ProcessEnv) => Promise<{
+        status: number | null;
+        stderr: string;
+    }>;
+}
+declare function createCodexCliProvider(config: CodexCliProviderConfig, deps?: CodexCliProviderDeps): LlmProvider;
+
+/**
+ * Result enrichment and JSON writing helpers.
+ */
+
+declare function redactBenchmarkResultSecrets<T>(value: T): T;
+declare function writeBenchmarkResult(result: BenchmarkResult, outputDir: string): Promise<string>;
+declare function getRemnicVersion(): Promise<string>;
+
+interface DiscoverAllProvidersOptions {
+    includeCodexCli?: boolean;
+}
 declare function createProvider(config: ProviderFactoryConfig): LlmProvider;
-declare function discoverAllProviders(): Promise<ProviderDiscoveryResult[]>;
+declare function discoverAllProviders(options?: DiscoverAllProvidersOptions): Promise<ProviderDiscoveryResult[]>;
 
+type BenchmarkAnswerMode = "default" | "strict" | "agentic-memory";
+type BenchmarkAnswerFormat = "auto" | "choice-letter" | "choice-number" | "instruction" | "short" | "short-with-specifics" | "structured";
 interface BenchmarkAnswerResult {
     finalAnswer: string;
     recalledText: string;
@@ -768,12 +1279,145 @@ interface BenchmarkAnswerResult {
     };
     model?: string;
 }
+interface BenchmarkQuestionContext {
+    benchmark?: string;
+    domain?: string;
+    task?: string;
+    taskType?: string;
+    qaType?: string;
+}
 declare function answerBenchmarkQuestion(options: {
     question: string;
     recalledText: string;
     responder?: BenchResponder;
+    answerMode?: BenchmarkAnswerMode;
+    answerFormat?: BenchmarkAnswerFormat;
+    questionContext?: BenchmarkQuestionContext;
+    retryUnknownWithEvidence?: boolean;
 }): Promise<BenchmarkAnswerResult>;
 
+interface LeaderboardArtifactWrite {
+    benchmark: string;
+    path: string;
+    format: string;
+    records: number;
+}
+interface AmaBenchLeaderboardRow {
+    episode_id: number | string;
+    answer_list: string[];
+}
+declare function writeLeaderboardArtifactsForResult(result: BenchmarkResult, outputDir: string): Promise<LeaderboardArtifactWrite[]>;
+declare function buildAmaBenchLeaderboardRows(result: BenchmarkResult): AmaBenchLeaderboardRow[];
+declare function serializeJsonl(rows: readonly AmaBenchLeaderboardRow[]): string;
+
+type AmaBenchDiagnosticRecallMode = "remnic-full" | "explicit-evidence-only" | "oracle-trajectory";
+type AmaBenchDiagnosticAnswererMode = "normal" | "strong";
+interface AmaBenchDiagnosticVariant {
+    id: string;
+    label: string;
+    recallMode: AmaBenchDiagnosticRecallMode;
+    answererMode: AmaBenchDiagnosticAnswererMode;
+    description: string;
+}
+declare const AMA_BENCH_DIAGNOSTIC_VARIANTS: readonly AmaBenchDiagnosticVariant[];
+declare function selectAmaBenchDiagnosticVariants(options?: {
+    ids?: string[];
+    includeStrong?: boolean;
+}): AmaBenchDiagnosticVariant[];
+interface AmaBenchDiagnosticAdapterOptions {
+    strongResponder?: BenchResponder;
+}
+declare function createAmaBenchDiagnosticAdapter(base: BenchMemoryAdapter, variant: AmaBenchDiagnosticVariant, options?: AmaBenchDiagnosticAdapterOptions): BenchMemoryAdapter;
+declare function buildOracleTrajectoryRecall(messages: readonly Message[], budgetChars?: number): string;
+declare function extractMarkdownSectionsByTitle(markdown: string, allowedTitles: readonly string[]): string;
+interface AmaBenchDiagnosticTaskRow {
+    variantId: string;
+    taskId: string;
+    episodeId?: string | number;
+    domain: string;
+    qaType: string;
+    taskType: string;
+    scores: Record<string, number>;
+    unknownLike: boolean;
+    recalledLength: number;
+    answeredLength: number;
+    recallSections: string[];
+    responderModel?: string;
+    judgeModel?: string;
+    crossJudgeModel?: string;
+    crossJudgeScore?: number;
+    evidence?: AmaBenchDiagnosticTaskEvidence;
+}
+interface AmaBenchDiagnosticTaskEvidence {
+    question: string;
+    expected: string;
+    actual: string;
+    recalledText: string;
+    truncatedFields?: string[];
+}
+interface AmaBenchDiagnosticBreakdown {
+    key: string;
+    taskCount: number;
+    unknownLikeRate: number;
+    scoreMeans: Record<string, number>;
+    scoreCounts: Record<string, number>;
+}
+interface AmaBenchDiagnosticVariantSummary {
+    variant: AmaBenchDiagnosticVariant;
+    usesFullRemnicRecallProcess: boolean;
+    isPrimaryFullSystemScore: boolean;
+    taskCount: number;
+    unknownLikeRate: number;
+    scoreMeans: Record<string, number>;
+    scoreCounts: Record<string, number>;
+    byDomain: AmaBenchDiagnosticBreakdown[];
+    byQaType: AmaBenchDiagnosticBreakdown[];
+    byDomainAndQaType: AmaBenchDiagnosticBreakdown[];
+    tasks: AmaBenchDiagnosticTaskRow[];
+}
+interface SanitizedDiagnosticProvider {
+    provider: string;
+    model: string;
+    baseUrl?: string;
+    reasoningEffort?: string;
+}
+interface AmaBenchDiagnosticMatrixArtifact {
+    schemaVersion: 1;
+    benchmark: "ama-bench";
+    generatedAt: string;
+    mode: BenchmarkMode;
+    config: {
+        runtimeProfile?: string;
+        adapterMode?: string;
+        datasetDir?: string;
+        limit?: number;
+        seed?: number;
+        systemProvider?: SanitizedDiagnosticProvider | null;
+        judgeProvider?: SanitizedDiagnosticProvider | null;
+        internalProvider?: SanitizedDiagnosticProvider | null;
+        amaBenchCrossJudgeProvider?: SanitizedDiagnosticProvider | null;
+        strongSystemProvider?: SanitizedDiagnosticProvider | null;
+        variantIds?: string[];
+        includeTaskEvidence?: boolean;
+        taskEvidenceMaxChars?: number;
+    };
+    variants: AmaBenchDiagnosticVariantSummary[];
+}
+interface AmaBenchDiagnosticRunContext {
+    runtimeProfile?: string;
+    hasResponder?: boolean;
+    includeTaskEvidence?: boolean;
+    taskEvidenceMaxChars?: number;
+}
+declare function buildAmaBenchDiagnosticVariantSummary(variant: AmaBenchDiagnosticVariant, result: BenchmarkResult, context?: AmaBenchDiagnosticRunContext): AmaBenchDiagnosticVariantSummary;
+declare function buildAmaBenchDiagnosticMatrixArtifact(args: {
+    mode: BenchmarkMode;
+    config?: AmaBenchDiagnosticMatrixArtifact["config"];
+    variants: AmaBenchDiagnosticVariantSummary[];
+    generatedAt?: string;
+}): AmaBenchDiagnosticMatrixArtifact;
+declare function isAmaBenchUnknownLikeAnswer(answer: string): boolean;
+
 /**
  * Sealed LLM-judge rubric loader, invocation, and score parser for the
  * Assistant bench tier.
@@ -911,77 +1555,215 @@ interface GatewayResponderOptions {
     workspaceDir?: string;
     llmFactory?: (gatewayConfig: GatewayConfig, runtimeContext: FallbackLlmRuntimeContext) => Pick<FallbackLlmClient, "chatCompletion">;
 }
-declare function createResponderFromProvider(provider: LlmProvider): BenchResponder;
+interface ProviderResponderOptions {
+    contextBudgetChars?: number;
+    promptBudgetChars?: number;
+}
+declare function createResponderFromProvider(provider: LlmProvider, options?: ProviderResponderOptions): BenchResponder;
 declare function createProviderBackedResponder(config: ProviderFactoryConfig, providerInstance?: LlmProvider): BenchResponder;
 declare function createProviderBackedJudge(config: ProviderFactoryConfig, providerInstance?: LlmProvider): BenchJudge;
+declare function createProviderBackedAmaBenchRecommendedJudge(config: ProviderFactoryConfig, providerInstance?: LlmProvider): BenchJudge;
 declare function createStructuredJudgeFromProvider(provider: LlmProvider): StructuredJudge;
 declare function createProviderBackedStructuredJudge(config: ProviderFactoryConfig, providerInstance?: LlmProvider): StructuredJudge;
 declare function createGatewayResponder(options: GatewayResponderOptions): BenchResponder;
 
 declare function createLiteLlmProvider(config: OpenAiCompatibleProviderConfig): LlmProvider;
 
-declare function createOllamaProvider(config: OllamaProviderConfig): LlmProvider;
-
 /**
- * Minimal OpenAI-compatible provider for phase 1 bench execution.
+ * Local-LLM bench provider — issue #566 slice 5.
+ *
+ * Talks to a user-hosted OpenAI-compatible endpoint (llama.cpp,
+ * vLLM, LM Studio, etc.) using the exact same wire contract
+ * (`/v1/chat/completions` + `/v1/models`) that the Remnic core
+ * `LocalLlmClient` uses. The goal is transport-level parity:
+ * anything `remnic bench published --provider local-llm` can reach
+ * is something the running plugin can also reach.
+ *
+ * Why this is a distinct provider from `openai`:
+ *
+ *   - The OpenAI-compatible provider treats `baseUrl` as optional
+ *     and defaults to `https://api.openai.com/v1`. That default is
+ *     wrong for local servers, and silently falling through to it
+ *     violates CLAUDE.md rule 51 (reject invalid user input).
+ *   - `local-llm` REQUIRES `baseUrl` at the CLI boundary so the
+ *     user must explicitly point at their server. A missing
+ *     base URL is a user error, not a default.
+ *   - Discovery for `local-llm` is reserved for the future — the
+ *     built-in `discoverAllProviders` probe does not assume a
+ *     local-llm URL is reachable. Users opt in with `--base-url`.
+ *
+ * See `packages/remnic-core/src/summarizer.ts` for the core-side
+ * `LocalLlmClient` invocation pattern and
+ * `packages/plugin-openclaw/openclaw.plugin.json` for the
+ * `localLlmUrl` / `localLlmModel` config that this provider
+ * mirrors.
  */
 
-declare function createOpenAiCompatibleProvider(config: OpenAiCompatibleProviderConfig): LlmProvider;
-
-type BenchModelSource = "plugin" | "gateway";
-interface ResolveBenchRuntimeProfileOptions {
-    runtimeProfile?: BenchRuntimeProfile;
-    remnicConfigPath?: string;
-    openclawConfigPath?: string;
-    modelSource?: BenchModelSource;
-    gatewayAgentId?: string;
-    fastGatewayAgentId?: string;
-    systemProvider?: BuiltInProvider;
-    systemModel?: string;
-    systemBaseUrl?: string;
-    judgeProvider?: BuiltInProvider;
-    judgeModel?: string;
-    judgeBaseUrl?: string;
-}
-interface ResolvedBenchRuntimeProfile {
-    profile: BenchRuntimeProfile;
-    remnicConfig: Record<string, unknown>;
-    effectiveRemnicConfig: Record<string, unknown>;
-    adapterOptions: {
-        configOverrides: Record<string, unknown>;
-        preserveRuntimeDefaults?: boolean;
-        responder?: BenchResponder;
-        judge?: BenchJudge;
-    };
-    systemProvider: ProviderConfig | null;
-    judgeProvider: ProviderConfig | null;
-}
-declare function resolveBenchRuntimeProfile(options: ResolveBenchRuntimeProfileOptions): Promise<ResolvedBenchRuntimeProfile>;
-
-/**
- * Published benchmark registry for @remnic/bench phase 1.
- */
+declare function createLocalLlmProvider(config: LocalLlmProviderConfig): LlmProvider;
 
-declare function listBenchmarks(): BenchmarkDefinition[];
-declare function getBenchmark(id: string): BenchmarkDefinition | undefined;
+declare function createOllamaProvider(config: OllamaProviderConfig): LlmProvider;
 
 /**
- * Result enrichment and JSON writing helpers.
+ * Minimal OpenAI-compatible provider for phase 1 bench execution.
  */
 
-declare function writeBenchmarkResult(result: BenchmarkResult, outputDir: string): Promise<string>;
+declare function createOpenAiCompatibleProvider(config: OpenAiCompatibleProviderConfig): LlmProvider;
 
 /**
- * Seed-sequence generation for benchmark runs.
+ * Shared types for the Assistant bench tier.
  *
- * Factored out of `benchmark.ts` so individual runners can reuse it without
- * triggering a circular import through `benchmark.ts -> registry.ts ->
- * runner.ts -> benchmark.ts`.
- */
-declare function buildBenchmarkRunSeeds(runCount: number, baseSeed?: number): number[];
-
-/**
- * Public benchmark execution helpers.
+ * Every Assistant benchmark shares the same shape:
+ *   - A synthetic memory graph (facts, stances, entities) the agent may read.
+ *   - A scenario prompt given to the agent.
+ *   - A sealed-rubric judge pass that scores the agent's output along
+ *     identity_accuracy / stance_coherence / novelty / calibration.
+ *
+ * The goal is reviewability: each benchmark folder ships a small fixture.ts
+ * that returns `AssistantScenario` values, and the runner wires the shared
+ * multi-run + bootstrap-CI infrastructure around them.
+ */
+
+interface AssistantMemoryFact {
+    id: string;
+    summary: string;
+    /**
+     * Free-form tags (topic, entity) used to render the memory-graph summary
+     * that is handed to the judge. Not shown to the agent.
+     */
+    tags?: string[];
+}
+interface AssistantStance {
+    topic: string;
+    position: string;
+}
+interface AssistantMemoryGraph {
+    userHandle: string;
+    userRole: string;
+    /** Fixed scenario date shown to the agent and judge for reproducible temporal reasoning. */
+    currentDate?: string;
+    facts: AssistantMemoryFact[];
+    stances: AssistantStance[];
+    openThreads: string[];
+}
+interface AssistantScenario {
+    id: string;
+    title: string;
+    scenarioPrompt: string;
+    memoryGraph: AssistantMemoryGraph;
+    /**
+     * Small label describing what the scenario is meant to exercise. Useful in
+     * dashboards for filtering. Never exposed to the agent.
+     */
+    focus: string;
+}
+/**
+ * Minimal agent contract for the Assistant tier. The agent receives the
+ * scenario prompt plus a pre-rendered memory view (analogous to what the
+ * Remnic recall stack would hand to a downstream chat model), and returns
+ * its final answer text.
+ */
+interface AssistantAgent {
+    respond(request: {
+        scenarioId: string;
+        prompt: string;
+        memoryView: string;
+        seed: number;
+        runIndex: number;
+        runCount: number;
+    }): Promise<string>;
+}
+interface AssistantRunnerOptions {
+    agent: AssistantAgent;
+    judge: StructuredJudge | undefined;
+    rubricId?: string;
+    /**
+     * Directory where per-run spot-check JSONL files are appended. Defaults to
+     * `<cwd>/benchmarks/results/spot-checks`.
+     */
+    spotCheckDir?: string;
+    /**
+     * Seed array for deterministic multi-run scheduling. When omitted the
+     * benchmark runner picks a fresh seed array via `buildBenchmarkRunSeeds`.
+     */
+    seeds?: number[];
+    /**
+     * Override used by tests and CLI smoke runs to cap iterations. Must be
+     * `>= 1`. The production contract is `>= 5` per the issue spec.
+     */
+    runCount?: number;
+    /**
+     * Random-number factory for bootstrap sampling. Injected in tests.
+     */
+    random?: () => number;
+}
+
+type BenchModelSource = "plugin" | "gateway";
+interface ResolveBenchRuntimeProfileOptions {
+    runtimeProfile?: BenchRuntimeProfile;
+    remnicConfigPath?: string;
+    openclawConfigPath?: string;
+    modelSource?: BenchModelSource;
+    gatewayAgentId?: string;
+    fastGatewayAgentId?: string;
+    systemProvider?: BuiltInProvider;
+    systemModel?: string;
+    systemBaseUrl?: string;
+    systemApiKey?: string;
+    systemCodexReasoningEffort?: ProviderConfig["reasoningEffort"];
+    systemResponderContextBudgetChars?: number;
+    systemResponderPromptBudgetChars?: number;
+    judgeProvider?: BuiltInProvider;
+    judgeModel?: string;
+    judgeBaseUrl?: string;
+    judgeApiKey?: string;
+    judgeCodexReasoningEffort?: ProviderConfig["reasoningEffort"];
+    internalProvider?: BuiltInProvider;
+    internalModel?: string;
+    internalBaseUrl?: string;
+    internalApiKey?: string;
+    internalDisableThinking?: boolean;
+    internalCodexReasoningEffort?: ProviderConfig["reasoningEffort"];
+    lcmObserveConcurrency?: number;
+    requestTimeout?: number;
+    drainTimeout?: number;
+    max429WaitMs?: number;
+    disableThinking?: boolean;
+}
+interface ResolvedBenchRuntimeProfile {
+    profile: BenchRuntimeProfile;
+    remnicConfig: Record<string, unknown>;
+    effectiveRemnicConfig: Record<string, unknown>;
+    adapterOptions: {
+        configOverrides: Record<string, unknown>;
+        preserveRuntimeDefaults?: boolean;
+        responder?: BenchResponder;
+        judge?: BenchJudge;
+        drainTimeoutMs?: number;
+    };
+    systemProvider: ProviderConfig | null;
+    judgeProvider: ProviderConfig | null;
+    internalProvider: ProviderConfig | null;
+}
+declare function resolveBenchRuntimeProfile(options: ResolveBenchRuntimeProfileOptions): Promise<ResolvedBenchRuntimeProfile>;
+
+/**
+ * Published benchmark registry for @remnic/bench phase 1.
+ */
+
+declare function listBenchmarks(): BenchmarkDefinition[];
+declare function getBenchmark(id: string): BenchmarkDefinition | undefined;
+
+/**
+ * Seed-sequence generation for benchmark runs.
+ *
+ * Factored out of `benchmark.ts` so individual runners can reuse it without
+ * triggering a circular import through `benchmark.ts -> registry.ts ->
+ * runner.ts -> benchmark.ts`.
+ */
+declare function buildBenchmarkRunSeeds(runCount: number, baseSeed?: number): number[];
+
+/**
+ * Public benchmark execution helpers.
  */
 
 declare function resolveBenchmarkRunCount(mode: BenchmarkMode, requestedIterations?: number): number;
@@ -1190,6 +1972,113 @@ declare function buildBenchmarkPublishFeed(outputDir: string, target: BenchmarkP
 declare function writeBenchmarkPublishFeed(feed: PublishedBenchmarkFeed, outputPath: string): Promise<string>;
 declare function renderBenchmarkResultExport(result: BenchmarkResult, format: BenchmarkExportFormat): string;
 
+interface HaystackTurn {
+    role: "user" | "assistant";
+    content: string;
+}
+interface LongMemEvalItem {
+    question_id: string | number;
+    question_type: string;
+    question: string;
+    answer: string;
+    question_date: string;
+    haystack_dates: string[];
+    haystack_session_ids: string[];
+    haystack_sessions: HaystackTurn[][];
+    answer_session_ids: string[];
+}
+
+interface LoCoMoQA {
+    question: string;
+    answer: string;
+    evidence: string[];
+    category: number;
+}
+interface LoCoMoConversation {
+    sample_id: string;
+    conversation: Record<string, unknown>;
+    qa: LoCoMoQA[];
+    event_summary?: unknown;
+    observation?: unknown;
+    session_summary?: unknown;
+}
+
+/**
+ * Shared dataset loader helpers for the published LongMemEval + LoCoMo
+ * benchmark runners. Wraps the fs probe + JSON parse + fallback logic
+ * previously duplicated inside each runner's `loadDataset` function.
+ *
+ * Contract:
+ *
+ *   - When `datasetDir` is defined, loaders probe the known canonical
+ *     filenames in order. The first readable file wins. If none are
+ *     readable, the result is `{ source: "missing", errors }`.
+ *   - When `datasetDir` is undefined (or resolves to `missing`) and
+ *     `mode === "quick"`, loaders return the bundled smoke fixture with
+ *     source `"smoke"` so the caller can surface a clear log message.
+ *   - When `mode === "full"` and no dataset is found, loaders return
+ *     `{ source: "missing", errors }` and callers must throw — full mode
+ *     never silently falls back to the smoke fixture.
+ *
+ * `scripts/bench/fetch-datasets.sh` documents the expected filenames; keep
+ * them in sync when adding new variants.
+ */
+
+/** Canonical LongMemEval-S filenames probed by the loader, in priority order. */
+declare const LONG_MEM_EVAL_DATASET_FILENAMES: readonly string[];
+/** Canonical LoCoMo-10 filenames probed by the loader, in priority order. */
+declare const LOCOMO_DATASET_FILENAMES: readonly string[];
+type DatasetSource = "dataset" | "smoke" | "missing";
+interface LoadedDataset<T> {
+    source: DatasetSource;
+    /** Filename relative to `datasetDir` when source === "dataset". */
+    filename?: string;
+    items: T[];
+    /** Parse/read errors encountered while probing candidate filenames. */
+    errors: string[];
+}
+interface LoadDatasetOptions {
+    mode: BenchmarkMode;
+    datasetDir?: string;
+    limit?: number;
+}
+/** Load LongMemEval-S from disk, falling back to the smoke fixture in quick mode. */
+declare function loadLongMemEvalS(options: LoadDatasetOptions): Promise<LoadedDataset<LongMemEvalItem>>;
+/**
+ * Load LoCoMo-10 from disk, falling back to the smoke fixture in quick mode.
+ *
+ * `parseFile` is optional — callers that need richer structural
+ * normalization (e.g. the LoCoMo runner's QA answer coercion) can pass
+ * their own parser. When omitted, a minimal parser is used that only
+ * asserts the top-level array + sample_id shape.
+ */
+declare function loadLoCoMo10(options: LoadDatasetOptions & {
+    parseFile?: (raw: string, filename: string) => LoCoMoConversation[];
+}): Promise<LoadedDataset<LoCoMoConversation>>;
+/**
+ * Build a friendly "dataset missing" error message that links operators to
+ * the fetch script. Callers use this when `mode === "full"` and the probe
+ * returned `source: "missing"`.
+ */
+declare function formatMissingDatasetError(benchmark: "longmemeval" | "locomo", datasetDir: string | undefined, filenames: readonly string[], errors: readonly string[]): string;
+
+/**
+ * BEAM runner migrated into @remnic/bench for phase 2.
+ */
+
+interface BeamDatasetPreview {
+    source: "dataset" | "smoke" | "missing";
+    files: string[];
+    items: number;
+    tasks: number;
+    errors: string[];
+}
+declare function loadBeamDatasetPreview(options: {
+    mode: BenchmarkMode;
+    datasetDir?: string;
+    limit?: number;
+}): Promise<BeamDatasetPreview>;
+
 /**
  * Hash verification utilities used by the benchmark integrity pipeline.
  *
@@ -1230,8 +2119,12 @@ declare function hashBytes(value: Uint8Array): string;
 /**
  * Canonicalize a JSON-serializable value so equivalent payloads produce the
  * same digest regardless of key insertion order.
+ *
+ * `space` matches the third argument of `JSON.stringify` — pass `2` (or any
+ * positive integer / indent string) when you want a pretty-printed output
+ * that is still byte-stable across runs. Default is compact output.
  */
-declare function canonicalJsonStringify(value: unknown): string;
+declare function canonicalJsonStringify(value: unknown, space?: string | number): string;
 declare function hashCanonicalJson(value: unknown): string;
 declare function isSha256Hex(value: unknown): value is string;
 declare function assertSha256Hex(value: unknown, label: string): string;
@@ -1284,9 +2177,10 @@ declare function loadSealKeyFromEnv(envName: string): Buffer | null;
  * }
  * ```
  *
- * `sealHash` is computed over the canonical JSON of `envelope` so two qrels
- * files encrypted with the same key produce distinct `sealHash` values only
- * when their plaintext differs.
+ * `sealHash` is computed over the canonical JSON of `envelope`, including the
+ * random IV and ciphertext. It identifies the sealed envelope artifact, not
+ * the plaintext qrels content. Use `envelope.plaintextHash` when stable
+ * plaintext identity is required across independently sealed artifacts.
  */
 
 interface SealedQrelsArtifact {
@@ -1393,7 +2287,7 @@ interface SeededRng {
  * Deterministic 32-bit PRNG. Mulberry32 is small, fast, and sufficient for
  * shuffling benchmark tasks. Do NOT use for cryptographic operations.
  */
-declare function createSeededRng(seed: number): SeededRng;
+declare function createSeededRng$1(seed: number): SeededRng;
 /**
  * Fisher-Yates shuffle using a seeded PRNG. Returns a new array.
  */
@@ -1576,89 +2470,6 @@ declare const chatFixture: FixtureGenerator;
 declare const SEALED_PROMPT_REGISTRY: Readonly<Record<string, string>>;
 declare const DEFAULT_ASSISTANT_RUBRIC_ID = "assistant-rubric-v1";
 
-/**
- * Shared types for the Assistant bench tier.
- *
- * Every Assistant benchmark shares the same shape:
- *   - A synthetic memory graph (facts, stances, entities) the agent may read.
- *   - A scenario prompt given to the agent.
- *   - A sealed-rubric judge pass that scores the agent's output along
- *     identity_accuracy / stance_coherence / novelty / calibration.
- *
- * The goal is reviewability: each benchmark folder ships a small fixture.ts
- * that returns `AssistantScenario` values, and the runner wires the shared
- * multi-run + bootstrap-CI infrastructure around them.
- */
-
-interface AssistantMemoryFact {
-    id: string;
-    summary: string;
-    /**
-     * Free-form tags (topic, entity) used to render the memory-graph summary
-     * that is handed to the judge. Not shown to the agent.
-     */
-    tags?: string[];
-}
-interface AssistantStance {
-    topic: string;
-    position: string;
-}
-interface AssistantMemoryGraph {
-    userHandle: string;
-    userRole: string;
-    facts: AssistantMemoryFact[];
-    stances: AssistantStance[];
-    openThreads: string[];
-}
-interface AssistantScenario {
-    id: string;
-    title: string;
-    scenarioPrompt: string;
-    memoryGraph: AssistantMemoryGraph;
-    /**
-     * Small label describing what the scenario is meant to exercise. Useful in
-     * dashboards for filtering. Never exposed to the agent.
-     */
-    focus: string;
-}
-/**
- * Minimal agent contract for the Assistant tier. The agent receives the
- * scenario prompt plus a pre-rendered memory view (analogous to what the
- * Remnic recall stack would hand to a downstream chat model), and returns
- * its final answer text.
- */
-interface AssistantAgent {
-    respond(request: {
-        scenarioId: string;
-        prompt: string;
-        memoryView: string;
-    }): Promise<string>;
-}
-interface AssistantRunnerOptions {
-    agent: AssistantAgent;
-    judge: StructuredJudge | undefined;
-    rubricId?: string;
-    /**
-     * Directory where per-run spot-check JSONL files are appended. Defaults to
-     * `<cwd>/benchmarks/results/spot-checks`.
-     */
-    spotCheckDir?: string;
-    /**
-     * Seed array for deterministic multi-run scheduling. When omitted the
-     * benchmark runner picks a fresh seed array via `buildBenchmarkRunSeeds`.
-     */
-    seeds?: number[];
-    /**
-     * Override used by tests and CLI smoke runs to cap iterations. Must be
-     * `>= 1`. The production contract is `>= 5` per the issue spec.
-     */
-    runCount?: number;
-    /**
-     * Random-number factory for bootstrap sampling. Injected in tests.
-     */
-    random?: () => number;
-}
-
 /**
  * Shared runner scaffolding for the Assistant bench tier.
  *
@@ -1757,4 +2568,598 @@ declare const ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS: AssistantScenario[];
 declare const assistantSynthesisDefinition: BenchmarkDefinition;
 declare function runAssistantSynthesisBenchmark(options: ResolvedRunBenchmarkOptions): Promise<BenchmarkResult>;
 
-export { ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkPublishFeedOptions, type BuiltInProvider, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, type CanaryAdapterOptions, type CanaryFloorCheck, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ASSISTANT_RUBRIC_ID, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, type LlmJudge, type LlmProvider, type LoadSealedQrelsOptions, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MultipleChoiceQuestion, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, type PersonalizationRetrievalCase, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RegressionDetail, type RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededRng, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TokenUsage, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, bootstrapMeanConfidenceInterval, buildBenchmarkPublishFeed, buildBenchmarkRunSeeds, buildJudgePayload, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, chatFixture, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeSealHash, containsAnswer, createAnthropicProvider, createCanaryAdapter, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createOllamaProvider, createOpenAiCompatibleProvider, createProvider, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, emailFixture, entityRecall, exactMatch, f1Score, generateReport, getBenchmark, getBenchmarkLowerIsBetter, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, llmJudgeScore, llmJudgeScoreDetailed, loadBaseline, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseCustomBenchmark, parseRubricResponse, parseSealedQrels, precisionAtK, projectFolderFixture, recallAtK, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBenchSuite, runBenchmark, runCustomBenchmarkFile, runExplain, runSealedJudge, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectFixtureVariant, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkPublishFeed, writeBenchmarkResult, zeroScores };
+interface ProceduralRecallE2eCase {
+    id: string;
+    prompt: string;
+    procedurePreamble: string;
+    procedureSteps: Array<{
+        order: number;
+        intent: string;
+    }>;
+    procedureTags: string[];
+    /** When true, `buildProcedureRecallSection` should return non-null markdown. */
+    expectNonNullSection: boolean;
+    proceduralEnabled?: boolean;
+}
+
+/**
+ * Scenario shape for the ablation harness. A superset of
+ * `ProceduralRecallE2eCase` with an `expectMatch` alias so downstream fixtures
+ * can be expressed in either vocabulary.
+ */
+interface ProceduralAblationScenario {
+    id: string;
+    prompt: string;
+    procedurePreamble: string;
+    procedureSteps: Array<{
+        order: number;
+        intent: string;
+    }>;
+    procedureTags: string[];
+    /**
+     * True when the prompt should recall the procedure. False for distractor /
+     * non-task-initiation prompts where we expect the gate to reject.
+     */
+    expectMatch: boolean;
+}
+interface ProceduralAblationPerCase {
+    id: string;
+    prompt: string;
+    expectMatch: boolean;
+    onMatched: boolean;
+    offMatched: boolean;
+    onScore: number;
+    offScore: number;
+}
+interface ProceduralAblationArtifact {
+    schemaVersion: 1;
+    fixture: {
+        path: string | null;
+        scenarioCount: number;
+    };
+    onScore: number;
+    offScore: number;
+    lift: number;
+    confidenceInterval: ConfidenceInterval;
+    perCase: ProceduralAblationPerCase[];
+    generatedAt: string;
+}
+/**
+ * Convert the existing `ProceduralRecallE2eCase` fixture into
+ * ablation-scenario shape. The ablation ALWAYS sweeps procedural on and off,
+ * so `expectMatch` must reflect what the prompt + procedure pair should do
+ * WHEN PROCEDURAL IS ON — not what the original row's `proceduralEnabled`
+ * flag produced.
+ *
+ * Gate-control rows in the e2e fixture (where `proceduralEnabled=false`
+ * produces `expectNonNullSection=false` only because of the gate, not the
+ * content) are excluded here: their ON-side outcome is content-dependent and
+ * not something this mapper can label correctly without re-running
+ * `buildProcedureRecallSection`. Callers that need those rows should write
+ * the scenario directly with an explicit `expectMatch`.
+ */
+declare function fixtureToAblationScenarios(fixture: ProceduralRecallE2eCase[]): ProceduralAblationScenario[];
+/**
+ * Default bootstrap seed used when no `random` / `seed` override is supplied.
+ * Fixing this makes CI bounds reproducible across CLI invocations — flaky CI
+ * bounds would break artifact-based comparisons and saved baselines.
+ */
+declare const DEFAULT_ABLATION_BOOTSTRAP_SEED = 1919249774;
+/**
+ * Mulberry32 seeded RNG. Inlined (and re-used from tests) so callers can get a
+ * deterministic default without needing an external dependency.
+ */
+declare function createSeededRandom(seed: number): () => number;
+interface RunProceduralAblationOptions {
+    scenarios: ProceduralAblationScenario[];
+    /** Path the ablation was loaded from (echoed back into the artifact). */
+    fixturePath?: string | null;
+    /** Bootstrap iterations for CI on the paired delta (default: 1_000). */
+    bootstrapIterations?: number;
+    /**
+     * Seeded RNG for the bootstrap. Defaults to
+     * `createSeededRandom(DEFAULT_ABLATION_BOOTSTRAP_SEED)` so CI bounds are
+     * deterministic across repeated CLI invocations. Pass `Math.random`
+     * explicitly to opt into non-deterministic sampling.
+     */
+    random?: () => number;
+    /**
+     * Convenience alternative to `random`: if provided (and `random` is not),
+     * a seeded mulberry32 RNG is built from this integer.
+     */
+    seed?: number;
+}
+/**
+ * Pure entrypoint — accepts a scenario list and returns the artifact. Reads
+ * and writes are isolated to the StorageManager temp directories the sides
+ * create and remove internally.
+ */
+declare function runProceduralAblation(options: RunProceduralAblationOptions): Promise<ProceduralAblationArtifact>;
+/**
+ * Load a scenario list from a JSON file. Validates the JSON is an object with
+ * a `scenarios` array (or a bare array) and each entry has the required
+ * fields. Rejects invalid input per CLAUDE.md rule 51 rather than silently
+ * defaulting.
+ */
+declare function loadAblationFixture(fixturePath: string): Promise<ProceduralAblationScenario[]>;
+/**
+ * CLI entrypoint. Resolves `--fixture <path>` (defaults to the built-in e2e
+ * fixture converted to ablation scenarios when unset) and writes the artifact
+ * to `--out <path>`. Validates inputs per CLAUDE.md rules 14 / 17 / 51.
+ */
+interface RunProceduralAblationCliArgs {
+    fixturePath: string | null;
+    outPath: string;
+    bootstrapIterations?: number;
+    random?: () => number;
+    /**
+     * Optional seed for the bootstrap RNG. When omitted the harness uses
+     * `DEFAULT_ABLATION_BOOTSTRAP_SEED` so CLI runs are reproducible by
+     * default.
+     */
+    seed?: number;
+}
+declare function runProceduralAblationCli(args: RunProceduralAblationCliArgs): Promise<ProceduralAblationArtifact>;
+
+/**
+ * Real-fixture procedural-recall scenarios (issue #567 PR 2/5).
+ *
+ * 20 synthetic but realistic scenarios grouped across four categories:
+ *
+ *   - exact-re-run          prompt matches a stored procedure near-verbatim;
+ *                           should recall when procedural is on.
+ *   - parameter-variation   prompt references the same intent with different
+ *                           nouns (service name, environment, ticket id);
+ *                           should recall on overlap + intent compatibility.
+ *   - decomposition         prompt starts a multi-step task whose steps match
+ *                           a stored runbook; should recall.
+ *   - distractor-rejection  prompt looks task-like but the stored procedure
+ *                           is unrelated — the gate should REJECT (expectMatch
+ *                           = false).
+ *
+ * All scenarios are deterministic and use ONLY token-overlap + intent
+ * classification semantics (no LLM calls). The deterministic stub LLM
+ * requirement from #567 is satisfied because `buildProcedureRecallSection`
+ * is a pure function of storage + prompt + config. A human runbook for
+ * exercising the gpt-4o-mini path lives in docs/benchmarks/procedural-recall.md.
+ *
+ * Scenarios are synthetic (no personal data), per CLAUDE.md public-repo
+ * privacy policy.
+ */
+
+type ProceduralRealScenarioCategory = "exact-re-run" | "parameter-variation" | "decomposition" | "distractor-rejection";
+interface ProceduralRealScenario extends ProceduralAblationScenario {
+    category: ProceduralRealScenarioCategory;
+    notes?: string;
+}
+declare const PROCEDURAL_REAL_SCENARIOS: ProceduralRealScenario[];
+/** Built-in smoke slice (first scenario from each category). */
+declare const PROCEDURAL_REAL_SCENARIOS_SMOKE: ProceduralRealScenario[];
+
+/**
+ * Types for the ADAM-style memory-extraction attack harness.
+ *
+ * See docs/security/memory-extraction-threat-model.md for the threat model
+ * this harness probes. The harness targets the modeled read-path surfaces
+ * enumerated in §4 of that document (recall / memory_search /
+ * memory_entities_list / …), driven in-process against a seeded memory
+ * fixture so tests do not need a running daemon.
+ */
+/**
+ * Attacker knowledge tier the harness simulates.
+ *
+ * These correspond to the T1/T2/T3 tiers in the threat model (§3):
+ *
+ * - `zero-knowledge` — attacker has no prior information about the memory
+ *   contents, must probe purely from seed vocabulary. Roughly T1 with a
+ *   minimal token (or T2 on a newly-provisioned namespace).
+ * - `same-namespace` — attacker holds a valid token for the same namespace as
+ *   the seeded memories and may use entity-graph enumeration as side channel.
+ *   This is the primary T2 tier.
+ * - `cross-namespace` — attacker holds a valid token for a *different*
+ *   namespace but attempts to leak memories from the victim namespace via
+ *   shared-namespace auto-promotion or debug snapshots (T3).
+ */
+type AttackerMode = "zero-knowledge" | "same-namespace" | "cross-namespace";
+/**
+ * A single seeded memory the harness treats as ground truth.
+ *
+ * Ground-truth labelling is intentionally coarse: `tokens` is the set of
+ * salient lowercase tokens that uniquely identify this memory to a human.
+ * A recovered query transcript is considered to have leaked the memory if
+ * the response contains a substring that covers a configurable fraction of
+ * these tokens (see `recoveryTokenOverlap`).
+ */
+interface SeededMemory {
+    /** Stable identifier. */
+    id: string;
+    /** Raw memory text as it would be stored. */
+    content: string;
+    /**
+     * Category bucket (fact / preference / decision / entity / …). Mirrors the
+     * buckets the threat model lists in §2 (Assets). Used by the harness only
+     * for reporting; not used in the attack loop itself.
+     */
+    category: "fact" | "preference" | "decision" | "entity" | "other";
+    /** Namespace the memory lives in. */
+    namespace: string;
+    /**
+     * Optional set of salient tokens that define "the attacker recovered this
+     * memory". If omitted, defaults to all alphanumeric tokens of length > 2 in
+     * `content`.
+     */
+    tokens?: string[];
+}
+/**
+ * One retrieval result returned by the target surface.
+ *
+ * The shape is intentionally narrower than `MemoryRecord` in core — the
+ * harness only needs the attacker-observable subset.
+ */
+interface AttackRetrievalHit {
+    /**
+     * Stable memory identifier if the surface exposes one. Attackers can use
+     * this as a side channel (memory IDs are disclosed by recall responses in
+     * the current MCP surface), so we model it explicitly.
+     */
+    memoryId?: string;
+    /** Namespace the memory came from, if the surface discloses it. */
+    namespace?: string;
+    /** Text content (summary or full) the surface returned. */
+    content: string;
+    /** Optional relevance score. */
+    score?: number;
+}
+/**
+ * Minimal contract an attack target must satisfy.
+ *
+ * Callers wrap the real `EngramAccessService.recall()` or a test stub. The
+ * harness deliberately does not depend on `@remnic/core` directly; PR 3
+ * will provide the binding to the real orchestrator.
+ */
+interface ExtractionAttackTarget {
+    /**
+     * Execute a recall query against the target and return its hits.
+     *
+     * Should throw (or return an empty array) when the target denies the
+     * query — the harness treats both as "no information gained".
+     */
+    recall(query: string, options?: AttackRecallOptions): Promise<AttackRetrievalHit[]>;
+    /**
+     * Optional side channel: enumerate known entity names. Present iff the
+     * attacker mode was granted access to `memory_entities_list`.
+     */
+    listEntities?(): Promise<string[]>;
+}
+interface AttackRecallOptions {
+    /** Top-K to request. Defaults to harness budget. */
+    topK?: number;
+    /** Namespace override (harness uses this for T3 cross-namespace probes). */
+    namespace?: string;
+}
+/**
+ * Deterministic PRNG interface. Callers can pass a seeded PRNG to make runs
+ * reproducible.
+ */
+interface HarnessRng {
+    /** Returns a float in [0, 1). */
+    next(): number;
+}
+/**
+ * Configuration for a single harness run.
+ */
+interface ExtractionAttackOptions {
+    target: ExtractionAttackTarget;
+    /** Ground-truth memories the harness is trying to recover. */
+    groundTruth: readonly SeededMemory[];
+    /** Attacker knowledge tier. */
+    attackerMode: AttackerMode;
+    /** Maximum number of recall queries the harness is allowed to issue. */
+    queryBudget: number;
+    /**
+     * Hyper-parameter for the adaptive loop: when the normalized entropy of
+     * the response distribution falls below this value, the harness switches
+     * to exploitation (repeating high-signal tokens) rather than exploration.
+     * Defaults to 0.3.
+     */
+    entropyThreshold?: number;
+    /**
+     * Deterministic RNG. Callers should pass a seeded instance to get
+     * reproducible results.
+     */
+    rng?: HarnessRng;
+    /**
+     * Seed vocabulary the attacker starts from. In `zero-knowledge` mode this
+     * is the *only* prior information; in `same-namespace` mode it is a seed
+     * for exploration. Defaults to a small English stop-list plus common
+     * personal-memory topics.
+     */
+    seedVocabulary?: readonly string[];
+    /**
+     * Fraction of ground-truth tokens a single retrieved response must cover
+     * to count as "recovered". Defaults to 0.5.
+     */
+    recoveryTokenOverlap?: number;
+    /** If true, every query and response is kept in `timeline`. */
+    captureTimeline?: boolean;
+    /** TopK to request per query. Defaults to 10. */
+    topK?: number;
+    /**
+     * Namespace the attacker addresses their queries to. When set, every
+     * `target.recall()` is invoked with `namespace: attackerNamespace`.
+     * Useful for T3-class runs where the caller wants to simulate an
+     * attacker holding a token for a specific cross-namespace tenant.
+     *
+     * Defaults:
+     * - `zero-knowledge`: undefined (target uses its own default).
+     * - `same-namespace`: undefined (target uses its own default).
+     * - `cross-namespace`: `"shared"` (matches the residual-leak path the
+     *   threat model calls out in §5, but callers targeting other namespace
+     *   models should pass an explicit value here).
+     */
+    attackerNamespace?: string;
+    /**
+     * Optional absolute deadline in ms since epoch. If the harness crosses it
+     * during the attack loop, it terminates early with a partial result. Used
+     * by tests to keep runs bounded.
+     */
+    deadlineMs?: number;
+    /**
+     * When true, a thrown error from `target.recall` aborts the attack
+     * loop and re-throws. Default false — errors are counted in
+     * `ExtractionAttackResult.backendErrorCount` so callers can distinguish
+     * genuine empty recalls from backend failures. Flip to true in CI
+     * gating scripts that must not silently publish ASR from a degraded
+     * target.
+     */
+    failOnBackendError?: boolean;
+}
+interface RecoveredMemory {
+    memoryId: string;
+    memory: SeededMemory;
+    recoveredContent: string;
+    queriesUsed: number;
+    /** Index into `timeline` that first recovered this memory. */
+    firstHitAt: number;
+}
+interface TimelineEntry {
+    query: string;
+    hits: AttackRetrievalHit[];
+    entropy: number;
+    newlyRecoveredMemoryIds: string[];
+    /** Which strategy chose this query. Useful for diagnosing the algorithm. */
+    strategy: "seed" | "exploit-entity" | "exploit-token" | "explore-random" | "explore-entropy";
+}
+interface ExtractionAttackResult {
+    /** Attack Success Rate: fraction of ground-truth memories recovered. */
+    asr: number;
+    /** Number of queries issued (may be less than budget on early exit). */
+    queriesIssued: number;
+    /** Attacker mode this run simulated. */
+    attackerMode: AttackerMode;
+    /** Recovered memories with per-memory metadata. */
+    recovered: RecoveredMemory[];
+    /** Ground-truth memories the attacker failed to recover within budget. */
+    missed: SeededMemory[];
+    /** Full query-by-query trace. Empty unless `captureTimeline: true`. */
+    timeline: TimelineEntry[];
+    /** Seconds of wall time spent inside the attack loop. */
+    durationMs: number;
+    /** True iff the run stopped because `deadlineMs` was reached. */
+    hitDeadline: boolean;
+    /**
+     * Number of `target.recall` calls that threw and were treated as empty
+     * hits. A high value means the harness was talking to a degraded
+     * backend — low/zero ASR in that case is not a security statement
+     * about the system, it is a measurement failure. Callers that want to
+     * fail-fast on backend errors can pass `failOnBackendError: true`.
+     */
+    backendErrorCount: number;
+}
+
+/**
+ * ADAM-style entropy-guided memory-extraction attack harness.
+ *
+ * Re-implements the entropy-guided adaptive querying strategy described in
+ * ADAM (arXiv:2604.09747, Apr 2026): the attacker issues a sequence of
+ * recall queries, observes the information gained from each response, and
+ * picks the next query to maximize expected entropy reduction over the
+ * remaining candidate memories.
+ *
+ * This is a clean-room re-implementation, not a port of any released
+ * codebase. The algorithm is:
+ *
+ *   1. Initialize candidate-token pool from the seed vocabulary and (mode
+ *      permitting) side channels like entity listings.
+ *   2. Loop until budget exhausted or all memories recovered:
+ *      a. Compute Shannon entropy over the attacker's current belief
+ *         distribution (`tokenFrequencies`). Low entropy => we have a
+ *         concentrated belief; exploit by querying the top tokens. High
+ *         entropy => we are uncertain; explore by querying a token we have
+ *         not tried yet.
+ *      b. Issue the chosen query against the target.
+ *      c. Update the belief from the response: tokens appearing in hit
+ *         content get their frequency bumped; tokens that keep appearing
+ *         alongside already-recovered content get deprioritized so the
+ *         attacker does not re-query the same region forever.
+ *      d. Check each hit against the ground-truth set; mark recoveries.
+ *   3. Emit ASR + per-memory leak log.
+ *
+ * The harness never calls an LLM. Token "information gain" is computed from
+ * lexical overlap against previously-seen hits; this is the non-LLM
+ * approximation the paper uses for its low-cost variant, and is adequate
+ * for measuring the attack surface's structural leakage.
+ */
+
+/**
+ * Tiny mulberry32 PRNG — stable across Node versions.
+ */
+declare function createSeededRng(seed: number): HarnessRng;
+/**
+ * Entry point. See `types.ts` for the options contract.
+ */
+declare function runExtractionAttack(options: ExtractionAttackOptions): Promise<ExtractionAttackResult>;
+
+/**
+ * Synthetic memory fixture and in-process target implementation for the
+ * extraction-attack harness tests.
+ *
+ * Everything in this file is synthetic. Per the public-repo privacy policy
+ * in CLAUDE.md, no real user data may ship in fixtures.
+ */
+
+/**
+ * 15 synthetic seeded memories covering fact, preference, decision, and
+ * entity categories across two namespaces. Intentionally mundane so no
+ * reader mistakes any of this for real personal data.
+ */
+declare const SYNTHETIC_MEMORIES: readonly SeededMemory[];
+/**
+ * A second namespace used for the cross-namespace test. This namespace's
+ * contents MUST stay separate from `victim` — the T3 attack should fail if
+ * the surface honors ACLs.
+ */
+declare const OTHER_NAMESPACE_MEMORIES: readonly SeededMemory[];
+interface SyntheticTargetOptions {
+    /** Memories visible through normal recall. */
+    memories: readonly SeededMemory[];
+    /** Entities the side channel should enumerate. */
+    entities?: readonly string[];
+    /**
+     * When true, the target enforces namespace ACLs: a recall with a namespace
+     * other than `allowedNamespace` returns an empty array. Models the T3
+     * mitigation path in the threat model §6.1.
+     */
+    enforceNamespaceAcl?: boolean;
+    /** The only namespace the attacker is entitled to read. */
+    allowedNamespace?: string;
+    /**
+     * When true, the target always includes memory IDs in hits. When false,
+     * the target masks IDs (models a mitigation where recall responses no
+     * longer disclose memory IDs as a side channel).
+     */
+    disclosesMemoryIds?: boolean;
+    /**
+     * How many hits to return per recall. Mirrors QMD behavior where cutoff
+     * is based on score threshold. Defaults to 5.
+     */
+    hitCap?: number;
+}
+/**
+ * Deterministic in-process target. Scores memories by token-overlap with the
+ * query and returns the top-K above zero.
+ */
+declare function createSyntheticTarget(options: SyntheticTargetOptions): ExtractionAttackTarget;
+
+/**
+ * Baseline measurement runner for the ADAM memory-extraction harness.
+ *
+ * Produces a reproducible set of ASR numbers for every attacker tier against
+ * a synthetic target that mirrors the current Remnic read-path behavior
+ * (memory IDs disclosed, namespace ACL enforced on cross-namespace reads).
+ *
+ * This is intentionally separate from the unit tests: tests keep budgets
+ * small so CI stays fast, whereas the baseline pushes the budget high enough
+ * for each tier to plateau. The output feeds into
+ * `docs/security/adam-baseline-2026-04.md`.
+ */
+
+interface BaselineScenario {
+    readonly name: string;
+    readonly attackerMode: AttackerMode;
+    readonly queryBudget: number;
+    readonly seed: number;
+    /** Ground truth the attacker is trying to recover. */
+    readonly groundTruth: readonly SeededMemory[];
+    /** Memories the target actually stores (may be a superset of groundTruth). */
+    readonly targetMemories: readonly SeededMemory[];
+    readonly entities?: readonly string[];
+    readonly enforceNamespaceAcl?: boolean;
+    readonly allowedNamespace?: string;
+    readonly disclosesMemoryIds?: boolean;
+    /** Attacker-held namespace. Forwarded as `attackerNamespace` to the runner. */
+    readonly attackerNamespace?: string;
+}
+interface BaselineRow {
+    readonly scenario: string;
+    readonly attackerMode: AttackerMode;
+    readonly queryBudget: number;
+    readonly queriesIssued: number;
+    readonly asr: number;
+    readonly recoveredIds: readonly string[];
+    readonly missedIds: readonly string[];
+    readonly durationMs: number;
+    /** Whether mitigations were active during this run. */
+    readonly mitigated?: boolean;
+}
+/**
+ * Scenarios used for the 2026-04 baseline. Kept deterministic via fixed seeds
+ * so the document remains reproducible.
+ */
+declare const DEFAULT_BASELINE_SCENARIOS: readonly BaselineScenario[];
+/**
+ * Executes every scenario once and returns a flat set of rows suitable for
+ * rendering as a markdown table.
+ */
+declare function runBaseline(scenarios?: readonly BaselineScenario[]): Promise<BaselineRow[]>;
+interface MitigatedBaselineConfig {
+    budgetHardLimit: number;
+    budgetWindowMs?: number;
+    /**
+     * Override for the principal's "home" namespace in the mitigated target.
+     * When set, this is passed as `principalNamespace` to `createMitigatedTarget`.
+     * When unset, falls back to `allowedNamespace ?? "default"`.
+     * Use this to decouple the budget's principal identity from the synthetic
+     * target's ACL namespace.
+     */
+    principalNamespaceOverride?: string;
+}
+declare const MITIGATED_BASELINE_SCENARIOS: readonly (BaselineScenario & MitigatedBaselineConfig)[];
+declare function runMitigatedBaseline(scenarios?: readonly (BaselineScenario & MitigatedBaselineConfig)[]): Promise<BaselineRow[]>;
+/**
+ * Renders a baseline run as a human-readable markdown fragment. The returned
+ * string is suitable for pasting into the baseline document.
+ */
+declare function renderBaselineMarkdown(rows: readonly BaselineRow[]): string;
+
+/**
+ * Mitigation-aware target wrapper for the ADAM extraction attack harness.
+ *
+ * Wraps a raw `ExtractionAttackTarget` and enforces:
+ * 1. Cross-namespace query budget (mirrors `CrossNamespaceBudget` from core)
+ * 2. Namespace ACL (carries forward from `createSyntheticTarget`)
+ *
+ * When the budget is exceeded, the wrapper returns empty hits instead of
+ * forwarding the query — simulating the real recall-path denial. This lets
+ * the harness re-measure ASR with mitigations active and compare against
+ * the unmitigated baseline.
+ */
+
+interface MitigatedTargetConfig {
+    /** Inner (unmitigated) target to wrap. */
+    target: ExtractionAttackTarget;
+    /**
+     * Maximum cross-namespace queries per `budgetWindowMs` window.
+     * Queries beyond this limit return empty hits.
+     */
+    budgetHardLimit: number;
+    /**
+     * Rolling window in ms for the budget counter. Defaults to 60_000.
+     */
+    budgetWindowMs?: number;
+    /**
+     * The principal's "home" namespace. Queries targeting a different
+     * namespace count against the budget; same-namespace queries are free.
+     */
+    principalNamespace: string;
+}
+/**
+ * Creates a mitigation-aware wrapper around a raw target.
+ *
+ * The wrapper tracks cross-namespace queries in a sliding window and
+ * returns empty hits when the budget is exceeded. Same-namespace queries
+ * pass through without counting.
+ */
+declare function createMitigatedTarget(config: MitigatedTargetConfig): ExtractionAttackTarget;
+
+export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, type CanaryAdapterOptions, type CanaryFloorCheck, type CodexCliProviderConfig, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MITIGATED_BASELINE_SCENARIOS, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEmailIngestionAdapterOptions, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TimelineEntry, type TokenUsage, type WriteBenchmarkArtifactResult, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, chatFixture, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeSealHash, containsAnswer, createSeededRng as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createCodexCliProvider, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$1 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, emailFixture, entityRecall, exactMatch, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatMissingDatasetError, generateReport, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadLoCoMo10, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseRubricResponse, parseSealedQrels, precisionAtK, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeLeaderboardArtifactsForResult, zeroScores };