@remnic/bench 9.3.729 → 9.3.731

package/dist/index.d.ts

@@ -225,8 +225,14 @@ type AmaBenchJudgeProtocol = "default" | "recommended";
  * `codex-cli` shells out to `codex exec` as an isolated benchmark-only
  * responder/judge target. It is intentionally not routed through Remnic
  * memory or OpenClaw gateway state.
+ *
+ * `claude-cli` shells out to `claude -p` (Claude Code headless) as an
+ * isolated benchmark-only responder/judge target, running against the
+ * operator's Claude subscription rather than a metered API key. Like
+ * `codex-cli`, it is intentionally not routed through Remnic memory or
+ * OpenClaw gateway state.
  */
-type BuiltInProvider = "openai" | "anthropic" | "ollama" | "litellm" | "local-llm" | "codex-cli";
+type BuiltInProvider = "openai" | "anthropic" | "ollama" | "litellm" | "local-llm" | "codex-cli" | "claude-cli";
 type BenchReasoningEffort = "low" | "medium" | "high" | "xhigh";
 interface ProviderConfig {
     provider: BuiltInProvider;
@@ -569,6 +575,280 @@ interface MemCorrectSystemAdapter {
     runMaintenance(): Promise<void>;
 }
 
+/**
+ * Shared infrastructure for third-party MemCorrect adapters (issue #1727).
+ *
+ * Each third-party memory system (Mem0, Zep, Letta) gets a MemCorrectSystemAdapter
+ * implementation that talks to its public HTTP API. These adapters exist so the
+ * paper can substantiate head-to-head claims: without them, "we beat X" is
+ * unsubstantiable.
+ *
+ * Design constraints (from the issue):
+ *
+ *   1. **No keys in CI.** Adapters require operator-provided credentials
+ *      (API key + endpoint) to run. Without them, every method throws
+ *      `MissingCredentialError` — which the bench harness treats as a
+ *      skip-with-reason, not a test failure. No network calls happen in CI.
+ *
+ *   2. **Injectable transport.** Each adapter accepts a `fetch` override so the
+ *      deterministic fixture smoke test can drive the full request/response
+ *      cycle without touching the network. The production path uses the global
+ *      `fetch`.
+ *
+ *   3. **Faithful to each system's normal path.** The adapter calls the same
+ *      public endpoints an integrator would use in production. Reaching into
+ *      internals would make the comparison meaningless (the issue's hard
+ *      constraint, inherited from #1584).
+ */
+
+/** Injectable fetch — matches the global `fetch` signature. */
+type FetchLike = typeof fetch;
+/** Common configuration every third-party adapter accepts. */
+interface ThirdPartyAdapterConfig {
+    /** Auth token / API key. Required to actually run (operator-provided). */
+    apiKey?: string;
+    /** Base URL of the deployment. Required for self-hosted; optional for hosted. */
+    baseUrl?: string;
+    /** Injectable fetch for deterministic testing. Defaults to global `fetch`. */
+    fetch?: FetchLike;
+    /** Per-request timeout in milliseconds. */
+    timeoutMs?: number;
+}
+/**
+ * Thrown when an adapter is asked to run without the operator-provided
+ * credentials it needs. The bench harness treats this as a skip-with-reason,
+ * not a failure — no third-party system is reachable in CI.
+ */
+declare class MissingCredentialError extends Error {
+    readonly reason: string;
+    constructor(system: string, missing: string[]);
+}
+
+/**
+ * Mem0 MemCorrect adapter (issue #1727 — highest-priority third-party adapter).
+ *
+ * Drives Mem0 through its public REST API so MemCorrect can score Mem0 on the
+ * same correction/steerability corpus as the Remnic adapter. This is the
+ * single highest-leverage missing comparison piece: Mem0 is the most-cited
+ * memory system, and "we beat Mem0 on non-resurrection" is unsubstantiable
+ * without this adapter.
+ *
+ * Two deployment modes are supported:
+ *
+ *   - **OSS self-hosted** (`mode: "oss"`): synchronous REST server (FastAPI).
+ *     Paths have no `/v1/` prefix: `POST /memories`, `POST /search`,
+ *     `DELETE /memories?user_id=…`. Operators self-host for reproducible
+ *     benchmarking — they control the deployment, the LLM, and there is no
+ *     rate limiting. This is the recommended mode for lab runs.
+ *
+ *   - **Hosted platform** (`mode: "hosted"`): `api.mem0.ai` with the V3
+ *     async pipeline. `POST /v3/memories/add/` returns an `event_id` that is
+ *     polled until `SUCCEEDED`. Search uses `POST /v3/memories/search/`.
+ *
+ * The adapter accepts an injectable `fetch` so the deterministic fixture
+ * smoke test exercises the full request/response cycle without a network.
+ * No keys are embedded; without operator-provided credentials every method
+ * throws `MissingCredentialError` (skip-with-reason — no keys in CI).
+ */
+
+interface Mem0AdapterConfig extends ThirdPartyAdapterConfig {
+    /**
+     * Deployment mode.
+     * - `"oss"` — self-hosted synchronous server (recommended for lab runs).
+     * - `"hosted"` — api.mem0.ai V3 async pipeline.
+     *
+     * Defaults to `"oss"` when a custom `baseUrl` is set, `"hosted"` otherwise.
+     */
+    mode?: "oss" | "hosted";
+    /** Prefix prepended to sessionKeys to namespace Mem0 user_ids. */
+    userIdPrefix?: string;
+    /**
+     * OSS authentication header mode.
+     * - "x-api-key" (default): send the API key as X-API-Key. Mem0's self-hosted
+     *   REST auth uses X-API-Key for per-user/API keys (m0sk_…, ADMIN_API_KEY);
+     *   Authorization: Bearer is reserved for dashboard JWTs
+     *   (https://docs.mem0.ai/open-source/features/rest-api#authentication).
+     * - "bearer": send Authorization: Bearer for operators who deploy JWT auth.
+     */
+    ossAuthMode?: "x-api-key" | "bearer";
+    /** Hosted-mode: interval between event-status polls (ms). */
+    pollIntervalMs?: number;
+    /** Hosted-mode: maximum poll attempts before timing out. */
+    maxPolls?: number;
+}
+/**
+ * MemCorrect adapter for Mem0. Construct with operator-provided credentials;
+ * pass a `fetch` override for deterministic testing.
+ *
+ * @example
+ * // Lab run (operator provides keys):
+ * const adapter = new Mem0MemCorrectAdapter({
+ *   mode: "oss",
+ *   baseUrl: process.env.MEM0_BASE_URL,
+ *   apiKey: process.env.MEM0_API_KEY,
+ * });
+ *
+ * @example
+ * // Keyless — every method throws MissingCredentialError (skip-with-reason):
+ * const adapter = new Mem0MemCorrectAdapter({});
+ */
+declare class Mem0MemCorrectAdapter implements MemCorrectSystemAdapter {
+    readonly label: string;
+    private readonly mode;
+    private readonly baseUrl;
+    private readonly apiKey;
+    private readonly userIdPrefix;
+    private readonly ossAuthMode;
+    private readonly pollIntervalMs;
+    private readonly maxPolls;
+    private readonly fetchImpl;
+    private readonly timeoutMs;
+    /** Session-scoped user_ids we have ingested under, for precise reset. */
+    private readonly knownSessions;
+    constructor(config?: Mem0AdapterConfig);
+    /** Whether this adapter has the credentials needed to run. */
+    isConfigured(): boolean;
+    private userIdFor;
+    private ensureReady;
+    private authHeaders;
+    reset(): Promise<void>;
+    ingestTurn(sessionKey: string, role: "user" | "assistant", text: string, _at: string): Promise<void>;
+    recall(query: string, sessionKey: string): Promise<string[]>;
+    correct(text: string, sessionKey: string, _at?: string): Promise<void>;
+    runMaintenance(): Promise<void>;
+    /** Poll the hosted event endpoint until the add is processed. */
+    private pollEvent;
+}
+
+/**
+ * Zep MemCorrect adapter (issue #1727 — second priority).
+ *
+ * Drives Zep through its public v2 REST API so MemCorrect can score Zep on the
+ * same correction/steerability corpus as the Remnic adapter.
+ *
+ * Zep's model: sessions are the ingestion unit; `memory.add` ingests chat
+ * messages into a session and builds a user-level knowledge graph; `memory.get`
+ * retrieves a relevance-ranked context string for the prompt. This adapter
+ * uses the documented high-level Memory API — the same path a Zep integrator
+ * follows in production. Reaching into the graph internals would not be a
+ * faithful exercise of Zep's normal recall path.
+ *
+ * REST endpoints used (base: `https://api.getzep.com/api/v2`):
+ *   - `POST   /sessions/{sessionId}`            — ensure session exists
+ *   - `POST   /sessions/{sessionId}/memory`     — add messages (ingest + correct)
+ *   - `POST   /graph/search`                    — query-driven fact recall
+ *   - `DELETE /sessions/{sessionId}`            — full clean slate (reset)
+ *
+ * The adapter accepts an injectable `fetch` so the deterministic fixture smoke
+ * test exercises the full request/response cycle without a network. No keys
+ * are embedded; without operator-provided credentials every method throws
+ * `MissingCredentialError` (skip-with-reason — no keys in CI).
+ */
+
+interface ZepAdapterConfig extends ThirdPartyAdapterConfig {
+    /** Prefix prepended to sessionKeys to namespace Zep session IDs. */
+    sessionPrefix?: string;
+    /**
+     * Milliseconds to wait for Zep's asynchronous graph processing to extract
+     * facts after an ingest before a scored probe reads them. Applied at the
+     * ingest→probe boundary (the MemCorrect runner records the baseline recall
+     * right after establishing turns and the uptake recall right after correct(),
+     * both before runMaintenance), and again in runMaintenance. Zep docs note
+     * ingestion "can take a few minutes"; this tunable makes scored reads
+     * reproducible. Default 0 (best-effort; raise for real Zep runs).
+     */
+    settleMs?: number;
+}
+/**
+ * MemCorrect adapter for Zep. Construct with operator-provided credentials;
+ * pass a `fetch` override for deterministic testing.
+ */
+declare class ZepMemCorrectAdapter implements MemCorrectSystemAdapter {
+    readonly label = "zep";
+    private readonly baseUrl;
+    private readonly apiKey;
+    private readonly sessionPrefix;
+    private readonly settleMs;
+    private readonly fetchImpl;
+    private readonly timeoutMs;
+    /** Sessions we have ensured exist, to avoid redundant POST /sessions calls. */
+    private readonly knownSessions;
+    /** True when turns have been ingested since the last settle, so recall() can
+     * wait for Zep's async graph pipeline before a scored read. */
+    private pendingIngest;
+    constructor(config?: ZepAdapterConfig);
+    isConfigured(): boolean;
+    private sessionIdFor;
+    private ensureReady;
+    private authHeaders;
+    reset(): Promise<void>;
+    /** Ensure the Zep session (and its user) exist before adding memory. */
+    private ensureSession;
+    ingestTurn(sessionKey: string, role: "user" | "assistant", text: string, _at: string): Promise<void>;
+    recall(query: string, sessionKey: string): Promise<string[]>;
+    correct(text: string, sessionKey: string, _at?: string): Promise<void>;
+    runMaintenance(): Promise<void>;
+}
+
+/**
+ * Letta MemCorrect adapter (issue #1727 — third priority).
+ *
+ * Drives Letta (formerly MemGPT) through its public REST API so MemCorrect
+ * can score Letta on the same correction/steerability corpus.
+ *
+ * Letta's model: stateful agents with editable memory blocks. Each
+ * MemCorrect session maps to one Letta agent. Messages are sent through
+ * `POST /v1/agents/{id}/messages`; the agent autonomously updates its memory
+ * blocks via the built-in `memory_insert` / `memory_replace` tools. Recall
+ * reads the memory blocks back — this is Letta's normal memory surface, not
+ * an internal hack.
+ *
+ * REST endpoints used (base: operator-provided Letta server URL):
+ *   - `POST   /v1/agents/`                    — create a stateful agent
+ *   - `POST   /v1/agents/{id}/messages`       — send a user message (ingest + correct)
+ *   - `GET    /v1/agents/{id}/core-memory/blocks` — read memory blocks (recall)
+ *   - `DELETE /v1/agents/{id}`                — destroy agent (reset)
+ *
+ * The adapter accepts an injectable `fetch` so the deterministic fixture
+ * smoke test exercises the full request/response cycle without a network.
+ */
+
+interface LettaAdapterConfig extends ThirdPartyAdapterConfig {
+    /** LLM model handle for created agents (e.g. "openai/gpt-4o"). Required. */
+    model?: string;
+    /** Prefix for agent names to namespace them from other Letta agents. */
+    agentNamePrefix?: string;
+    /** Persona block content for created agents. */
+    personaBlock?: string;
+}
+/**
+ * MemCorrect adapter for Letta. Construct with operator-provided credentials;
+ * pass a `fetch` override for deterministic testing.
+ */
+declare class LettaMemCorrectAdapter implements MemCorrectSystemAdapter {
+    readonly label = "letta";
+    private readonly baseUrl;
+    private readonly apiKey;
+    private readonly model;
+    private readonly agentNamePrefix;
+    private readonly personaBlock;
+    private readonly fetchImpl;
+    private readonly timeoutMs;
+    /** Maps MemCorrect sessionKey → Letta agent_id. */
+    private readonly agentsBySession;
+    constructor(config?: LettaAdapterConfig);
+    isConfigured(): boolean;
+    private ensureReady;
+    private authHeaders;
+    reset(): Promise<void>;
+    /** Create a Letta agent for the session if one does not yet exist. */
+    private ensureAgent;
+    ingestTurn(sessionKey: string, role: "user" | "assistant", text: string, _at: string): Promise<void>;
+    recall(_query: string, sessionKey: string): Promise<string[]>;
+    correct(text: string, sessionKey: string, _at?: string): Promise<void>;
+    runMaintenance(): Promise<void>;
+}
+
 /**
  * Shared types for inbox fixture generators.
  */
@@ -766,6 +1046,19 @@ interface CodexCliProviderConfig extends ProviderBaseConfig {
      */
     diagnosticsMode?: "metadata" | "full";
 }
+interface ClaudeCliProviderConfig extends ProviderBaseConfig {
+    provider?: "claude-cli";
+    /** Optional executable override for tests or non-standard Claude Code installs. */
+    executable?: string;
+    /**
+     * Max concurrent `claude -p` invocations. Defaults to 1 (fully
+     * serialized) — the operator's Claude Max plan enforces shared 5-hour
+     * and weekly usage caps, so the bench harness must not fire concurrent
+     * CLI invocations against it even when the harness's own trial
+     * concurrency setting is higher.
+     */
+    concurrency?: number;
+}
 type ProviderFactoryConfig = (OpenAiCompatibleProviderConfig & {
     provider: "openai" | "litellm";
 }) | (AnthropicProviderConfig & {
@@ -776,6 +1069,8 @@ type ProviderFactoryConfig = (OpenAiCompatibleProviderConfig & {
     provider: "local-llm";
 }) | (CodexCliProviderConfig & {
     provider: "codex-cli";
+}) | (ClaudeCliProviderConfig & {
+    provider: "claude-cli";
 });
 interface ProviderDiscoveryResult {
     provider: BuiltInProvider;
@@ -1378,6 +1673,30 @@ declare function loadBenchmarkArtifact(filePath: string): Promise<{
 
 declare function createAnthropicProvider(config: AnthropicProviderConfig): LlmProvider;
 
+interface ClaudeCliRunRequest {
+    executable: string;
+    args: string[];
+    input: string;
+    cwd: string;
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    env: NodeJS.ProcessEnv;
+}
+interface ClaudeCliRunResult {
+    status: number | null;
+    signal: NodeJS.Signals | null;
+    stdout: string;
+    stderr: string;
+}
+interface ClaudeCliProviderDeps {
+    runClaudeCli?: (request: ClaudeCliRunRequest) => Promise<ClaudeCliRunResult>;
+    runClaudeVersion?: (executable: string, env: NodeJS.ProcessEnv) => Promise<{
+        status: number | null;
+        stderr: string;
+    }>;
+}
+declare function createClaudeCliProvider(config: ClaudeCliProviderConfig, deps?: ClaudeCliProviderDeps): LlmProvider;
+
 interface CodexCliRunRequest {
     executable: string;
     args: string[];
@@ -3412,6 +3731,88 @@ interface RunProceduralAblationCliArgs {
 }
 declare function runProceduralAblationCli(args: RunProceduralAblationCliArgs): Promise<ProceduralAblationArtifact>;
 
+/**
+ * Single-flag ablation matrix for the published-benchmark ablation suite
+ * (issues #1574 §"Ablations" and #1730).
+ *
+ * Each ablation is a reproducible run config: a named cell that flips exactly
+ * one Remnic config flag relative to the {@link buildBenchBaselineRemnicConfig}
+ * baseline, so the delta against the matching baseline artifact isolates that
+ * flag's effect. The matrix is pure data — no I/O — so it is trivially
+ * testable and the runner script (`scripts/bench/run-ablation-matrix.ts`) is
+ * just a thin shell over the public bench API.
+ *
+ * The three axes come straight from #1574's "Ablations" section:
+ *   1. Memory Worth recall multiplier (`recallMemoryWorthFilterEnabled`)
+ *   2. Contradiction scan — implemented via the INLINE write-path gate
+ *      `contradictionDetectionEnabled` (NOT the `contradictionScan` cron,
+ *      which only registers a scheduled job that never fires during a bench
+ *      replay, so it would measure nothing).
+ *   3. Graph / temporal recall (`graphRecallEnabled` + `multiGraphMemoryEnabled`
+ *      — orchestrator.ts §1379 requires BOTH for graph_mode — + the full-mode
+ *      graph assist gate).
+ *
+ * Baseline state of each flag (the raw `config.ts` parse default, since the
+ * bench baseline config does NOT override these):
+ *   - `recallMemoryWorthFilterEnabled`: **true** (default; #1574 baseline ran
+ *     with it ON, so the ablation cell turns it OFF to measure the cost of
+ *     removing it).
+ *   - `contradictionDetectionEnabled`: **false** (config.ts §2078; ablation
+ *     cell turns it ON so write-path supersessions land before answering).
+ *   - `graphRecallEnabled` / `multiGraphMemoryEnabled`: **false** (default;
+ *     ablation cell turns BOTH on + the full-mode graph assist).
+ *
+ * `trustScoreEnabled` is deliberately NOT an axis here: it is the unified
+ * stage from #1577 and subsumes the Memory Worth multiplier when on (rule 39).
+ * Its ablation belongs to the #1577 / #1585 model-lab track, not the
+ * single-flag publishable-artifact track this matrix serves.
+ *
+ * Every cell carries a `baselineState` note so a reader of a committed
+ * artifact can tell which direction the flag was flipped without cross-
+ * referencing config.ts. The runner stamps this into the artifact `note`.
+ */
+
+/** Identifier for one ablation cell. Stable across runs; used in artifact notes + filenames. */
+type SingleFlagAblationId = "memory-worth-off" | "contradiction-scan-on" | "graph-recall-on";
+/** The config-override payload merged into `adapterOptions.configOverrides`. */
+type AblationConfigOverrides = Record<string, unknown>;
+/** One reproducible ablation cell. */
+interface SingleFlagAblationCell {
+    /** Stable cell id; appears in artifact notes and STATUS logs. */
+    id: SingleFlagAblationId;
+    /** Human-readable label for tables / STATUS files. */
+    label: string;
+    /** Which #1574 ablation axis this cell exercises. */
+    axis: "memory-worth" | "contradiction-scan" | "graph-recall";
+    /** One-line description of what the cell flips + why, for the artifact `note`. */
+    description: string;
+    /** State of the flag in the baseline run (so the delta direction is unambiguous). */
+    baselineState: string;
+    /** The Remnic config overrides that define this cell (merged over the baseline). */
+    configOverrides: AblationConfigOverrides;
+    /** The single top-level flag key this cell toggles (for grep / ratchet checks). */
+    primaryFlag: "recallMemoryWorthFilterEnabled" | "contradictionDetectionEnabled" | "graphRecallEnabled";
+}
+/**
+ * The canonical 3-cell single-flag ablation matrix (issue #1574 §"Ablations",
+ * verified/produced for the paper under issue #1730).
+ *
+ * Order is stable: memory-worth → contradiction-scan → graph-recall. The
+ * runner executes them in this order; tests assert the exact order so a
+ * reordered matrix is a visible review signal, not a silent change.
+ */
+declare const SINGLE_FLAG_ABLATION_MATRIX: readonly SingleFlagAblationCell[];
+/**
+ * The default benchmark the ablation matrix targets. LoCoMo is the headline
+ * long-conversation benchmark and the one the #1574 baseline artifacts cover
+ * at full scale (1986 QA across 10 conversations); the ablation cells compare
+ * against that baseline. LongMemEval ablations are a documented follow-up
+ * (issue #1730 scope: coordinate with, not duplicate, the rest of the paper).
+ */
+declare const DEFAULT_ABLATION_BENCHMARK: PublishedBenchmarkId;
+/** Look up a cell by id. Throws on unknown id (fail fast at the runner boundary). */
+declare function getAblationCell(id: SingleFlagAblationId): SingleFlagAblationCell;
+
 /**
  * Real-fixture procedural-recall scenarios (issue #567 PR 2/5).
  *
@@ -4222,4 +4623,4 @@ declare function checkCodingGraphRegression(report: CodingGraphBenchReport, base
  */
 declare function buildBaselineFromReport(report: CodingGraphBenchReport, note: string): CodingGraphBaseline;
 
-export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type CodexCliProviderConfig, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type MemCorrectGeneratorOptions, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
+export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };