@remnic/bench 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,1757 @@
+import { GatewayConfig, EngramAccessService } from '@remnic/core';
+
+/**
+ * Types for the ingestion benchmark tier.
+ */
+type GoldEntityType = "person" | "org" | "project" | "topic" | "event" | "location";
+interface GoldEntity {
+    id: string;
+    name: string;
+    type: GoldEntityType;
+    aliases?: string[];
+}
+interface GoldLink {
+    source: string;
+    target: string;
+    relation: string;
+    bidirectional: boolean;
+}
+interface GoldPage {
+    title: string;
+    requiredFields: string[];
+    expectTimeline: boolean;
+    expectExecSummary: boolean;
+    expectSeeAlso: string[];
+}
+interface GoldGraph {
+    entities: GoldEntity[];
+    links: GoldLink[];
+    pages: GoldPage[];
+}
+interface ExtractedEntity {
+    name: string;
+    type: string;
+    sourceFile: string;
+}
+interface ExtractedLink {
+    source: string;
+    target: string;
+    relation: string;
+}
+interface ExtractedPage {
+    path: string;
+    title: string;
+    frontmatter: Record<string, unknown>;
+    hasExecSummary: boolean;
+    hasTimeline: boolean;
+    seeAlso: string[];
+    content: string;
+}
+interface MemoryGraph {
+    entities: ExtractedEntity[];
+    links: ExtractedLink[];
+    pages: ExtractedPage[];
+}
+interface IngestionLog {
+    commandsIssued: string[];
+    promptsShown: string[];
+    errors: string[];
+    durationMs: number;
+}
+interface IngestionBenchAdapter {
+    ingest(inputDir: string): Promise<IngestionLog>;
+    getMemoryGraph(): Promise<MemoryGraph>;
+    reset(): Promise<void>;
+    destroy(): Promise<void>;
+}
+declare const REQUIRED_FRONTMATTER_FIELDS: readonly ["title", "type", "state", "created", "see-also"];
+
+/**
+ * Shared adapter contract for benchmarks running against Remnic memory systems.
+ */
+interface Message {
+    role: "user" | "assistant" | "system";
+    content: string;
+}
+interface SearchResult {
+    turnIndex: number;
+    role: string;
+    snippet: string;
+    sessionId: string;
+    score?: number;
+}
+interface MemoryStats {
+    totalMessages: number;
+    totalSummaryNodes: number;
+    maxDepth: number;
+}
+interface BenchResponse {
+    text: string;
+    tokens: {
+        input: number;
+        output: number;
+    };
+    latencyMs: number;
+    model: string;
+}
+interface BenchResponder {
+    respond(question: string, recalledText: string): Promise<BenchResponse>;
+}
+interface BenchJudgeResult {
+    score: number;
+    tokens: {
+        input: number;
+        output: number;
+    };
+    latencyMs: number;
+    model?: string;
+}
+interface BenchJudge {
+    score(question: string, predicted: string, expected: string): Promise<number>;
+    scoreWithMetrics?(question: string, predicted: string, expected: string): Promise<BenchJudgeResult>;
+}
+interface BenchMemoryAdapter {
+    store(sessionId: string, messages: Message[]): Promise<void>;
+    recall(sessionId: string, query: string, budgetChars?: number): Promise<string>;
+    search(query: string, limit: number, sessionId?: string): Promise<SearchResult[]>;
+    reset(sessionId?: string): Promise<void>;
+    getStats(sessionId?: string): Promise<MemoryStats>;
+    destroy(): Promise<void>;
+    responder?: BenchResponder;
+    judge?: BenchJudge;
+}
+type LlmJudge = BenchJudge;
+type MemorySystem = BenchMemoryAdapter;
+
+/**
+ * Integrity-facing additions to `BenchmarkResult.meta`.
+ *
+ * These fields are required for every published result and checked by the
+ * publishing pipeline. See `docs/bench/integrity.md` for the rotation policy.
+ */
+declare const BENCHMARK_SPLIT_TYPES: readonly ["public", "holdout"];
+type BenchmarkSplitType = (typeof BENCHMARK_SPLIT_TYPES)[number];
+interface BenchmarkIntegrityMeta {
+    /**
+     * Which dataset split produced this result. Public leaderboard scores
+     * only accept `holdout` results; `public` results are for self-reporting
+     * and iteration.
+     */
+    splitType: BenchmarkSplitType;
+    /** SHA-256 of the sealed qrels artifact used by the judge. */
+    qrelsSealedHash: string;
+    /** SHA-256 of the rendered judge prompt (post-template expansion). */
+    judgePromptHash: string;
+    /** SHA-256 of the dataset payload as served to the runner. */
+    datasetHash: string;
+    /**
+     * Score the canary adapter scored on the same benchmark during the audit
+     * run that produced this result. Must stay below the benchmark's floor.
+     * Omitted only during the canary's own run.
+     */
+    canaryScore?: number;
+}
+declare const INTEGRITY_META_FIELDS: readonly ["splitType", "qrelsSealedHash", "judgePromptHash", "datasetHash"];
+declare const BENCHMARK_INTEGRITY_META_SCHEMA: {
+    readonly type: "object";
+    readonly required: readonly ["splitType", "qrelsSealedHash", "judgePromptHash", "datasetHash"];
+    readonly properties: {
+        readonly splitType: {
+            readonly type: "string";
+            readonly enum: readonly ["public", "holdout"];
+        };
+        readonly qrelsSealedHash: {
+            readonly type: "string";
+            readonly pattern: "^[0-9a-f]{64}$";
+        };
+        readonly judgePromptHash: {
+            readonly type: "string";
+            readonly pattern: "^[0-9a-f]{64}$";
+        };
+        readonly datasetHash: {
+            readonly type: "string";
+            readonly pattern: "^[0-9a-f]{64}$";
+        };
+        readonly canaryScore: {
+            readonly type: "number";
+        };
+    };
+};
+declare function integrityMetaIsComplete(value: unknown): value is BenchmarkIntegrityMeta;
+/**
+ * Throw a descriptive error listing every missing or malformed integrity
+ * field. Used by the publishing pipeline.
+ */
+declare function assertIntegrityMetaPresent(value: unknown): asserts value is BenchmarkIntegrityMeta;
+
+type BenchmarkMode = "full" | "quick";
+type BenchmarkTier = "published" | "remnic" | "custom";
+type BenchmarkStatus = "ready" | "planned";
+type BenchmarkCategory = "agentic" | "retrieval" | "conversational" | "ingestion";
+type BenchRuntimeProfile = "baseline" | "real" | "openclaw-chain";
+type BuiltInProvider = "openai" | "anthropic" | "ollama" | "litellm";
+interface ProviderConfig {
+    provider: BuiltInProvider;
+    model: string;
+    baseUrl?: string;
+}
+interface TaskTokenUsage {
+    input: number;
+    output: number;
+}
+interface TaskResult {
+    taskId: string;
+    question: string;
+    expected: string;
+    actual: string;
+    scores: Record<string, number>;
+    latencyMs: number;
+    tokens: TaskTokenUsage;
+    details?: Record<string, unknown>;
+}
+interface MetricAggregate {
+    mean: number;
+    median: number;
+    stdDev: number;
+    min: number;
+    max: number;
+}
+type AggregateMetrics = Record<string, MetricAggregate>;
+interface ConfidenceInterval {
+    lower: number;
+    upper: number;
+    level: number;
+}
+type EffectSizeInterpretation = "negligible" | "small" | "medium" | "large";
+interface EffectSizeSummary {
+    cohensD: number;
+    interpretation: EffectSizeInterpretation;
+}
+interface ComparisonMetricDelta {
+    baseline: number;
+    candidate: number;
+    delta: number;
+    percentChange: number;
+    effectSize: EffectSizeSummary;
+    ciOnDelta?: ConfidenceInterval;
+}
+interface ComparisonResult {
+    benchmark: string;
+    metricDeltas: Record<string, ComparisonMetricDelta>;
+    verdict: "pass" | "regression" | "improvement";
+}
+interface StatisticalReport {
+    confidenceIntervals: Record<string, ConfidenceInterval>;
+    bootstrapSamples: number;
+    effectSizes?: Record<string, EffectSizeSummary>;
+    pairedComparison?: {
+        baselineId: string;
+        pValue: number;
+        ciOnDelta: ConfidenceInterval;
+    };
+}
+interface BenchmarkResult {
+    meta: {
+        id: string;
+        benchmark: string;
+        benchmarkTier: BenchmarkTier;
+        version: string;
+        remnicVersion: string;
+        gitSha: string;
+        timestamp: string;
+        mode: BenchmarkMode;
+        runCount: number;
+        seeds: number[];
+        /**
+         * Which dataset split produced this result. Public leaderboard scores
+         * only accept `holdout`; `public` is for self-reporting and iteration.
+         */
+        splitType?: BenchmarkSplitType;
+        /** SHA-256 of the sealed qrels artifact used by the judge. */
+        qrelsSealedHash?: string;
+        /** SHA-256 of the rendered judge prompt (post-template expansion). */
+        judgePromptHash?: string;
+        /** SHA-256 of the dataset payload as served to the runner. */
+        datasetHash?: string;
+        /**
+         * Canary-adapter score from the audit run that produced this result.
+         * Must stay below the benchmark's canary floor.
+         */
+        canaryScore?: number;
+    };
+    config: {
+        runtimeProfile?: BenchRuntimeProfile | null;
+        systemProvider: ProviderConfig | null;
+        judgeProvider: ProviderConfig | null;
+        adapterMode: string;
+        remnicConfig: Record<string, unknown>;
+    };
+    cost: {
+        totalTokens: number;
+        inputTokens: number;
+        outputTokens: number;
+        estimatedCostUsd: number;
+        totalLatencyMs: number;
+        meanQueryLatencyMs: number;
+    };
+    results: {
+        tasks: TaskResult[];
+        aggregates: AggregateMetrics;
+        statistics?: StatisticalReport;
+    };
+    environment: {
+        os: string;
+        nodeVersion: string;
+        hardware?: string;
+    };
+}
+interface BenchmarkMeta {
+    name: string;
+    version: string;
+    description: string;
+    category: BenchmarkCategory;
+    citation?: string;
+    /**
+     * Optional integrity metadata declared on the benchmark itself (as opposed
+     * to each result). When set, the publishing pipeline pins result-time
+     * integrity hashes against these values.
+     */
+    integrity?: BenchmarkIntegrityMeta;
+}
+
+interface BenchmarkDefinition {
+    id: string;
+    title: string;
+    tier: BenchmarkTier;
+    status: BenchmarkStatus;
+    runnerAvailable: boolean;
+    meta: BenchmarkMeta;
+}
+interface RunBenchmarkOptions {
+    mode?: BenchmarkMode;
+    datasetDir?: string;
+    outputDir?: string;
+    limit?: number;
+    seed?: number;
+    adapterMode?: string;
+    runtimeProfile?: BenchRuntimeProfile | null;
+    system: BenchMemoryAdapter;
+    ingestionAdapter?: IngestionBenchAdapter;
+    systemProvider?: ProviderConfig | null;
+    judgeProvider?: ProviderConfig | null;
+    remnicConfig?: Record<string, unknown>;
+}
+interface ResolvedRunBenchmarkOptions extends RunBenchmarkOptions {
+    mode: BenchmarkMode;
+    benchmark: BenchmarkDefinition;
+}
+type BenchTier = "exact_match" | "category_match" | "keyword_overlap" | "high_confidence" | "semantic_search" | "full_search" | "no_results";
+interface TierDetail {
+    tier: BenchTier;
+    latencyMs: number;
+    resultsCount: number;
+}
+interface ExplainResult {
+    query: string;
+    tiersUsed: BenchTier[];
+    tierResults: TierDetail[];
+    durationMs: number;
+    totalDurationMs: number;
+}
+interface RecallMetrics {
+    query: string;
+    latencyMs: number;
+    tiersUsed: BenchTier[];
+    throughput: number;
+    resultsCount: number;
+    totalDurationMs: number;
+    tierDetails: TierDetail[];
+}
+interface BenchmarkReport {
+    timestamp: string;
+    queries: Array<{
+        query: string;
+        tiersUsed: BenchTier[];
+        durationMs: number;
+        resultsCount: number;
+        throughput: number;
+        tierDetails: TierDetail[];
+    }>;
+    totalDurationMs: number;
+}
+interface BenchmarkSuiteResult {
+    results: RecallMetrics[];
+    report: BenchmarkReport;
+    totalDurationMs: number;
+    regressions: RegressionDetail[];
+}
+interface SavedBaseline {
+    version: number;
+    timestamp: string;
+    metrics: Record<string, number>;
+}
+interface RegressionGateResult {
+    passed: boolean;
+    regressions: RegressionDetail[];
+}
+interface RegressionDetail {
+    metric: string;
+    currentValue: number;
+    baselineValue: number;
+    tolerance: number;
+    passed: boolean;
+}
+interface BenchConfig {
+    queries?: string[];
+    iterations?: number;
+    regressionTolerance?: number;
+    baselinePath?: string;
+    reportPath?: string;
+    seed?: number;
+    explain?: boolean;
+}
+
+/**
+ * Custom benchmark schema types.
+ */
+
+type CustomBenchmarkScoring = "exact_match" | "f1" | "rouge_l" | "llm_judge";
+interface CustomBenchmarkTask {
+    question: string;
+    expected: string;
+    tags?: string[];
+}
+interface CustomBenchmarkSpec {
+    name: string;
+    description?: string;
+    version?: string;
+    category?: BenchmarkCategory;
+    citation?: string;
+    scoring: CustomBenchmarkScoring;
+    tasks: CustomBenchmarkTask[];
+}
+
+/**
+ * Shared types for inbox fixture generators.
+ */
+
+interface GeneratedFile {
+    relativePath: string;
+    content: string;
+}
+interface FixtureOutput {
+    id: string;
+    description: string;
+    files: GeneratedFile[];
+    goldGraph: GoldGraph;
+}
+interface FixtureGenerator {
+    id: string;
+    description: string;
+    generate(): FixtureOutput;
+}
+
+/**
+ * Package-owned Remnic adapters used by the phase-1 benchmark CLI surface.
+ */
+
+interface RemnicAdapterOptions {
+    configOverrides?: Record<string, unknown>;
+    preserveRuntimeDefaults?: boolean;
+    responder?: BenchResponder;
+    judge?: BenchJudge;
+}
+declare const createLightweightAdapter: (options?: RemnicAdapterOptions) => Promise<BenchMemoryAdapter>;
+declare const createRemnicAdapter: (options?: RemnicAdapterOptions) => Promise<BenchMemoryAdapter>;
+
+/**
+ * Minimal LLM provider contract for the bench engine.
+ */
+
+interface CompletionOpts {
+    systemPrompt?: string;
+    temperature?: number;
+    maxTokens?: number;
+    headers?: Record<string, string>;
+}
+interface CompletionResult {
+    text: string;
+    tokens: {
+        input: number;
+        output: number;
+    };
+    latencyMs: number;
+    model: string;
+}
+interface DiscoveredModel {
+    id: string;
+    name: string;
+    contextLength: number;
+    capabilities: ("completion" | "embedding" | "vision")[];
+    quantization?: string;
+    parameterCount?: string;
+}
+interface ProviderBaseConfig {
+    model: string;
+    baseUrl?: string;
+    apiKey?: string;
+    headers?: Record<string, string>;
+}
+interface OpenAiCompatibleProviderConfig extends ProviderBaseConfig {
+    provider?: "openai" | "litellm";
+}
+interface AnthropicProviderConfig extends ProviderBaseConfig {
+    provider?: "anthropic";
+    anthropicVersion?: string;
+}
+interface OllamaProviderConfig extends ProviderBaseConfig {
+    provider?: "ollama";
+}
+type ProviderFactoryConfig = (OpenAiCompatibleProviderConfig & {
+    provider: "openai" | "litellm";
+}) | (AnthropicProviderConfig & {
+    provider: "anthropic";
+}) | (OllamaProviderConfig & {
+    provider: "ollama";
+});
+interface ProviderDiscoveryResult {
+    provider: BuiltInProvider;
+    models: DiscoveredModel[];
+}
+interface TokenUsage {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+}
+interface LlmProvider {
+    id: string;
+    name: string;
+    provider: BuiltInProvider;
+    complete(prompt: string, opts?: CompletionOpts): Promise<CompletionResult>;
+    embed?(texts: string[]): Promise<number[][]>;
+    discover?(): Promise<DiscoveredModel[]>;
+    getUsage(): TokenUsage;
+    resetUsage(): void;
+}
+
+/**
+ * JSON schema contract for BenchmarkResult payloads.
+ */
+declare const BENCHMARK_RESULT_SCHEMA: {
+    readonly type: "object";
+    readonly required: readonly ["meta", "config", "cost", "results", "environment"];
+    readonly properties: {
+        readonly meta: {
+            readonly type: "object";
+            readonly required: readonly ["id", "benchmark", "benchmarkTier", "version", "remnicVersion", "gitSha", "timestamp", "mode", "runCount", "seeds"];
+            readonly properties: {
+                readonly id: {
+                    readonly type: "string";
+                };
+                readonly benchmark: {
+                    readonly type: "string";
+                };
+                readonly benchmarkTier: {
+                    readonly type: "string";
+                    readonly enum: readonly ["published", "remnic", "custom"];
+                };
+                readonly version: {
+                    readonly type: "string";
+                };
+                readonly remnicVersion: {
+                    readonly type: "string";
+                };
+                readonly gitSha: {
+                    readonly type: "string";
+                };
+                readonly timestamp: {
+                    readonly type: "string";
+                };
+                readonly mode: {
+                    readonly type: "string";
+                    readonly enum: readonly ["full", "quick"];
+                };
+                readonly runCount: {
+                    readonly type: "number";
+                };
+                readonly seeds: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "number";
+                    };
+                };
+                readonly splitType: {
+                    readonly type: "string";
+                    readonly enum: readonly ["public", "holdout"];
+                };
+                readonly qrelsSealedHash: {
+                    readonly type: "string";
+                    readonly pattern: "^[0-9a-f]{64}$";
+                };
+                readonly judgePromptHash: {
+                    readonly type: "string";
+                    readonly pattern: "^[0-9a-f]{64}$";
+                };
+                readonly datasetHash: {
+                    readonly type: "string";
+                    readonly pattern: "^[0-9a-f]{64}$";
+                };
+                readonly canaryScore: {
+                    readonly type: "number";
+                };
+            };
+        };
+        readonly config: {
+            readonly type: "object";
+            readonly required: readonly ["systemProvider", "judgeProvider", "adapterMode", "remnicConfig"];
+            readonly properties: {
+                readonly runtimeProfile: {
+                    readonly anyOf: readonly [{
+                        readonly type: "null";
+                    }, {
+                        readonly type: "string";
+                        readonly enum: readonly ["baseline", "real", "openclaw-chain"];
+                    }];
+                };
+                readonly systemProvider: {
+                    readonly anyOf: readonly [{
+                        readonly type: "null";
+                    }, {
+                        readonly type: "object";
+                        readonly required: readonly ["provider", "model"];
+                        readonly properties: {
+                            readonly provider: {
+                                readonly type: "string";
+                            };
+                            readonly model: {
+                                readonly type: "string";
+                            };
+                            readonly baseUrl: {
+                                readonly type: "string";
+                            };
+                        };
+                    }];
+                };
+                readonly judgeProvider: {
+                    readonly anyOf: readonly [{
+                        readonly type: "null";
+                    }, {
+                        readonly type: "object";
+                        readonly required: readonly ["provider", "model"];
+                        readonly properties: {
+                            readonly provider: {
+                                readonly type: "string";
+                            };
+                            readonly model: {
+                                readonly type: "string";
+                            };
+                            readonly baseUrl: {
+                                readonly type: "string";
+                            };
+                        };
+                    }];
+                };
+                readonly adapterMode: {
+                    readonly type: "string";
+                };
+                readonly remnicConfig: {
+                    readonly type: "object";
+                };
+            };
+        };
+        readonly cost: {
+            readonly type: "object";
+            readonly required: readonly ["totalTokens", "inputTokens", "outputTokens", "estimatedCostUsd", "totalLatencyMs", "meanQueryLatencyMs"];
+            readonly properties: {
+                readonly totalTokens: {
+                    readonly type: "number";
+                };
+                readonly inputTokens: {
+                    readonly type: "number";
+                };
+                readonly outputTokens: {
+                    readonly type: "number";
+                };
+                readonly estimatedCostUsd: {
+                    readonly type: "number";
+                };
+                readonly totalLatencyMs: {
+                    readonly type: "number";
+                };
+                readonly meanQueryLatencyMs: {
+                    readonly type: "number";
+                };
+            };
+        };
+        readonly results: {
+            readonly type: "object";
+            readonly required: readonly ["tasks", "aggregates"];
+            readonly properties: {
+                readonly tasks: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "object";
+                        readonly required: readonly ["taskId", "question", "expected", "actual", "scores", "latencyMs", "tokens"];
+                        readonly properties: {
+                            readonly taskId: {
+                                readonly type: "string";
+                            };
+                            readonly question: {
+                                readonly type: "string";
+                            };
+                            readonly expected: {
+                                readonly type: "string";
+                            };
+                            readonly actual: {
+                                readonly type: "string";
+                            };
+                            readonly scores: {
+                                readonly type: "object";
+                            };
+                            readonly latencyMs: {
+                                readonly type: "number";
+                            };
+                            readonly tokens: {
+                                readonly type: "object";
+                                readonly required: readonly ["input", "output"];
+                                readonly properties: {
+                                    readonly input: {
+                                        readonly type: "number";
+                                    };
+                                    readonly output: {
+                                        readonly type: "number";
+                                    };
+                                };
+                            };
+                        };
+                    };
+                };
+                readonly aggregates: {
+                    readonly type: "object";
+                };
+                readonly statistics: {
+                    readonly type: "object";
+                };
+            };
+        };
+        readonly environment: {
+            readonly type: "object";
+            readonly required: readonly ["os", "nodeVersion"];
+            readonly properties: {
+                readonly os: {
+                    readonly type: "string";
+                };
+                readonly nodeVersion: {
+                    readonly type: "string";
+                };
+                readonly hardware: {
+                    readonly type: "string";
+                };
+            };
+        };
+    };
+};
+
+declare function createAnthropicProvider(config: AnthropicProviderConfig): LlmProvider;
+
+declare function createProvider(config: ProviderFactoryConfig): LlmProvider;
+declare function discoverAllProviders(): Promise<ProviderDiscoveryResult[]>;
+
+interface BenchmarkAnswerResult {
+    finalAnswer: string;
+    recalledText: string;
+    answeredText: string;
+    latencyMs: number;
+    tokens: {
+        input: number;
+        output: number;
+    };
+    model?: string;
+}
+declare function answerBenchmarkQuestion(options: {
+    question: string;
+    recalledText: string;
+    responder?: BenchResponder;
+}): Promise<BenchmarkAnswerResult>;
+
+/**
+ * Sealed LLM-judge rubric loader, invocation, and score parser for the
+ * Assistant bench tier.
+ *
+ * Sealing contract:
+ *   1. The rubric prompt lives in the in-process registry
+ *      (`sealed-prompts/index.ts`) and is never exposed to the
+ *      system-under-test.
+ *   2. The rubric text's SHA-256 digest is embedded into every run result so
+ *      any change to the prompt is detectable by consumers of the bench feed.
+ *   3. Rotations are additive — add a new registry entry and a matching
+ *      `.md` mirror, do not edit old ones.
+ */
+declare const ASSISTANT_RUBRIC_DIMENSIONS: readonly ["identity_accuracy", "stance_coherence", "novelty", "calibration"];
+type AssistantRubricDimension = (typeof ASSISTANT_RUBRIC_DIMENSIONS)[number];
+type AssistantRubricScores = Record<AssistantRubricDimension, number>;
+interface SealedRubric {
+    id: string;
+    version: string;
+    prompt: string;
+    sha256: string;
+}
+interface SealedJudgeInput {
+    taskId: string;
+    scenario: string;
+    memorySummary: string;
+    assistantOutput: string;
+}
+interface SealedJudgeDecision {
+    taskId: string;
+    rubricId: string;
+    rubricSha256: string;
+    scores: AssistantRubricScores;
+    notes: string;
+    rawResponse: string;
+    parseOk: boolean;
+}
+/**
+ * Rich structured-judge contract for the Assistant tier. Unlike
+ * `BenchJudge.score()`, which returns a scalar, structured judges return the
+ * raw JSON response text so we can parse the full multi-dimension rubric.
+ */
+interface StructuredJudge {
+    evaluate(request: {
+        system: string;
+        user: string;
+        rubricId: string;
+        taskId: string;
+    }): Promise<string>;
+}
+interface SpotCheckLogger {
+    log(decision: SealedJudgeDecision, context: SealedJudgeInput): void;
+}
+/**
+ * Load a sealed rubric prompt from the in-process registry by id.
+ *
+ * The returned object captures the canonical text and a SHA-256 digest which
+ * callers are expected to store in benchmark results so reviewers can verify
+ * the exact rubric text used for a given run.
+ */
+declare function loadSealedRubric(id?: string, options?: {
+    registry?: Readonly<Record<string, string>>;
+}): SealedRubric;
+/**
+ * Verify that a registered rubric still matches an expected digest. Useful in
+ * tests and in CI gates that want to catch accidental edits to sealed text.
+ */
+declare function verifyRubricDigest(expectedSha256: string, options?: {
+    id?: string;
+    registry?: Readonly<Record<string, string>>;
+}): boolean;
+/**
+ * Build the judge message payload for a single task. Keeps the rubric prompt
+ * on the system side of the conversation and the task-specific substitutions
+ * in a user message so the judge never leaks rubric text back into the SUT
+ * path.
+ */
+declare function buildJudgePayload(rubric: SealedRubric, input: SealedJudgeInput): {
+    system: string;
+    user: string;
+};
+/**
+ * Invoke a structured judge with the sealed rubric and parse the response.
+ *
+ * When `judge` is `undefined` we return a parse_error decision with all-zero
+ * scores so the caller can still complete the benchmark with a visible signal
+ * that the judge was missing.
+ */
+declare function runSealedJudge(judge: StructuredJudge | undefined, rubric: SealedRubric, input: SealedJudgeInput, options?: {
+    spotCheckLogger?: SpotCheckLogger;
+}): Promise<SealedJudgeDecision>;
+/**
+ * Parse a judge response string as rubric JSON. Exported for unit tests and
+ * for judge adapters that return the raw response directly.
+ */
+declare function parseRubricResponse(raw: string): {
+    scores: AssistantRubricScores;
+    notes: string;
+    ok: boolean;
+};
+/**
+ * Spot-check logger that appends selected judge decisions to a JSONL file.
+ * The caller controls the `runId` to keep logs grouped per-run.
+ *
+ * Logging is a diagnostic side effect, so any filesystem error (non-writable
+ * directory, path conflict with an existing file, mid-run ENOSPC, etc.) is
+ * caught and downgraded to a one-time `console.warn` rather than aborting
+ * the benchmark run. We also fail-safe the `mkdirSync` at construction
+ * time: if the directory cannot be created, we return a no-op logger so
+ * callers can still run the benchmark end-to-end.
+ */
+declare function createSpotCheckFileLogger(options: {
+    runId: string;
+    directory: string;
+    sampleRate?: number;
+    random?: () => number;
+    sampleSize?: number;
+}): SpotCheckLogger;
+/**
+ * Create a deterministic spot-check logger useful in tests: always picks the
+ * first `sampleSize` decisions regardless of random draw.
+ */
+declare function createDeterministicSpotCheckLogger(options: {
+    runId: string;
+    directory: string;
+    sampleSize?: number;
+}): SpotCheckLogger;
+declare function zeroScores(): AssistantRubricScores;
+declare function clampScore(value: number): number;
+
+interface GatewayResponderOptions {
+    gatewayConfig?: GatewayConfig;
+    agentId?: string;
+}
+declare function createResponderFromProvider(provider: LlmProvider): BenchResponder;
+declare function createProviderBackedResponder(config: ProviderFactoryConfig, providerInstance?: LlmProvider): BenchResponder;
+declare function createProviderBackedJudge(config: ProviderFactoryConfig, providerInstance?: LlmProvider): BenchJudge;
+declare function createStructuredJudgeFromProvider(provider: LlmProvider): StructuredJudge;
+declare function createProviderBackedStructuredJudge(config: ProviderFactoryConfig, providerInstance?: LlmProvider): StructuredJudge;
+declare function createGatewayResponder(options: GatewayResponderOptions): BenchResponder;
+
+declare function createLiteLlmProvider(config: OpenAiCompatibleProviderConfig): LlmProvider;
+
+declare function createOllamaProvider(config: OllamaProviderConfig): LlmProvider;
+
+/**
+ * Minimal OpenAI-compatible provider for phase 1 bench execution.
+ */
+
+declare function createOpenAiCompatibleProvider(config: OpenAiCompatibleProviderConfig): LlmProvider;
+
+type BenchModelSource = "plugin" | "gateway";
+interface ResolveBenchRuntimeProfileOptions {
+    runtimeProfile?: BenchRuntimeProfile;
+    remnicConfigPath?: string;
+    openclawConfigPath?: string;
+    modelSource?: BenchModelSource;
+    gatewayAgentId?: string;
+    fastGatewayAgentId?: string;
+    systemProvider?: BuiltInProvider;
+    systemModel?: string;
+    systemBaseUrl?: string;
+    judgeProvider?: BuiltInProvider;
+    judgeModel?: string;
+    judgeBaseUrl?: string;
+}
+interface ResolvedBenchRuntimeProfile {
+    profile: BenchRuntimeProfile;
+    remnicConfig: Record<string, unknown>;
+    effectiveRemnicConfig: Record<string, unknown>;
+    adapterOptions: {
+        configOverrides: Record<string, unknown>;
+        preserveRuntimeDefaults?: boolean;
+        responder?: BenchResponder;
+        judge?: BenchJudge;
+    };
+    systemProvider: ProviderConfig | null;
+    judgeProvider: ProviderConfig | null;
+}
+declare function resolveBenchRuntimeProfile(options: ResolveBenchRuntimeProfileOptions): Promise<ResolvedBenchRuntimeProfile>;
+
+/**
+ * Published benchmark registry for @remnic/bench phase 1.
+ */
+
+declare function listBenchmarks(): BenchmarkDefinition[];
+declare function getBenchmark(id: string): BenchmarkDefinition | undefined;
+
+/**
+ * Result enrichment and JSON writing helpers.
+ */
+
+declare function writeBenchmarkResult(result: BenchmarkResult, outputDir: string): Promise<string>;
+
+/**
+ * Seed-sequence generation for benchmark runs.
+ *
+ * Factored out of `benchmark.ts` so individual runners can reuse it without
+ * triggering a circular import through `benchmark.ts -> registry.ts ->
+ * runner.ts -> benchmark.ts`.
+ */
+declare function buildBenchmarkRunSeeds(runCount: number, baseSeed?: number): number[];
+
+/**
+ * Public benchmark execution helpers.
+ */
+
+declare function resolveBenchmarkRunCount(mode: BenchmarkMode, requestedIterations?: number): number;
+
+declare function orchestrateBenchmarkRuns<T>(mode: BenchmarkMode, executeRun: (seed: number, runIndex: number) => Promise<T>, requestedIterations?: number, baseSeed?: number): Promise<{
+    runCount: number;
+    seeds: number[];
+    runs: T[];
+}>;
+declare function runBenchmark(benchmarkId: string, options: RunBenchmarkOptions): Promise<BenchmarkResult>;
+declare function loadBaseline(baselinePath?: string): SavedBaseline | undefined;
+declare function saveBaseline(baselinePath: string, baseline: SavedBaseline): void;
+declare function runExplain(service: EngramAccessService, query: string): Promise<ExplainResult>;
+declare function runBenchSuite(service: EngramAccessService, config?: BenchConfig): Promise<BenchmarkSuiteResult>;
+declare function checkRegression(metrics: Record<string, number>, baseline: SavedBaseline | undefined, tolerance: number): RegressionGateResult;
+declare function generateReport(results: RecallMetrics[], reportPath?: string): BenchmarkReport;
+
+/**
+ * Shared scoring utilities for bench runners.
+ */
+
+declare function exactMatch(predicted: string, expected: string | number | unknown): number;
+declare function f1Score(predicted: string, expected: string | number | unknown): number;
+declare function rougeL(predicted: string, expected: string | number | unknown): number;
+declare function recallAtK(retrieved: string[], relevant: string[], k: number): number;
+declare function precisionAtK(retrieved: string[], relevant: string[], k: number): number;
+declare function containsAnswer(predicted: string, expected: string | number | unknown): number;
+declare function llmJudgeScore(judge: {
+    score(question: string, predicted: string, expected: string): Promise<number>;
+    scoreWithMetrics?(question: string, predicted: string, expected: string): Promise<BenchJudgeResult>;
+} | undefined, question: string, predicted: string, expected: string): Promise<number>;
+declare function llmJudgeScoreDetailed(judge: {
+    score(question: string, predicted: string, expected: string): Promise<number>;
+    scoreWithMetrics?(question: string, predicted: string, expected: string): Promise<BenchJudgeResult>;
+} | undefined, question: string, predicted: string, expected: string): Promise<BenchJudgeResult>;
+declare function timed<T>(fn: () => Promise<T>): Promise<{
+    result: T;
+    durationMs: number;
+}>;
+declare function aggregateTaskScores(metricsList: Array<Record<string, number>>): AggregateMetrics;
+
+interface BootstrapOptions {
+    iterations?: number;
+    level?: number;
+    random?: () => number;
+}
+declare function bootstrapMeanConfidenceInterval(values: number[], options?: BootstrapOptions): ConfidenceInterval;
+declare function pairedDeltaConfidenceInterval(candidateValues: number[], baselineValues: number[], options?: BootstrapOptions): ConfidenceInterval;
+
+declare function cohensD(candidateValues: number[], baselineValues: number[]): number;
+declare function interpretEffectSize(cohensDValue: number): EffectSizeInterpretation;
+
+declare function compareResults(baseline: BenchmarkResult, candidate: BenchmarkResult, threshold?: number, lowerIsBetter?: ReadonlySet<string>): ComparisonResult;
+declare function getBenchmarkLowerIsBetter(benchmarkId: string): ReadonlySet<string>;
+
+/**
+ * Dataset-contamination guard.
+ *
+ * Published benchmark results carry a `datasetHash` in `BenchmarkResult.meta`
+ * so the publishing pipeline can reject results whose dataset hash is known
+ * to appear in an LLM's training corpus. The contamination list starts empty
+ * and is extended as new contamination reports arrive.
+ *
+ * Entries are SHA-256 hex digests. Callers pass a `ContaminationManifest`
+ * rather than a bare array so provenance / justification can be attached
+ * alongside the hash. This keeps the audit trail visible when a result is
+ * rejected.
+ */
+interface ContaminationEntry {
+    /** SHA-256 of the dataset payload as published. */
+    datasetHash: string;
+    /** Human-readable reason the dataset is considered contaminated. */
+    reason: string;
+    /** Optional citation / URL documenting the contamination report. */
+    reference?: string;
+    /** ISO-8601 timestamp when the entry was added. */
+    addedAt: string;
+}
+interface ContaminationManifest {
+    version: 1;
+    entries: ContaminationEntry[];
+}
+interface ContaminationCheckResult {
+    /** The dataset hash examined. */
+    datasetHash: string;
+    /** True when the dataset hash is NOT present on the contamination list. */
+    clean: boolean;
+    /** When `clean === false`, the matching manifest entry. */
+    matched?: ContaminationEntry;
+}
+/**
+ * Start with an empty list; upstream tooling populates this as contamination
+ * reports surface. Keeping the default list empty avoids hard-coding public
+ * values that could become stale.
+ */
+declare const EMPTY_CONTAMINATION_MANIFEST: ContaminationManifest;
+declare function isContaminationManifest(value: unknown): value is ContaminationManifest;
+declare function isContaminationEntry(value: unknown): value is ContaminationEntry;
+declare function checkDatasetContamination(datasetHash: string, manifest?: ContaminationManifest): ContaminationCheckResult;
+/**
+ * Merge an additional contamination entry into an existing manifest. Duplicate
+ * hashes are collapsed (first-write wins) so manifests can be safely merged
+ * across sources without ballooning.
+ */
+declare function addContaminationEntry(manifest: ContaminationManifest, entry: ContaminationEntry): ContaminationManifest;
+declare function mergeContaminationManifests(...manifests: ContaminationManifest[]): ContaminationManifest;
+
+interface StoredBenchmarkResultSummary {
+    id: string;
+    path: string;
+    benchmark: string;
+    timestamp: string;
+    mode: BenchmarkMode;
+}
+interface StoredBenchmarkBaseline {
+    name: string;
+    savedAt: string;
+    result: BenchmarkResult;
+    source?: {
+        id: string;
+        path: string;
+    };
+}
+interface StoredBenchmarkBaselineSummary {
+    name: string;
+    path: string;
+    benchmark: string;
+    timestamp: string;
+    resultId: string;
+    resultTimestamp: string;
+    mode: BenchmarkMode;
+}
+type BenchmarkExportFormat = "json" | "csv" | "html";
+type BenchmarkPublishTarget = "remnic-ai";
+interface PublishedBenchmarkFeedEntry {
+    benchmark: string;
+    benchmarkTier: BenchmarkResult["meta"]["benchmarkTier"];
+    resultId: string;
+    timestamp: string;
+    mode: BenchmarkMode;
+    remnicVersion: string;
+    gitSha: string;
+    taskCount: number;
+    aggregateMetrics: BenchmarkResult["results"]["aggregates"];
+    cost: BenchmarkResult["cost"];
+    environment: BenchmarkResult["environment"];
+    integrity: {
+        splitType: NonNullable<BenchmarkResult["meta"]["splitType"]>;
+        qrelsSealedHash: string;
+        judgePromptHash: string;
+        datasetHash: string;
+        canaryScore?: number;
+    };
+}
+interface BuildBenchmarkPublishFeedOptions {
+    /**
+     * Contamination manifest applied to every candidate result. A result whose
+     * `datasetHash` matches an entry is dropped from the published feed.
+     * Defaults to the empty manifest.
+     */
+    contaminationManifest?: ContaminationManifest;
+}
+interface PublishedBenchmarkFeed {
+    target: BenchmarkPublishTarget;
+    generatedAt: string;
+    benchmarks: PublishedBenchmarkFeedEntry[];
+    /**
+     * Records for every candidate result that was considered but dropped from
+     * this feed because of an integrity concern. Exposed so tooling can surface
+     * the dropped runs without grep-ing logs.
+     */
+    skipped?: PublishSkipRecord[];
+}
+declare function defaultBenchmarkBaselineDir(): string;
+declare function defaultBenchmarkPublishPath(target: BenchmarkPublishTarget): string;
+declare function loadBenchmarkResult(filePath: string): Promise<BenchmarkResult>;
+declare function listBenchmarkResults(outputDir: string): Promise<StoredBenchmarkResultSummary[]>;
+declare function saveBenchmarkBaseline(baselineDir: string, name: string, result: BenchmarkResult, source?: {
+    id: string;
+    path: string;
+}): Promise<string>;
+declare function loadBenchmarkBaseline(filePath: string): Promise<StoredBenchmarkBaseline>;
+declare function listBenchmarkBaselines(baselineDir: string): Promise<StoredBenchmarkBaselineSummary[]>;
+declare function resolveBenchmarkResultReference(outputDir: string, reference: string): Promise<StoredBenchmarkResultSummary | undefined>;
+declare function deleteBenchmarkResults(outputDir: string, references: string[]): Promise<{
+    deleted: StoredBenchmarkResultSummary[];
+    missing: string[];
+}>;
+/**
+ * Throws if the result is missing any required integrity field. Called
+ * explicitly by tooling (e.g. `remnic bench publish --strict`) that needs to
+ * surface integrity gaps as errors rather than silently skipping the run.
+ * The feed builder uses `isResultPublishable` below to filter non-fatal
+ * conditions (public split, missing integrity) so a single bad result does
+ * not block publishing older, valid holdout runs.
+ */
+declare function assertPublishableIntegrity(result: BenchmarkResult, target: BenchmarkPublishTarget): void;
+type PublishSkipReason = "missing-integrity" | "non-holdout-split" | "contaminated-dataset";
+interface PublishSkipRecord {
+    resultId: string;
+    path: string;
+    reason: PublishSkipReason;
+    detail: string;
+}
+declare function buildBenchmarkPublishFeed(outputDir: string, target: BenchmarkPublishTarget, options?: BuildBenchmarkPublishFeedOptions): Promise<PublishedBenchmarkFeed>;
+declare function writeBenchmarkPublishFeed(feed: PublishedBenchmarkFeed, outputPath: string): Promise<string>;
+declare function renderBenchmarkResultExport(result: BenchmarkResult, format: BenchmarkExportFormat): string;
+
+/**
+ * Hash verification utilities used by the benchmark integrity pipeline.
+ *
+ * These helpers produce deterministic SHA-256 digests for sealed artifacts:
+ * qrels payloads, judge prompts, dataset files, and encrypted seals. They are
+ * intentionally simple and rely only on Node's built-in crypto module so the
+ * bench package can verify seals without additional dependencies.
+ *
+ * Rules of the road:
+ * - Hashes are lowercase hex strings. Always compare with `timingSafeEqual`.
+ * - Structured inputs are serialized with sorted keys so equivalent objects
+ *   produce identical digests. This aligns with CLAUDE.md gotcha #38.
+ * - The AES-GCM seal helpers use 256-bit keys and 96-bit IVs; they are a
+ *   thin interface so CI + tests can exercise the flow without reaching for
+ *   a KMS. Production deployments should wire a real key-management backend.
+ */
+declare const INTEGRITY_HASH_ALGORITHM: "sha256";
+declare const INTEGRITY_CIPHER_ALGORITHM: "aes-256-gcm";
+interface SealedArtifact {
+    /** Version marker for the seal envelope. */
+    version: 1;
+    /** Symmetric cipher identifier. */
+    algorithm: typeof INTEGRITY_CIPHER_ALGORITHM;
+    /** Base64-encoded 96-bit IV. */
+    iv: string;
+    /** Base64-encoded 128-bit auth tag. */
+    tag: string;
+    /** Base64-encoded ciphertext. */
+    ciphertext: string;
+    /**
+     * SHA-256 of the plaintext payload. Verified after decryption as a
+     * defence-in-depth check against silent key rotation or ciphertext drift.
+     */
+    plaintextHash: string;
+}
+declare function hashString(value: string): string;
+declare function hashBytes(value: Uint8Array): string;
+/**
+ * Canonicalize a JSON-serializable value so equivalent payloads produce the
+ * same digest regardless of key insertion order.
+ */
+declare function canonicalJsonStringify(value: unknown): string;
+declare function hashCanonicalJson(value: unknown): string;
+declare function isSha256Hex(value: unknown): value is string;
+declare function assertSha256Hex(value: unknown, label: string): string;
+/**
+ * Constant-time equality check for hex digests. Returns `false` when inputs
+ * differ in length — `timingSafeEqual` would otherwise throw.
+ */
+declare function safeHexEqual(expected: string, actual: string): boolean;
+/**
+ * Encrypt a plaintext payload with AES-256-GCM, returning a seal envelope.
+ * The caller owns the key. A 96-bit IV is drawn from `crypto.randomBytes`
+ * for each call — never reuse keys across predictable IVs.
+ */
+declare function sealPayload(plaintext: string, key: Buffer): SealedArtifact;
+declare function openSeal(seal: SealedArtifact, key: Buffer): string;
+/**
+ * Load a 32-byte AES key from an environment variable. The variable must
+ * contain a base64-encoded 256-bit key. Returns `null` when unset so callers
+ * can degrade gracefully in environments without a key-management backend.
+ *
+ * The input is validated against a strict base64 pattern before decoding
+ * because Node's `Buffer.from(x, "base64")` silently ignores non-base64
+ * characters and never throws — accepting a malformed key would surface
+ * only later as an opaque decryption or hash mismatch error.
+ */
+declare function loadSealKeyFromEnv(envName: string): Buffer | null;
+
+/**
+ * Sealed qrels loader.
+ *
+ * The threat model (see `docs/bench/integrity.md`) is that the runner-side
+ * adapter never sees ground-truth answers: they live only inside the judge /
+ * scorer process. This module enforces that boundary by:
+ *
+ * 1. Loading a sealed qrels artifact from disk.
+ * 2. Verifying its declared SHA-256 hash against the expected value pinned
+ *    in the benchmark's metadata (the "seal hash").
+ * 3. Decrypting the payload only when a caller provides the correct seal
+ *    key. Callers that only need the seal hash (e.g. the runner emitting
+ *    `BenchmarkResult.meta.qrelsSealedHash`) never receive plaintext.
+ *
+ * The artifact format is JSON:
+ *
+ * ```json
+ * {
+ *   "benchmark": "<benchmark-id>",
+ *   "version": 1,
+ *   "sealHash": "<sha256-of-envelope-without-sealHash>",
+ *   "envelope": { SealedArtifact }
+ * }
+ * ```
+ *
+ * `sealHash` is computed over the canonical JSON of `envelope` so two qrels
+ * files encrypted with the same key produce distinct `sealHash` values only
+ * when their plaintext differs.
+ */
+
+interface SealedQrelsArtifact {
+    benchmark: string;
+    version: 1;
+    sealHash: string;
+    envelope: SealedArtifact;
+}
+interface SealedQrelsHandle {
+    benchmark: string;
+    sealHash: string;
+    /**
+     * Returns the decrypted qrels JSON as a string. Callers must pass the
+     * seal key explicitly; the handle never caches plaintext.
+     */
+    unseal(key: Buffer): unknown;
+}
+interface LoadSealedQrelsOptions {
+    /**
+     * Expected seal hash pinned at benchmark registration. If provided the
+     * loader rejects the artifact when the computed hash does not match.
+     */
+    expectedSealHash?: string;
+    /**
+     * Benchmark ID the artifact must declare. When omitted, any benchmark ID
+     * is accepted so tooling can inspect unknown artifacts.
+     */
+    expectedBenchmarkId?: string;
+}
+declare function isSealedQrelsArtifact(value: unknown): value is SealedQrelsArtifact;
+declare function computeSealHash(envelope: SealedArtifact): string;
+declare function parseSealedQrels(raw: string, options?: LoadSealedQrelsOptions): SealedQrelsHandle;
+declare function loadSealedQrels(filePath: string, options?: LoadSealedQrelsOptions): Promise<SealedQrelsHandle>;
+/**
+ * Serialize a sealed qrels artifact to the canonical on-disk shape. Useful
+ * for tooling that authors new qrels files.
+ */
+declare function serializeSealedQrels(artifact: SealedQrelsArtifact): string;
+
+/**
+ * Canary adapter for exploit-detection runs.
+ *
+ * A canary adapter never actually solves a benchmark task. It returns a
+ * deterministic, deliberately-wrong response to every query so that the
+ * exploit-audit workflow can measure how much score a benchmark assigns to
+ * a do-nothing system. If the canary scores above the configured floor
+ * (default `0.1`) on any benchmark, the benchmark is flagged as exploitable
+ * and demoted until fixed.
+ *
+ * This adapter must never be used in production bench runs; it exists only
+ * for the `bench-exploit-audit` CI workflow.
+ */
+
+/** The fixed reply the canary returns for every `recall`. */
+declare const CANARY_FIXED_RECALL = "__remnic_canary_response__";
+/** The score floor the canary must NOT exceed for any benchmark. */
+declare const CANARY_SCORE_FLOOR = 0.1;
+interface CanaryAdapterOptions {
+    /**
+     * Override the response string used by `recall`. Useful for running two
+     * canary variants side-by-side (e.g. empty string vs fixed string).
+     */
+    response?: string;
+    /**
+     * If true, `search` returns an empty array instead of a single fake hit.
+     * Some benchmarks rely on the retrieval surface; keeping the default
+     * "one fake hit" covers retrieval-style scorers too.
+     */
+    emptySearch?: boolean;
+}
+declare function createCanaryAdapter(options?: CanaryAdapterOptions): BenchMemoryAdapter;
+interface CanaryFloorCheck {
+    benchmark: string;
+    score: number;
+    floor: number;
+    passed: boolean;
+}
+/**
+ * Compare a canary score against the configured floor. Returns a structured
+ * result rather than throwing so callers can aggregate failures across an
+ * entire benchmark suite before reporting.
+ */
+declare function assertCanaryUnderFloor(benchmark: string, score: number, floor?: number): CanaryFloorCheck;
+
+/**
+ * Randomization helpers for the integrity pipeline.
+ *
+ * These helpers remove position-in-prompt and fixture-layout exploits:
+ * - `shuffleTasks` randomizes task order per run so a memorized task position
+ *   cannot be exploited.
+ * - `rotateDistractors` rotates multiple-choice answer positions and the set
+ *   of distractors so answer-position memorization is defeated.
+ * - `selectFixtureVariant` picks a variant by seed so each run exercises a
+ *   different fixture graph layout.
+ *
+ * All helpers are seeded. A seeded mulberry32 PRNG gives deterministic,
+ * reproducible shuffles that do not rely on `Math.random`.
+ */
+interface SeededRng {
+    /** Returns a pseudo-random number in `[0, 1)`. */
+    next(): number;
+}
+/**
+ * Deterministic 32-bit PRNG. Mulberry32 is small, fast, and sufficient for
+ * shuffling benchmark tasks. Do NOT use for cryptographic operations.
+ */
+declare function createSeededRng(seed: number): SeededRng;
+/**
+ * Fisher-Yates shuffle using a seeded PRNG. Returns a new array.
+ */
+declare function shuffleTasks<T>(tasks: readonly T[], seed: number): T[];
+interface MultipleChoiceQuestion<T> {
+    /** Correct answer. Must appear in `distractors` or be prepended below. */
+    correct: T;
+    /** Distractor pool. The correct answer may or may not be present. */
+    distractors: readonly T[];
+}
+interface RotatedChoices<T> {
+    /** The choices in rotated order. */
+    choices: T[];
+    /** The index of the correct answer in `choices`. */
+    correctIndex: number;
+}
+/**
+ * Rotate the distractor set and answer position for a multiple-choice
+ * question. The full choice pool is `[correct, ...distractors]` with
+ * duplicates removed; the pool is shuffled and the correct-answer index is
+ * reported back to the caller. Callers re-score against `correctIndex`.
+ */
+declare function rotateDistractors<T>(question: MultipleChoiceQuestion<T>, seed: number): RotatedChoices<T>;
+interface FixtureVariant<T> {
+    id: string;
+    value: T;
+}
+/**
+ * Pick one fixture variant by seed. Stable: the same seed always returns the
+ * same variant index for a given variant list length.
+ */
+declare function selectFixtureVariant<T>(variants: readonly FixtureVariant<T>[], seed: number): FixtureVariant<T>;
+
+/**
+ * YAML custom benchmark loader.
+ */
+
+declare function parseCustomBenchmark(source: string): CustomBenchmarkSpec;
+declare function loadCustomBenchmarkFile(filePath: string): Promise<CustomBenchmarkSpec>;
+
+/**
+ * Custom benchmark runner.
+ */
+
+declare function runCustomBenchmarkFile(filePath: string, options: RunBenchmarkOptions): Promise<BenchmarkResult>;
+
+type SchemaTierName = "clean" | "dirty";
+interface SchemaTierPageFrontmatter {
+    title?: string;
+    type?: string;
+    state?: string;
+    created?: string;
+    seeAlso?: string[];
+    timeline?: string[];
+}
+interface SchemaTierPage {
+    id: string;
+    owner: string;
+    namespace: string;
+    canonicalTitle: string;
+    title: string;
+    type: string;
+    createdAt: string;
+    aliases: string[];
+    body: string;
+    frontmatter: SchemaTierPageFrontmatter;
+    seeAlso: string[];
+    timeline: string[];
+    dirtySignals: string[];
+}
+interface PersonalizationRetrievalCase {
+    id: string;
+    query: string;
+    expectedPageIds: string[];
+    expectedNamespace: string;
+    expectedOwner: string;
+}
+interface TemporalRetrievalCase {
+    id: string;
+    query: string;
+    window: {
+        start: string;
+        end: string;
+    };
+    expectedPageIds: string[];
+}
+interface AbstentionRetrievalCase {
+    id: string;
+    query: string;
+    reason: "missing_fact" | "cross_tenant" | "hallucination_bait";
+}
+interface SchemaTierCorpus {
+    pages: SchemaTierPage[];
+}
+interface SchemaTierFixture {
+    seed: number;
+    clean: SchemaTierCorpus;
+    dirty: SchemaTierCorpus;
+    personalizationCases: PersonalizationRetrievalCase[];
+    temporalCases: TemporalRetrievalCase[];
+    abstentionCases: AbstentionRetrievalCase[];
+}
+declare function buildSchemaTierFixture(seed?: number): SchemaTierFixture;
+declare function buildSchemaTierSmokeFixture(seed?: number): SchemaTierFixture;
+declare const SCHEMA_TIER_FIXTURE: SchemaTierFixture;
+declare const SCHEMA_TIER_SMOKE_FIXTURE: SchemaTierFixture;
+
+/**
+ * Scoring utilities for ingestion benchmarks.
+ */
+
+declare function matchEntity(extracted: ExtractedEntity, gold: GoldEntity): boolean;
+declare function entityRecall(extracted: ExtractedEntity[], gold: GoldEntity[]): {
+    overall: number;
+    byType: Record<string, number>;
+};
+declare function linkMatches(extracted: ExtractedLink, gold: GoldLink): boolean;
+declare function backlinkF1(extracted: ExtractedLink[], gold: GoldLink[]): {
+    precision: number;
+    recall: number;
+    f1: number;
+};
+declare function schemaCompleteness(pages: ExtractedPage[], goldPages: GoldPage[], requiredFields: readonly string[]): {
+    overall: number;
+    fieldCoverage: Record<string, number>;
+};
+
+/**
+ * Synthetic email fixture generator.
+ *
+ * Produces a well-formed mbox file (~10-12 messages across 5 threads) covering
+ * the entities defined in email-gold.ts. All data is entirely synthetic — no
+ * real PII is present.
+ */
+
+declare const emailFixture: FixtureGenerator;
+
+/**
+ * Synthetic project-folder fixture for ingestion benchmarks.
+ *
+ * Generates a nested directory of markdown, JSON, and text files simulating
+ * a project workspace. All names, organisations, and content are entirely fictional.
+ */
+
+declare const projectFolderFixture: FixtureGenerator;
+
+/**
+ * Synthetic calendar ICS fixture for ingestion benchmarks.
+ *
+ * Generates a VCALENDAR file with recurring and one-off events.
+ * All names, organisations, and content are entirely fictional.
+ */
+
+declare const calendarFixture: FixtureGenerator;
+
+/**
+ * Synthetic chat transcript fixture for ingestion benchmarks.
+ *
+ * Generates a Slack-style JSON transcript across three channels and one DM.
+ * All names, organisations, and content are entirely fictional.
+ */
+
+declare const chatFixture: FixtureGenerator;
+
+/**
+ * Sealed rubric prompt registry.
+ *
+ * The canonical form of each sealed rubric prompt is a frozen string literal
+ * in this registry. The matching `.md` file in this directory is a
+ * human-readable mirror kept for reviewers — the `.md` is never loaded at
+ * runtime. This keeps bundling trivial (no filesystem assets) while still
+ * letting reviewers audit rubric text as prose.
+ *
+ * Rotation policy:
+ *   - Never edit an existing entry in place.
+ *   - Add a new key (`assistant-rubric-v2`, etc.) and ship a matching `.md`.
+ *   - Keep the old entry available so historical benchmark results remain
+ *     reproducible.
+ */
+declare const SEALED_PROMPT_REGISTRY: Readonly<Record<string, string>>;
+declare const DEFAULT_ASSISTANT_RUBRIC_ID = "assistant-rubric-v1";
+
+/**
+ * Shared types for the Assistant bench tier.
+ *
+ * Every Assistant benchmark shares the same shape:
+ *   - A synthetic memory graph (facts, stances, entities) the agent may read.
+ *   - A scenario prompt given to the agent.
+ *   - A sealed-rubric judge pass that scores the agent's output along
+ *     identity_accuracy / stance_coherence / novelty / calibration.
+ *
+ * The goal is reviewability: each benchmark folder ships a small fixture.ts
+ * that returns `AssistantScenario` values, and the runner wires the shared
+ * multi-run + bootstrap-CI infrastructure around them.
+ */
+
+interface AssistantMemoryFact {
+    id: string;
+    summary: string;
+    /**
+     * Free-form tags (topic, entity) used to render the memory-graph summary
+     * that is handed to the judge. Not shown to the agent.
+     */
+    tags?: string[];
+}
+interface AssistantStance {
+    topic: string;
+    position: string;
+}
+interface AssistantMemoryGraph {
+    userHandle: string;
+    userRole: string;
+    facts: AssistantMemoryFact[];
+    stances: AssistantStance[];
+    openThreads: string[];
+}
+interface AssistantScenario {
+    id: string;
+    title: string;
+    scenarioPrompt: string;
+    memoryGraph: AssistantMemoryGraph;
+    /**
+     * Small label describing what the scenario is meant to exercise. Useful in
+     * dashboards for filtering. Never exposed to the agent.
+     */
+    focus: string;
+}
+/**
+ * Minimal agent contract for the Assistant tier. The agent receives the
+ * scenario prompt plus a pre-rendered memory view (analogous to what the
+ * Remnic recall stack would hand to a downstream chat model), and returns
+ * its final answer text.
+ */
+interface AssistantAgent {
+    respond(request: {
+        scenarioId: string;
+        prompt: string;
+        memoryView: string;
+    }): Promise<string>;
+}
+interface AssistantRunnerOptions {
+    agent: AssistantAgent;
+    judge: StructuredJudge | undefined;
+    rubricId?: string;
+    /**
+     * Directory where per-run spot-check JSONL files are appended. Defaults to
+     * `<cwd>/benchmarks/results/spot-checks`.
+     */
+    spotCheckDir?: string;
+    /**
+     * Seed array for deterministic multi-run scheduling. When omitted the
+     * benchmark runner picks a fresh seed array via `buildBenchmarkRunSeeds`.
+     */
+    seeds?: number[];
+    /**
+     * Override used by tests and CLI smoke runs to cap iterations. Must be
+     * `>= 1`. The production contract is `>= 5` per the issue spec.
+     */
+    runCount?: number;
+    /**
+     * Random-number factory for bootstrap sampling. Injected in tests.
+     */
+    random?: () => number;
+}
+
+/**
+ * Shared runner scaffolding for the Assistant bench tier.
+ *
+ * Builds the `BenchmarkResult` shape used by the existing dashboard, but with
+ * per-dimension rubric scores (identity_accuracy, stance_coherence, novelty,
+ * calibration) and bootstrap 95% confidence intervals attached. Each
+ * scenario is executed `runCount` times (default 5) and the per-run means
+ * feed the bootstrap so the dashboard can render error bars.
+ */
+
+declare function runAssistantBenchmark(definition: BenchmarkDefinition, scenarios: AssistantScenario[], resolved: ResolvedRunBenchmarkOptions, runnerOptions: AssistantRunnerOptions): Promise<BenchmarkResult>;
+declare function renderMemoryViewForAgent(graph: AssistantMemoryGraph): string;
+declare function renderMemorySummaryForJudge(graph: AssistantMemoryGraph): string;
+
+/**
+ * Default assistant agent + judge wiring for the Assistant bench tier.
+ *
+ * The assistant tier is designed to be driven by a real provider-backed agent
+ * and a provider-backed structured judge, but we must also run deterministic
+ * smoke tests under `--test` and in CI without network access.
+ *
+ * This module provides:
+ *   - `resolveAssistantAgent()` — returns an `AssistantAgent` built from the
+ *     injected `resolved.remnicConfig.assistantAgent` hook if present, else
+ *     falls back to a deterministic agent that stringifies the memory view.
+ *   - `resolveStructuredJudge()` — mirror for the structured judge.
+ *
+ * Injection happens through `remnicConfig` because that field is already the
+ * benchmark-framework's pass-through channel for runner-specific config. The
+ * CLI will set it; tests set it directly on the options record.
+ */
+
+declare const ASSISTANT_AGENT_CONFIG_KEY = "assistantAgent";
+declare const ASSISTANT_JUDGE_CONFIG_KEY = "assistantJudge";
+declare const ASSISTANT_SEEDS_CONFIG_KEY = "assistantSeeds";
+declare const ASSISTANT_SPOT_CHECK_DIR_KEY = "assistantSpotCheckDir";
+declare const ASSISTANT_RUBRIC_ID_KEY = "assistantRubricId";
+declare function resolveAssistantAgent(resolved: ResolvedRunBenchmarkOptions): AssistantAgent;
+declare function resolveStructuredJudge(resolved: ResolvedRunBenchmarkOptions): StructuredJudge | undefined;
+declare function resolveAssistantSeeds(resolved: ResolvedRunBenchmarkOptions): number[] | undefined;
+declare function resolveAssistantSpotCheckDir(resolved: ResolvedRunBenchmarkOptions): string | undefined;
+declare function resolveAssistantRubricId(resolved: ResolvedRunBenchmarkOptions): string | undefined;
+
+declare const ASSISTANT_MORNING_BRIEF_SCENARIOS: AssistantScenario[];
+declare const ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS: AssistantScenario[];
+
+/**
+ * Assistant bench: proactive morning brief.
+ *
+ * Exercises whether the assistant can surface what the user should know and
+ * act on first when they sit down in the morning. Scored by a sealed rubric
+ * along identity_accuracy, stance_coherence, novelty, and calibration.
+ */
+
+declare const assistantMorningBriefDefinition: BenchmarkDefinition;
+declare function runAssistantMorningBriefBenchmark(options: ResolvedRunBenchmarkOptions): Promise<BenchmarkResult>;
+
+declare const ASSISTANT_MEETING_PREP_SCENARIOS: AssistantScenario[];
+declare const ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS: AssistantScenario[];
+
+/**
+ * Assistant bench: meeting prep.
+ *
+ * Given an upcoming meeting and attendees, generate a prep brief. Judged on
+ * attendee-context accuracy, topic recall, and open-thread surfacing.
+ */
+
+declare const assistantMeetingPrepDefinition: BenchmarkDefinition;
+declare function runAssistantMeetingPrepBenchmark(options: ResolvedRunBenchmarkOptions): Promise<BenchmarkResult>;
+
+declare const ASSISTANT_NEXT_BEST_ACTION_SCENARIOS: AssistantScenario[];
+declare const ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS: AssistantScenario[];
+
+/**
+ * Assistant bench: next-best-action.
+ *
+ * Given current state, what should the user do next? Judged on grounding in
+ * the memory graph (not generic advice) and on calibration — abstaining on
+ * weak-evidence questions rather than confidently inventing answers.
+ */
+
+declare const assistantNextBestActionDefinition: BenchmarkDefinition;
+declare function runAssistantNextBestActionBenchmark(options: ResolvedRunBenchmarkOptions): Promise<BenchmarkResult>;
+
+declare const ASSISTANT_SYNTHESIS_SCENARIOS: AssistantScenario[];
+declare const ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS: AssistantScenario[];
+
+/**
+ * Assistant bench: multi-document synthesis with stance.
+ *
+ * "What does the brain think about X?" — the agent must integrate across
+ * multiple memory items and reflect the user's previously-expressed stance,
+ * rather than regurgitating the single top-k chunk.
+ */
+
+declare const assistantSynthesisDefinition: BenchmarkDefinition;
+declare function runAssistantSynthesisBenchmark(options: ResolvedRunBenchmarkOptions): Promise<BenchmarkResult>;
+
+export { ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkPublishFeedOptions, type BuiltInProvider, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, type CanaryAdapterOptions, type CanaryFloorCheck, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ASSISTANT_RUBRIC_ID, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, type LlmJudge, type LlmProvider, type LoadSealedQrelsOptions, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MultipleChoiceQuestion, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, type PersonalizationRetrievalCase, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RegressionDetail, type RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededRng, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TokenUsage, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, bootstrapMeanConfidenceInterval, buildBenchmarkPublishFeed, buildBenchmarkRunSeeds, buildJudgePayload, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, chatFixture, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeSealHash, containsAnswer, createAnthropicProvider, createCanaryAdapter, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createOllamaProvider, createOpenAiCompatibleProvider, createProvider, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, emailFixture, entityRecall, exactMatch, f1Score, generateReport, getBenchmark, getBenchmarkLowerIsBetter, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, llmJudgeScore, llmJudgeScoreDetailed, loadBaseline, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseCustomBenchmark, parseRubricResponse, parseSealedQrels, precisionAtK, projectFolderFixture, recallAtK, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBenchSuite, runBenchmark, runCustomBenchmarkFile, runExplain, runSealedJudge, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectFixtureVariant, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkPublishFeed, writeBenchmarkResult, zeroScores };