@remnic/bench 9.6.18 → 9.6.20

package/dist/index.d.ts

@@ -104,6 +104,25 @@ interface BenchResponse {
 interface BenchPhaseControl {
     signal?: AbortSignal;
 }
+type BenchRecallSupportStatus = "supported" | "weak" | "empty" | "unavailable" | "backend_failure";
+/**
+ * Answer-time support signal for the exact recall text supplied to the
+ * responder. `weak` is intentionally explicit: callers must not infer it from
+ * a zero-hit auxiliary search when `recalledText` contains evidence from other
+ * retrieval tiers.
+ */
+interface BenchRecallSupportAssessment {
+    status: BenchRecallSupportStatus;
+    reason?: string;
+    evidenceCount?: number;
+    maxScore?: number;
+    supportThreshold?: number;
+}
+interface BenchRecallSupportRequest {
+    query: string;
+    recalledText: string;
+    sessionIds: readonly string[];
+}
 interface BenchResponder {
     respond(question: string, recalledText: string, control?: BenchPhaseControl): Promise<BenchResponse>;
 }
@@ -116,6 +135,20 @@ interface BenchJudgeResult {
     latencyMs: number;
     model?: string;
 }
+interface MemCorrectJudgeRequest {
+    taskId: string;
+    query: string;
+    retiredContent: string[];
+    correctedContent: string[];
+    postCorrectionRecall: string[];
+    postMaintenanceRecall: string[];
+    postReingestRecall: string[];
+}
+interface MemCorrectJudgeResult extends BenchJudgeResult {
+    decision: "pass" | "partial" | "fail";
+    reason: string;
+    rubricVersion: string;
+}
 interface BenchJudge {
     score(question: string, predicted: string, expected: string, control?: BenchPhaseControl): Promise<number>;
     scoreWithMetrics?(question: string, predicted: string, expected: string, control?: BenchPhaseControl): Promise<BenchJudgeResult>;
@@ -126,10 +159,18 @@ interface BenchJudge {
      * judge prompt would change the metric contract.
      */
     scoreBinaryPrompt?(prompt: string, control?: BenchPhaseControl): Promise<BenchJudgeResult>;
+    judgeMemCorrectCorrectionAcceptance?(request: MemCorrectJudgeRequest, control?: BenchPhaseControl): Promise<MemCorrectJudgeResult>;
+    judgeMemCorrectStaleMemoryHarm?(request: MemCorrectJudgeRequest, control?: BenchPhaseControl): Promise<MemCorrectJudgeResult>;
 }
 interface BenchMemoryAdapter {
     store(sessionId: string, messages: Message[], control?: BenchPhaseControl): Promise<void>;
     recall(sessionId: string, query: string, budgetChars?: number, options?: BenchRecallOptions, control?: BenchPhaseControl): Promise<string>;
+    /**
+     * Optionally assess support using the exact, final recall context that will
+     * be sent to the responder. Implementations may return `weak` only from
+     * explicit evidence-confidence signals derived from that context.
+     */
+    assessRecallSupport?(request: BenchRecallSupportRequest, control?: BenchPhaseControl): Promise<BenchRecallSupportAssessment>;
     search(query: string, limit: number, sessionId?: string, control?: BenchPhaseControl): Promise<SearchResult[]>;
     /**
      * Optional explicit-correction surface (issue #1584 plan item 2a). Routes a
@@ -248,6 +289,8 @@ type BenchReasoningEffort = "low" | "medium" | "high" | "xhigh";
 interface ProviderConfig {
     provider: BuiltInProvider;
     model: string;
+    /** Versioned grading rubric used by judge providers, persisted for reproducibility. */
+    rubricVersion?: string;
     baseUrl?: string;
     apiKey?: string;
     retryOptions?: {
@@ -440,6 +483,8 @@ interface RunBenchmarkOptions {
     amaBenchJudgeProtocol?: AmaBenchJudgeProtocol;
     amaBenchCrossJudge?: BenchJudge;
     amaBenchCrossJudgeProvider?: ProviderConfig | null;
+    /** Live specialized judge used by MemCorrect; never persisted in result config. */
+    memCorrectJudge?: BenchJudge;
     /**
      * Force-disable the content-keyed judge-result cache (#1573 PR1). When
      * true, every judge call reaches the underlying model regardless of
@@ -898,6 +943,107 @@ interface RemnicAdapterOptions {
 declare const createLightweightAdapter: (options?: RemnicAdapterOptions) => Promise<BenchMemoryAdapter>;
 declare const createRemnicAdapter: (options?: RemnicAdapterOptions) => Promise<BenchMemoryAdapter>;
 
+type McpBackendErrorCode = "backend_unusable" | "transport_failure" | "tool_failure" | "invalid_response";
+type McpBackendResult<T> = {
+    ok: true;
+    value: T;
+} | {
+    ok: false;
+    error: McpBackendErrorCode;
+    detail: string;
+    cause?: unknown;
+};
+declare class McpMemoryBackendError extends Error {
+    readonly code: McpBackendErrorCode;
+    readonly detail: string;
+    constructor(result: Extract<McpBackendResult<never>, {
+        ok: false;
+    }>);
+}
+interface McpStdioTransportConfig {
+    type: "stdio";
+    command: string;
+    args?: string[];
+    cwd?: string;
+    env?: Record<string, string>;
+}
+interface McpHttpTransportConfig {
+    type: "http";
+    url: string;
+    bearerToken?: string;
+    headers?: Record<string, string>;
+}
+type McpMemoryTransportConfig = McpStdioTransportConfig | McpHttpTransportConfig;
+type McpToolOperation = "store" | "recall" | "correct" | "reset";
+type McpArgumentSemantic = "namespace" | "sessionId" | "content" | "role" | "timestamp" | "query" | "limit";
+interface McpToolMappingEntry {
+    name: string;
+    /** Map semantic input names to the server tool's argument names. */
+    arguments?: Partial<Record<McpArgumentSemantic, string>>;
+    /** Optional dot path into structured/JSON output, e.g. `data.memories`. */
+    resultPath?: string;
+}
+type McpToolMappingValue = string | McpToolMappingEntry;
+type McpMemoryToolMapping = Partial<Record<McpToolOperation, McpToolMappingValue>>;
+interface McpListedTool {
+    name: string;
+    inputSchema?: {
+        properties?: Record<string, unknown>;
+    };
+}
+interface McpToolCallResult {
+    content?: Array<{
+        type?: string;
+        text?: string;
+    }>;
+    structuredContent?: Record<string, unknown>;
+    isError?: boolean;
+}
+interface McpToolClient {
+    listTools(control?: BenchPhaseControl): Promise<McpListedTool[]>;
+    callTool(name: string, args: Record<string, unknown>, control?: BenchPhaseControl): Promise<McpToolCallResult>;
+    close(): Promise<void>;
+}
+interface McpMemoryAdapterOptions {
+    transport: McpMemoryTransportConfig;
+    tools?: McpMemoryToolMapping;
+    /** Stable override for reproducible tests; production defaults are unique. */
+    namespacePrefix?: string;
+    label?: string;
+    skipPreflight?: boolean;
+    /** Bounds connection, discovery, and the conformance canary. */
+    timeoutMs?: number;
+    clientFactory?: (transport: McpMemoryTransportConfig, control?: BenchPhaseControl) => Promise<McpToolClient>;
+}
+type McpConformanceResult = {
+    ok: true;
+    value: {
+        tools: Record<McpToolOperation, string>;
+        namespace: string;
+    };
+} | {
+    ok: false;
+    error: "backend_unusable";
+    detail: string;
+    cause?: unknown;
+};
+interface McpBenchMemoryAdapter extends BenchMemoryAdapter {
+    readonly label: string;
+    readonly namespacePrefix: string;
+    preflight(control?: BenchPhaseControl): Promise<McpConformanceResult>;
+}
+declare function createMcpMemoryAdapter(options: McpMemoryAdapterOptions): Promise<McpBenchMemoryAdapter>;
+/** Create the packaged, deterministic, keyless stdio MCP demo adapter. */
+declare function createMcpDemoMemoryAdapter(options?: Omit<McpMemoryAdapterOptions, "transport">): Promise<McpBenchMemoryAdapter>;
+interface McpMemCorrectAdapter extends MemCorrectSystemAdapter {
+    readonly namespacePrefix: string;
+    preflight(): Promise<McpConformanceResult>;
+    destroy(): Promise<void>;
+}
+declare function createMcpMemCorrectAdapter(options: McpMemoryAdapterOptions): Promise<McpMemCorrectAdapter>;
+/** MemCorrect facade over the packaged deterministic demo MCP server. */
+declare function createMcpDemoMemCorrectAdapter(options?: Omit<McpMemoryAdapterOptions, "transport">): Promise<McpMemCorrectAdapter>;
+
 interface TimeoutGuardOptions {
     benchmarkId: string;
     timeoutMs?: number;
@@ -980,6 +1126,8 @@ interface DiscoveredModel {
 }
 interface ProviderBaseConfig {
     model: string;
+    /** Versioned grading rubric used by judge providers, persisted for reproducibility. */
+    rubricVersion?: string;
     baseUrl?: string;
     apiKey?: string;
     headers?: Record<string, string>;
@@ -1223,6 +1371,9 @@ declare const BENCHMARK_RESULT_SCHEMA: {
                             readonly model: {
                                 readonly type: "string";
                             };
+                            readonly rubricVersion: {
+                                readonly type: "string";
+                            };
                             readonly baseUrl: {
                                 readonly type: "string";
                             };
@@ -1386,6 +1537,11 @@ interface BenchmarkReproManifestResult {
     seeds: number[];
     taskCount: number;
     configHash: string;
+    judge: {
+        provider: string;
+        model: string;
+        rubricVersion: string | null;
+    } | null;
 }
 interface BenchmarkReproManifest {
     schemaVersion: number;
@@ -1551,6 +1707,20 @@ interface BenchmarkArtifactJudgeCalibration {
     threshold: number;
     /** True when `kappa < threshold` — local judge unreliable for this benchmark. */
     warning: boolean;
+    /** Optional paired-bootstrap percentile interval (added by issue #1877). */
+    confidenceInterval?: {
+        lower: number;
+        upper: number;
+        level: number;
+    };
+    /** Number of paired bootstrap resamples used for the interval. */
+    bootstrapSamples?: number;
+    /** SHA-256 of the exact ordered answer set used for calibration. */
+    answerSetHash?: string;
+    /** Stored benchmark result that supplies the pinned answer payload. */
+    sourceResultId?: string;
+    /** Ordered, bounded task ids that make the pinned calibration slice auditable. */
+    sliceQuestionIds?: readonly string[];
 }
 interface BenchmarkArtifactPerTaskScore {
     /** Runner-assigned task ID (stable across reruns). */
@@ -1708,6 +1878,66 @@ interface ClaudeCliProviderDeps {
 }
 declare function createClaudeCliProvider(config: ClaudeCliProviderConfig, deps?: ClaudeCliProviderDeps): LlmProvider;
 
+type StructuredJudgeErrorCode = "api_error" | "rate_limited" | "refusal" | "malformed_response" | "malformed_verdict" | "incomplete_response" | "transport_error" | "aborted";
+interface StructuredJudgeTelemetry {
+    model: string;
+    rubricVersion: string;
+    inputTokens: number;
+    outputTokens: number;
+    latencyMs: number;
+    errorCode?: StructuredJudgeErrorCode;
+    httpStatus?: number;
+}
+interface StructuredJudgeVerdict {
+    score: number;
+    decision: "pass" | "partial" | "fail";
+    reason: string;
+}
+type StructuredJudgeVerdictResult = {
+    ok: true;
+    verdict: StructuredJudgeVerdict;
+    telemetry: StructuredJudgeTelemetry;
+} | {
+    ok: false;
+    error: {
+        code: StructuredJudgeErrorCode;
+        message: string;
+        retryable: boolean;
+        httpStatus?: number;
+    };
+    telemetry: StructuredJudgeTelemetry;
+};
+interface StructuredVerdictRequest {
+    rubric: string;
+    rubricVersion: string;
+    input: string;
+    signal?: AbortSignal;
+    maxTokens?: number;
+}
+interface AssistantRubricRequest {
+    system: string;
+    user: string;
+    rubricId: string;
+}
+interface StructuredJudgeProvider extends LlmProvider {
+    judge(request: StructuredVerdictRequest): Promise<StructuredJudgeVerdictResult>;
+    evaluateAssistantRubric(request: AssistantRubricRequest): Promise<string>;
+    createJudgeError?(failure: Extract<StructuredJudgeVerdictResult, {
+        ok: false;
+    }>): Error;
+}
+declare class StructuredJudgeError extends Error {
+    readonly code: StructuredJudgeErrorCode;
+    readonly retryable: boolean;
+    readonly httpStatus?: number;
+    readonly telemetry: StructuredJudgeTelemetry;
+    constructor(failure: Extract<StructuredJudgeVerdictResult, {
+        ok: false;
+    }>);
+}
+declare function isStructuredJudgeProvider(provider: LlmProvider): provider is StructuredJudgeProvider;
+declare function createStructuredBenchJudge(provider: StructuredJudgeProvider, rubricVersion?: string): BenchJudge;
+
 interface CodexCliRunRequest {
     executable: string;
     args: string[];
@@ -1731,8 +1961,13 @@ interface CodexCliProviderDeps {
         status: number | null;
         stderr: string;
     }>;
+    runCodexLoginStatus?: (executable: string, env: NodeJS.ProcessEnv) => Promise<{
+        status: number | null;
+        stdout: string;
+        stderr: string;
+    }>;
 }
-declare function createCodexCliProvider(config: CodexCliProviderConfig, deps?: CodexCliProviderDeps): LlmProvider;
+declare function createCodexCliProvider(config: CodexCliProviderConfig, deps?: CodexCliProviderDeps): StructuredJudgeProvider;
 
 /**
  * Result enrichment and JSON writing helpers.
@@ -1776,6 +2011,7 @@ declare function answerBenchmarkQuestion(options: {
     answerFormat?: BenchmarkAnswerFormat;
     questionContext?: BenchmarkQuestionContext;
     retryUnknownWithEvidence?: boolean;
+    recallSupport?: BenchRecallSupportAssessment;
 }): Promise<BenchmarkAnswerResult>;
 
 interface LeaderboardArtifactWrite {
@@ -2091,6 +2327,82 @@ declare function createOllamaProvider(config: OllamaProviderConfig): LlmProvider
 
 declare function createOpenAiCompatibleProvider(config: OpenAiCompatibleProviderConfig): LlmProvider;
 
+/**
+ * OpenAI Responses API provider dedicated to benchmark judging.
+ *
+ * This intentionally sits beside openai-compatible.ts. The latter targets
+ * Chat Completions-compatible third-party servers and must remain unchanged.
+ */
+
+declare const DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL = "gpt-5.6";
+type OpenAiResponsesJudgeErrorCode = StructuredJudgeErrorCode;
+type OpenAiResponsesJudgeTelemetry = StructuredJudgeTelemetry;
+type OpenAiResponsesVerdict = StructuredJudgeVerdict;
+type OpenAiResponsesVerdictResult = StructuredJudgeVerdictResult;
+interface OpenAiResponsesProviderConfig extends Omit<OpenAiCompatibleProviderConfig, "provider" | "model"> {
+    provider?: "openai";
+    model?: string;
+    rubricVersion?: string;
+}
+declare class OpenAiResponsesJudgeError extends Error {
+    readonly code: OpenAiResponsesJudgeErrorCode;
+    readonly retryable: boolean;
+    readonly httpStatus?: number;
+    readonly telemetry: OpenAiResponsesJudgeTelemetry;
+    constructor(failure: Extract<OpenAiResponsesVerdictResult, {
+        ok: false;
+    }>);
+}
+declare class OpenAiResponsesProvider implements StructuredJudgeProvider {
+    readonly provider: "openai";
+    readonly id: string;
+    readonly name: string;
+    readonly rubricVersion: string;
+    private readonly config;
+    private usage;
+    private readonly telemetryEvents;
+    constructor(config?: OpenAiResponsesProviderConfig);
+    complete(prompt: string, opts?: CompletionOpts): Promise<CompletionResult>;
+    judge(request: StructuredVerdictRequest): Promise<OpenAiResponsesVerdictResult>;
+    evaluateAssistantRubric(request: {
+        system: string;
+        user: string;
+        rubricId: string;
+    }): Promise<string>;
+    getUsage(): TokenUsage;
+    resetUsage(): void;
+    getTelemetryEvents(): OpenAiResponsesJudgeTelemetry[];
+    createJudgeError(failure: Extract<OpenAiResponsesVerdictResult, {
+        ok: false;
+    }>): OpenAiResponsesJudgeError;
+    private parseResponse;
+    private failure;
+    private transportFailure;
+    private asTransportError;
+    private telemetry;
+    private recordTelemetry;
+    private recordUsage;
+    private responsesUrl;
+    private headers;
+}
+declare function createOpenAiResponsesProvider(config?: OpenAiResponsesProviderConfig): OpenAiResponsesProvider;
+declare function createOpenAiResponsesBenchJudge(config?: OpenAiResponsesProviderConfig, provider?: OpenAiResponsesProvider): BenchJudge;
+declare function judgeMemCorrectCorrectionAcceptance(provider: OpenAiResponsesProvider, input: string, signal?: AbortSignal): Promise<OpenAiResponsesVerdictResult>;
+declare function judgeMemCorrectStaleMemoryHarm(provider: OpenAiResponsesProvider, input: string, signal?: AbortSignal): Promise<OpenAiResponsesVerdictResult>;
+
+/**
+ * Versioned GPT-5.6 judge rubrics for Build Week issue #1870.
+ *
+ * Keep these prompts stable once an artifact cites their version. A wording
+ * change is a measurement change and must receive a new version string.
+ */
+declare const OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION = "openai-responses-bench-v1";
+declare const MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION = "memcorrect-correction-acceptance-v1";
+declare const MEMCORRECT_STALE_HARM_RUBRIC_VERSION = "memcorrect-stale-memory-harm-v1";
+declare const GENERAL_ANSWER_JUDGE_RUBRIC: string;
+declare const MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC: string;
+declare const MEMCORRECT_STALE_HARM_RUBRIC: string;
+
 /**
  * Shared types for the Assistant bench tier.
  *
@@ -2647,6 +2959,70 @@ declare function interpretEffectSize(cohensDValue: number): EffectSizeInterpreta
 declare function compareResults(baseline: BenchmarkResult, candidate: BenchmarkResult, threshold?: number, lowerIsBetter?: ReadonlySet<string>): ComparisonResult;
 declare function getBenchmarkLowerIsBetter(benchmarkId: string): ReadonlySet<string>;
 
+interface LoComoProfileArtifactEvidence {
+    artifact: BenchmarkArtifact;
+    reference: string;
+    sha256: string;
+}
+interface LoComoMetricDelta {
+    baselineMean: number;
+    realMean: number;
+    delta: number;
+    aggregateContribution: number;
+    wins: number;
+    losses: number;
+    ties: number;
+}
+interface LoComoCategoryDelta {
+    category: string;
+    taskCount: number;
+    metrics: Record<string, LoComoMetricDelta>;
+}
+interface LoComoTaskRegression {
+    taskId: string;
+    category: string;
+    baselineScore: number;
+    realScore: number;
+    delta: number;
+}
+interface LoComoProfileDeltaReport {
+    schemaVersion: 1;
+    benchmarkId: "locomo";
+    comparison: {
+        baseline: {
+            reference: string;
+            sha256: string;
+        };
+        real: {
+            reference: string;
+            sha256: string;
+        };
+        datasetVersion: string;
+        model: string;
+        seed: number;
+        gitSha: string;
+        tier: string;
+    };
+    taskCount: number;
+    primaryMetric: string;
+    metrics: string[];
+    overall: Record<string, LoComoMetricDelta>;
+    categories: LoComoCategoryDelta[];
+    topRegressions: LoComoTaskRegression[];
+    evidenceBoundary: {
+        scoreDiagnosis: "complete";
+        recallRootCause: "requires-paired-recall-receipts";
+    };
+}
+interface DiagnoseLoComoProfileDeltaOptions {
+    baseline: LoComoProfileArtifactEvidence;
+    real: LoComoProfileArtifactEvidence;
+    primaryMetric?: string;
+    maxRegressions?: number;
+}
+declare function diagnoseLoComoProfileDelta(options: DiagnoseLoComoProfileDeltaOptions): LoComoProfileDeltaReport;
+declare function renderLoComoProfileDeltaMarkdown(report: LoComoProfileDeltaReport): string;
+
 /**
  * Dataset-contamination guard.
  *
@@ -2699,6 +3075,13 @@ declare function checkDatasetContamination(datasetHash: string, manifest?: Conta
 declare function addContaminationEntry(manifest: ContaminationManifest, entry: ContaminationEntry): ContaminationManifest;
 declare function mergeContaminationManifests(...manifests: ContaminationManifest[]): ContaminationManifest;
 
+interface ReportCardProvenanceContext {
+    /** Human-readable reference to the manifest that covers this result. */
+    manifestReference?: string;
+    /** Reproducibility manifest artifact hash, when the manifest records one. */
+    artifactHash?: string;
+}
+
 interface StoredBenchmarkResultSummary {
     id: string;
     path: string;
@@ -2738,6 +3121,8 @@ interface PublishedBenchmarkFeedEntry {
     aggregateMetrics: BenchmarkResult["results"]["aggregates"];
     cost: BenchmarkResult["cost"];
     environment: BenchmarkResult["environment"];
+    /** Self-contained HTML report card. Optional so older persisted feeds remain valid. */
+    reportCardHtml?: string;
     integrity: {
         splitType: NonNullable<BenchmarkResult["meta"]["splitType"]>;
         qrelsSealedHash: string;
@@ -2767,6 +3152,11 @@ interface PublishedBenchmarkFeed {
 }
 declare function defaultBenchmarkBaselineDir(): string;
 declare function defaultBenchmarkPublishPath(target: BenchmarkPublishTarget): string;
+/**
+ * Resolve explicit report-card provenance from an actual adjacent manifest.
+ * A stale manifest that does not name the selected result is ignored.
+ */
+declare function loadBenchmarkReportCardProvenance(outputDir: string, resultId: string): Promise<ReportCardProvenanceContext>;
 declare function loadBenchmarkResult(filePath: string): Promise<BenchmarkResult>;
 declare function listBenchmarkResults(outputDir: string): Promise<StoredBenchmarkResultSummary[]>;
 declare function saveBenchmarkBaseline(baselineDir: string, name: string, result: BenchmarkResult, source?: {
@@ -2798,7 +3188,9 @@ interface PublishSkipRecord {
 }
 declare function buildBenchmarkPublishFeed(outputDir: string, target: BenchmarkPublishTarget, options?: BuildBenchmarkPublishFeedOptions): Promise<PublishedBenchmarkFeed>;
 declare function writeBenchmarkPublishFeed(feed: PublishedBenchmarkFeed, outputPath: string): Promise<string>;
-declare function renderBenchmarkResultExport(result: BenchmarkResult, format: BenchmarkExportFormat): string;
+declare function renderBenchmarkResultExport(result: BenchmarkResult, format: BenchmarkExportFormat, options?: {
+    reportCardProvenance?: ReportCardProvenanceContext;
+}): string;
 
 interface HaystackTurn {
     role: "user" | "assistant";
@@ -3328,6 +3720,25 @@ interface CohenKappaResult {
     /** Distinct category labels seen across both raters (sorted). */
     categories: readonly JudgeCategory[];
 }
+interface KappaConfidenceInterval {
+    lower: number;
+    upper: number;
+    level: number;
+}
+interface BootstrapKappaOptions {
+    /** Number of paired bootstrap resamples. */
+    iterations?: number;
+    /** Confidence level in (0, 1). */
+    level?: number;
+    /** Optional deterministic seed. Derived from the labels when omitted. */
+    seed?: number;
+}
+interface BootstrapKappaResult {
+    confidenceInterval: KappaConfidenceInterval;
+    bootstrapSamples: number;
+}
+declare const DEFAULT_KAPPA_BOOTSTRAP_SAMPLES = 2000;
+declare const DEFAULT_KAPPA_CONFIDENCE_LEVEL = 0.95;
 /**
  * Compute Cohen's kappa from two parallel arrays of category labels.
  *
@@ -3344,6 +3755,15 @@ interface CohenKappaResult {
  * function total and returns 0 defensively.
  */
 declare function computeCohensKappa(raterA: readonly JudgeCategory[], raterB: readonly JudgeCategory[]): CohenKappaResult;
+/**
+ * Deterministic paired-bootstrap percentile interval for Cohen's kappa.
+ *
+ * Each resample draws paired verdict indexes, preserving the dependence
+ * between the two raters. The default seed is derived from the complete label
+ * vectors, so the same calibration answer set and verdicts produce byte-for-
+ * byte identical confidence bounds across reruns.
+ */
+declare function bootstrapCohensKappaConfidenceInterval(raterA: readonly JudgeCategory[], raterB: readonly JudgeCategory[], options?: BootstrapKappaOptions): BootstrapKappaResult;
 /** Default correct/incorrect decision threshold for a 0..1 judge score. */
 declare const DEFAULT_JUDGE_BINARIZATION_THRESHOLD = 0.5;
 /**
@@ -3398,12 +3818,20 @@ interface JudgeCalibrationIdentities {
  * the attach path treats absent identities as "unbound, attach anyway" to
  * preserve backwards compatibility.
  */
-type LoadedJudgeCalibrationState = BenchmarkArtifactJudgeCalibration & Partial<JudgeCalibrationIdentities>;
+type LoadedJudgeCalibrationState = BenchmarkArtifactJudgeCalibration & Partial<JudgeCalibrationIdentities> & {
+    /** Stored-result id that pins the answer payload across recalibrations. */
+    sourceResultId?: string;
+    /** Stable hash of the exact question/predicted/expected triples judged. */
+    answerSetHash?: string;
+    /** Question ids in the pinned calibration slice, in verdict order. */
+    sliceQuestionIds?: readonly string[];
+};
 /**
- * Fixed slice size per benchmark. The issue specifies a 50-question slice; a
+ * Fixed slice size per benchmark. Issue #1877 raises the calibration target
+ * from 50 to 200 to reduce slice sensitivity; a
  * benchmark with fewer available questions uses all of them.
  */
-declare const CALIBRATION_SLICE_SIZE = 50;
+declare const CALIBRATION_SLICE_SIZE = 200;
 /**
  * Minimum number of completed tasks a stored result must have to be a valid
  * calibration source (codex P2 review). A `--limit 1` full run produces
@@ -3455,6 +3883,14 @@ interface RunJudgeCalibrationOptions {
     sliceSize?: number;
     /** Override the warning threshold (default 0.7). */
     threshold?: number;
+    /** Override paired-bootstrap resamples (default 2,000; mainly for tests). */
+    bootstrapSamples?: number;
+    /** Override confidence level (default 0.95). */
+    confidenceLevel?: number;
+    /** Exact persisted question-id slice to reuse instead of reselecting. */
+    pinnedQuestionIds?: readonly string[];
+    /** Fail before judge calls when the pinned answer payload changed. */
+    expectedAnswerSetHash?: string;
 }
 interface JudgeCalibrationResult extends CohenKappaResult {
     benchmarkId: string;
@@ -3464,6 +3900,12 @@ interface JudgeCalibrationResult extends CohenKappaResult {
     threshold: number;
     /** True when `kappa < threshold` — local judge is unreliable for this benchmark. */
     warning: boolean;
+    /** Paired-bootstrap percentile interval for kappa. */
+    confidenceInterval: KappaConfidenceInterval;
+    /** Number of paired-bootstrap resamples. */
+    bootstrapSamples: number;
+    /** SHA-256 of the exact ordered answer triples used for calibration. */
+    answerSetHash: string;
     /** Per-question verdict pairs, in slice order. */
     verdicts: readonly CalibrationVerdictPair[];
 }
@@ -3501,7 +3943,9 @@ declare function runJudgeCalibration(options: RunJudgeCalibrationOptions): Promi
  * judge pair that produced the kappa so a later run can refuse a stale kappa
  * for a different pair (codex P2 review); omitted on pre-binding state files.
  */
-declare function writeJudgeCalibrationState(result: JudgeCalibrationResult, calibrationDir: string, identities?: JudgeCalibrationIdentities): Promise<string>;
+declare function writeJudgeCalibrationState(result: JudgeCalibrationResult, calibrationDir: string, identities?: JudgeCalibrationIdentities, provenance?: {
+    sourceResultId: string;
+}): Promise<string>;
 /**
  * Load a previously persisted calibration result for a benchmark. Returns
  * `undefined` when no calibration has been run yet (the run path treats
@@ -4634,4 +5078,4 @@ declare function checkCodingGraphRegression(report: CodingGraphBenchReport, base
  */
 declare function buildBaselineFromReport(report: CodingGraphBenchReport, note: string): CodingGraphBaseline;
 
-export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
+export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoComoCategoryDelta, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoComoProfileDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };