@remnic/bench 9.3.701 → 9.3.703

package/dist/index.d.ts

@@ -1168,6 +1168,43 @@ interface BenchmarkArtifactEnvironment {
     /** Optional CPU architecture (arm64/x64/...). */
     arch?: string;
 }
+/**
+ * Additive tier metadata (issue #1573). Local-lab regression runs (Tier L)
+ * record `tier: "local"`; frontier leaderboard runs (Tier F) record
+ * `"frontier"`. Omitted on older artifacts — consumers treat absence as
+ * frontier for backwards compatibility (frontier is the historical default).
+ */
+type BenchmarkArtifactTier = "local" | "frontier";
+/**
+ * Hardware envelope for a local-lab run (issue #1573). Recorded so a Tier L
+ * number is never conflated with a Tier F number: the GPU, VRAM, and model
+ * quantization pin exactly what produced the result. Optional on all runs;
+ * expected (and audited) on `tier: "local"` artifacts.
+ */
+interface BenchmarkArtifactHardware {
+    /** Short GPU product id, e.g. "NVIDIA RTX 3090". */
+    gpu: string;
+    /** VRAM in gigabytes (e.g. 24). */
+    vramGb: number;
+    /** Model quantization label, e.g. "Q4_K_M" or "AWQ-int4". */
+    quantization: string;
+}
+/**
+ * Cross-tier judge calibration result recorded on local artifacts (issue
+ * #1573 PR3). The Cohen's kappa between the local and frontier judges over a
+ * fixed calibration slice; below `threshold` the local judge is flagged
+ * unreliable for the benchmark and `warning` is set.
+ */
+interface BenchmarkArtifactJudgeCalibration {
+    /** Cohen's kappa in [-1, 1] between local and frontier judge verdicts. */
+    kappa: number;
+    /** Number of paired judgements the kappa was computed over. */
+    sampleSize: number;
+    /** Kappa threshold below which `warning` is set. */
+    threshold: number;
+    /** True when `kappa < threshold` — local judge unreliable for this benchmark. */
+    warning: boolean;
+}
 interface BenchmarkArtifactPerTaskScore {
     /** Runner-assigned task ID (stable across reruns). */
     taskId: string;
@@ -1203,6 +1240,24 @@ interface BenchmarkArtifact {
     /** Total wall-clock duration in milliseconds. */
     durationMs: number;
     env: BenchmarkArtifactEnvironment;
+    /**
+     * Two-tier provenance (issue #1573). `"local"` for local-lab regression
+     * runs, `"frontier"` for leaderboard runs. Optional and additive: older
+     * artifacts omit it.
+     */
+    tier?: BenchmarkArtifactTier;
+    /**
+     * Hardware envelope for local-lab runs (issue #1573). Optional; recorded
+     * for `tier: "local"` artifacts so the GPU/VRAM/quantization that produced
+     * a number travel with it.
+     */
+    hardware?: BenchmarkArtifactHardware;
+    /**
+     * Cross-tier judge calibration (issue #1573 PR3). The Cohen's kappa between
+     * the local and frontier judges over the calibration slice; lands in
+     * subsequent local artifacts after `remnic bench judge-calibrate`.
+     */
+    judgeCalibration?: BenchmarkArtifactJudgeCalibration;
     /** Optional explanatory note (e.g. "--limit 100"). Never contains PII. */
     note?: string;
 }
@@ -1219,6 +1274,12 @@ interface BuildBenchmarkArtifactInput {
     categoryFor?: (task: TaskResult) => string | undefined;
     /** Optional free-form note (e.g. `"--limit 100"`). */
     note?: string;
+    /** Optional two-tier provenance tag (issue #1573). */
+    tier?: BenchmarkArtifactTier;
+    /** Optional hardware envelope for local-lab runs (issue #1573). */
+    hardware?: BenchmarkArtifactHardware;
+    /** Optional cross-tier judge calibration result (issue #1573 PR3). */
+    judgeCalibration?: BenchmarkArtifactJudgeCalibration;
 }
 /**
  * Build a `BenchmarkArtifact` from a runner's `BenchmarkResult`.
@@ -2866,6 +2927,219 @@ declare const chatFixture: FixtureGenerator;
 declare const SEALED_PROMPT_REGISTRY: Readonly<Record<string, string>>;
 declare const DEFAULT_ASSISTANT_RUBRIC_ID = "assistant-rubric-v1";
 
+/**
+ * Cohen's kappa — inter-rater agreement for the cross-tier judge calibration
+ * (issue #1573 PR3).
+ *
+ * The local-lab protocol (#1573) needs a single number that says whether the
+ * cheap local judge agrees with the expensive frontier judge closely enough to
+ * trust it for regression runs. Cohen's kappa is the standard measure: it
+ * corrects raw agreement for the agreement you would see by chance, so a
+ * reported 0.9 on a binary task where both judges say "correct" 95% of the
+ * time is not actually impressive.
+ *
+ * This module is pure: it operates on parallel arrays of category labels and
+ * has no I/O, no globals, and no module-level mutable state (rule 11). The
+ * calibration orchestration that turns numeric judge scores into labels and
+ * drives both judges lives in `./calibration-slice.ts`.
+ */
+/** A rater-assigned category label. Free-form string (e.g. "correct"). */
+type JudgeCategory = string;
+interface CohenKappaResult {
+    /** Cohen's kappa in [-1, 1]. 1 = perfect agreement, 0 = chance, <0 = systematic disagreement. */
+    kappa: number;
+    /** Observed proportional agreement (fraction of identical labels). In [0, 1]. */
+    observedAgreement: number;
+    /** Expected chance agreement given the marginal distributions. In [0, 1]. */
+    expectedAgreement: number;
+    /** Number of paired judgements. */
+    sampleSize: number;
+    /** Distinct category labels seen across both raters (sorted). */
+    categories: readonly JudgeCategory[];
+}
+/**
+ * Compute Cohen's kappa from two parallel arrays of category labels.
+ *
+ * Throws when the arrays have mismatched lengths or are empty — kappa on zero
+ * samples is meaningless and callers should surface that as an operator error
+ * rather than a fabricated 0.
+ *
+ * Degenerate case: when every item lands in a single category for both raters,
+ * the chance-agreement denominator collapses to zero. By convention perfect
+ * agreement in that case returns kappa = 1 (the raters agreed on everything;
+ * there just was nothing to distinguish). Imperfect agreement with a zero
+ * denominator is impossible (a single shared category forces agreement), so
+ * the branch is unreachable for well-formed input — but the guard keeps the
+ * function total and returns 0 defensively.
+ */
+declare function computeCohensKappa(raterA: readonly JudgeCategory[], raterB: readonly JudgeCategory[]): CohenKappaResult;
+/** Default correct/incorrect decision threshold for a 0..1 judge score. */
+declare const DEFAULT_JUDGE_BINARIZATION_THRESHOLD = 0.5;
+/**
+ * Map a numeric judge score to a binary "correct"/"incorrect" category label.
+ * Scores ≥ threshold → "correct"; otherwise "incorrect". Non-finite scores
+ * (NaN/Infinity from a broken judge) are bucketed as "incorrect" so a single
+ * bad verdict does not crash calibration — they still count as disagreement
+ * against a finite frontier verdict.
+ */
+declare function binarizeJudgeScore(score: number, threshold?: number): JudgeCategory;
+
+/**
+ * Cross-tier judge calibration — issue #1573 PR3.
+ *
+ * The two-tier protocol (#1573) trusts local-judge numbers for regression only
+ * when the local judge agrees with the frontier judge closely enough. This
+ * module owns three things:
+ *
+ *   1. A deterministic, content-free calibration slice — a reproducible
+ *      selection of question ids per benchmark. The slice is "committed" in
+ *      the sense that the selection algorithm is fixed and content-free
+ *      (rule: no dataset content in repo per docs/benchmarks.md ethics): the
+ *      same universe of question ids always yields the same slice, so two
+ *      operators running the same dataset compare the same questions.
+ *   2. `runJudgeCalibration` — runs both judges over the slice's cached
+ *      answers, bins each verdict to a category, and reports Cohen's kappa.
+ *   3. The kappa threshold + warning that downstream local artifacts carry
+ *      (see the `judgeCalibration` field on `BenchmarkArtifact`).
+ *
+ * The module is I/O-free except for the judge calls the caller injects; it has
+ * no module-level mutable state (rule 11) and never interpolates model/answer
+ * text into shell strings (rule 10).
+ */
+
+/**
+ * Judge identities recorded alongside a persisted calibration so a later run
+ * can verify the kappa was computed for the SAME judge pair (issue #1573 PR3,
+ * codex P2 review). Without this binding, a run that swaps the local-lab
+ * manifest or the frontier judge would inherit a stale kappa computed for a
+ * different pair.
+ */
+interface JudgeCalibrationIdentities {
+    localJudgeProvider: string;
+    localJudgeModel: string;
+    frontierJudgeProvider: string;
+    frontierJudgeModel: string;
+}
+/**
+ * The full persisted calibration record as loaded from disk: the artifact
+ * subset plus the (optional) judge identities that produced it. Identities
+ * are optional because state files written before they existed still load —
+ * the attach path treats absent identities as "unbound, attach anyway" to
+ * preserve backwards compatibility.
+ */
+type LoadedJudgeCalibrationState = BenchmarkArtifactJudgeCalibration & Partial<JudgeCalibrationIdentities>;
+/**
+ * Fixed slice size per benchmark. The issue specifies a 50-question slice; a
+ * benchmark with fewer available questions uses all of them.
+ */
+declare const CALIBRATION_SLICE_SIZE = 50;
+/**
+ * Minimum number of completed tasks a stored result must have to be a valid
+ * calibration source (codex P2 review). A `--limit 1` full run produces
+ * `mode === "full"` with a single task, yielding a degenerate one-sample κ
+ * (often 1.0). Below this floor, Cohen's kappa is statistically meaningless.
+ * Benchmarks with fewer total questions are unaffected — the slice uses all
+ * available tasks, but a capped run of a larger benchmark is rejected.
+ */
+declare const MIN_CALIBRATION_SOURCE_TASKS = 10;
+/**
+ * Kappa below this triggers a loud "local judge unreliable for this
+ * benchmark" warning in the report and on the artifact. 0.7 is the
+ * conventional "substantial agreement" cut-off (Landis & Koch 1977).
+ */
+declare const JUDGE_CALIBRATION_KAPPA_THRESHOLD = 0.7;
+/** A cached answer the calibration runs both judges over. */
+interface CalibrationAnswer {
+    /** Stable question id (matches `BenchmarkArtifactPerTaskScore.taskId`). */
+    questionId: string;
+    /** The prompt/question text the responder was asked. */
+    question: string;
+    /** The responder's produced answer text. */
+    predicted: string;
+    /** The reference / gold answer text. */
+    expected: string;
+}
+/** Per-question verdict pair produced while running calibration. */
+interface CalibrationVerdictPair {
+    questionId: string;
+    localCategory: JudgeCategory;
+    frontierCategory: JudgeCategory;
+}
+interface RunJudgeCalibrationOptions {
+    /** Benchmark id the calibration is scoped to (recorded in the result). */
+    benchmarkId: string;
+    /** The cheap local-lab judge (Tier L). */
+    localJudge: BenchJudge;
+    /** The expensive frontier judge (Tier F) — the gold standard. */
+    frontierJudge: BenchJudge;
+    /** Cached answers to judge; the slice is selected from these by question id. */
+    answers: readonly CalibrationAnswer[];
+    /**
+     * Maps a numeric judge score to a category label. Defaults to
+     * `binarizeJudgeScore` (correct/incorrect at 0.5). Both judges share one
+     * binning function so they are compared on the same scale.
+     */
+    binScore?: (score: number) => JudgeCategory;
+    /** Override the slice size (default 50; mainly for tests). */
+    sliceSize?: number;
+    /** Override the warning threshold (default 0.7). */
+    threshold?: number;
+}
+interface JudgeCalibrationResult extends CohenKappaResult {
+    benchmarkId: string;
+    /** Question ids that made up the calibrated slice. */
+    sliceQuestionIds: readonly string[];
+    /** Configured warning threshold. */
+    threshold: number;
+    /** True when `kappa < threshold` — local judge is unreliable for this benchmark. */
+    warning: boolean;
+    /** Per-question verdict pairs, in slice order. */
+    verdicts: readonly CalibrationVerdictPair[];
+}
+/**
+ * Select the calibration slice from a universe of question ids.
+ *
+ * The selection is deterministic and content-free: ids are ordered by the
+ * hex digest of `sha256(id)` and the first `size` are taken. This commits the
+ * slice to a fixed algorithm rather than a hardcoded id list (which we cannot
+ * ship without the dataset). Same universe → same slice, every time, across
+ * operators and reruns — so the slice is reproducible and comparable.
+ */
+declare function selectCalibrationSlice(questionIds: readonly string[], size?: number): string[];
+/**
+ * Run both judges over the calibration slice and report Cohen's kappa.
+ *
+ * For each answer in the slice the local and frontier judges each score
+ * `(question, predicted, expected)`; each numeric score is binned to a
+ * category (default: correct/incorrect) and the two parallel category arrays
+ * feed `computeCohensKappa`. The result is exactly the value that lands in
+ * subsequent local artifacts' `judgeCalibration.kappa`.
+ */
+declare function runJudgeCalibration(options: RunJudgeCalibrationOptions): Promise<JudgeCalibrationResult>;
+/**
+ * Persist a calibration result so subsequent local artifacts can carry the
+ * kappa (issue #1573 done-when: "a kappa number that lands in subsequent
+ * local artifacts"). The state is a single JSON file per benchmark under
+ * `<calibrationDir>/<benchmarkId>.json`, written atomically via temp-write-
+ * then-rename (cursor review: a direct writeFile can leave truncated JSON on
+ * crash, which loadJudgeCalibrationState would silently drop as a miss).
+ *
+ * Only the artifact-relevant subset (`BenchmarkArtifactJudgeCalibration`) is
+ * persisted — never the per-question verdicts or answer text (repo ethics +
+ * rule 10: nothing interpolated into shell). Optional `identities` record the
+ * judge pair that produced the kappa so a later run can refuse a stale kappa
+ * for a different pair (codex P2 review); omitted on pre-binding state files.
+ */
+declare function writeJudgeCalibrationState(result: JudgeCalibrationResult, calibrationDir: string, identities?: JudgeCalibrationIdentities): Promise<string>;
+/**
+ * Load a previously persisted calibration result for a benchmark. Returns
+ * `undefined` when no calibration has been run yet (the run path treats
+ * absence as "no calibration recorded" and omits `judgeCalibration` from
+ * the artifact). A corrupt/unparseable file is a miss returning `undefined`,
+ * never a crash (rule 34) — the operator re-runs `judge-calibrate`.
+ */
+declare function loadJudgeCalibrationState(benchmarkId: string, calibrationDir: string): Promise<LoadedJudgeCalibrationState | undefined>;
+
 /**
  * Shared runner scaffolding for the Assistant bench tier.
  *
@@ -3558,4 +3832,4 @@ interface MitigatedTargetConfig {
  */
 declare function createMitigatedTarget(config: MitigatedTargetConfig): ExtractionAttackTarget;
 
-export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, type CanaryAdapterOptions, type CanaryFloorCheck, type CodexCliProviderConfig, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MITIGATED_BASELINE_SCENARIOS, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEmailIngestionAdapterOptions, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TimelineEntry, type TokenUsage, type WriteBenchmarkArtifactResult, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, chatFixture, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeSealHash, containsAnswer, createSeededRng as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createCodexCliProvider, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$1 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeLeaderboardArtifactsForResult, zeroScores };
+export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type CodexCliProviderConfig, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEmailIngestionAdapterOptions, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TimelineEntry, type TokenUsage, type WriteBenchmarkArtifactResult, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, chatFixture, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createCodexCliProvider, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$1 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };

package/dist/index.js

@@ -5729,6 +5729,10 @@ function buildBenchmarkArtifact(input) {
   const startedAt = new Date(startedMs).toISOString();
   const finishedAt = new Date(finishedMs).toISOString();
   const durationMs = Math.max(0, finishedMs - startedMs);
+  const resultCalibration = readJudgeCalibrationFromBenchmarkOptions(
+    result.config.benchmarkOptions?.judgeCalibration
+  );
+  const judgeCalibration = input.judgeCalibration ?? resultCalibration;
   return {
     schemaVersion: BENCHMARK_ARTIFACT_SCHEMA_VERSION,
     benchmarkId: input.benchmarkId,
@@ -5750,9 +5754,26 @@ function buildBenchmarkArtifact(input) {
       os: result.environment.os,
       ...result.environment.hardware ? { arch: result.environment.hardware } : {}
     },
-    ...input.note !== void 0 ? { note: input.note } : {}
+    ...input.note !== void 0 ? { note: input.note } : {},
+    ...input.tier !== void 0 ? { tier: input.tier } : {},
+    ...input.hardware !== void 0 ? { hardware: input.hardware } : {},
+    ...judgeCalibration !== void 0 ? { judgeCalibration } : {}
   };
 }
+function readJudgeCalibrationFromBenchmarkOptions(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return void 0;
+  }
+  const record = value;
+  const kappa = record.kappa;
+  const sampleSize = record.sampleSize;
+  const threshold = record.threshold;
+  const warning = record.warning;
+  if (typeof kappa !== "number" || !Number.isFinite(kappa) || typeof sampleSize !== "number" || !Number.isFinite(sampleSize) || typeof threshold !== "number" || !Number.isFinite(threshold) || typeof warning !== "boolean") {
+    return void 0;
+  }
+  return { kappa, sampleSize, threshold, warning };
+}
 function buildBenchmarkArtifactFilename(artifact) {
   const date = sanitizeSegment(artifact.startedAt.slice(0, 10));
   const sha = sanitizeSegment((artifact.system.gitSha || "unknown").slice(0, 7));
@@ -5818,6 +5839,27 @@ function parseBenchmarkArtifact(raw) {
   requireString(env, "os");
   requireOptionalString(env, "arch", "env.arch");
   requireOptionalString(record, "note", "note");
+  if (record.tier !== void 0 && record.tier !== "local" && record.tier !== "frontier") {
+    throw new Error(
+      `BenchmarkArtifact tier must be "local" or "frontier" when provided; got ${String(record.tier)}.`
+    );
+  }
+  if (record.hardware !== void 0) {
+    const hardware = requireObject(record, "hardware");
+    requireString(hardware, "gpu");
+    requireNumber(hardware, "vramGb");
+    requireString(hardware, "quantization");
+  }
+  if (record.judgeCalibration !== void 0) {
+    const calibration = requireObject(record, "judgeCalibration");
+    requireNumber(calibration, "kappa");
+    requireNumber(calibration, "sampleSize");
+    requireNumber(calibration, "threshold");
+    const warning = calibration.warning;
+    if (typeof warning !== "boolean") {
+      throw new Error(`BenchmarkArtifact judgeCalibration.warning must be a boolean; got ${String(warning)}.`);
+    }
+  }
   const metrics = requireObject(record, "metrics");
   for (const [key, value] of Object.entries(metrics)) {
     if (typeof value !== "number" || !Number.isFinite(value)) {
@@ -32037,10 +32079,236 @@ var chatFixture = {
   }
 };
 
+// src/judges/calibration-slice.ts
+import { createHash as createHash10, randomBytes as randomBytes3 } from "crypto";
+import { mkdir as mkdir16, readFile as readFile21, rename as rename3, unlink as unlink3, writeFile as writeFile15 } from "fs/promises";
+import path34 from "path";
+
+// src/judges/cohen-kappa.ts
+function computeCohensKappa(raterA, raterB) {
+  if (raterA.length !== raterB.length) {
+    throw new Error(
+      `computeCohensKappa: rater arrays must have equal length; got ${raterA.length} and ${raterB.length}.`
+    );
+  }
+  const sampleSize = raterA.length;
+  if (sampleSize === 0) {
+    throw new Error("computeCohensKappa: cannot compute kappa over zero paired judgements.");
+  }
+  const countA = /* @__PURE__ */ new Map();
+  const countB = /* @__PURE__ */ new Map();
+  let observedAgreements = 0;
+  for (let index = 0; index < sampleSize; index += 1) {
+    const labelA = raterA[index];
+    const labelB = raterB[index];
+    if (labelA === labelB) {
+      observedAgreements += 1;
+    }
+    countA.set(labelA, (countA.get(labelA) ?? 0) + 1);
+    countB.set(labelB, (countB.get(labelB) ?? 0) + 1);
+  }
+  const categories = /* @__PURE__ */ new Set([...countA.keys(), ...countB.keys()]);
+  const observedAgreement = observedAgreements / sampleSize;
+  let expectedAgreement = 0;
+  for (const category of categories) {
+    const probA = (countA.get(category) ?? 0) / sampleSize;
+    const probB = (countB.get(category) ?? 0) / sampleSize;
+    expectedAgreement += probA * probB;
+  }
+  const denominator = 1 - expectedAgreement;
+  let kappa;
+  if (denominator === 0) {
+    kappa = observedAgreement === 1 ? 1 : 0;
+  } else {
+    kappa = (observedAgreement - expectedAgreement) / denominator;
+  }
+  return {
+    kappa,
+    observedAgreement,
+    expectedAgreement,
+    sampleSize,
+    categories: [...categories].sort()
+  };
+}
+var DEFAULT_JUDGE_BINARIZATION_THRESHOLD = 0.5;
+function binarizeJudgeScore(score, threshold = DEFAULT_JUDGE_BINARIZATION_THRESHOLD) {
+  if (typeof score !== "number" || !Number.isFinite(score)) {
+    return "incorrect";
+  }
+  return score >= threshold ? "correct" : "incorrect";
+}
+
+// src/judges/calibration-slice.ts
+var CALIBRATION_SLICE_SIZE = 50;
+var MIN_CALIBRATION_SOURCE_TASKS = 10;
+var JUDGE_CALIBRATION_KAPPA_THRESHOLD = 0.7;
+function selectCalibrationSlice(questionIds, size = CALIBRATION_SLICE_SIZE) {
+  if (!Number.isInteger(size) || size <= 0) {
+    throw new Error(`selectCalibrationSlice: size must be a positive integer; got ${String(size)}.`);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const unique = [];
+  for (const id of questionIds) {
+    if (typeof id !== "string" || id.length === 0) {
+      throw new Error(`selectCalibrationSlice: question ids must be non-empty strings; got ${String(id)}.`);
+    }
+    if (!seen.has(id)) {
+      seen.add(id);
+      unique.push(id);
+    }
+  }
+  return unique.map((id) => ({ id, digest: createHash10("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
+}
+async function runJudgeCalibration(options) {
+  const binScore = options.binScore ?? ((score) => binarizeJudgeScore(score));
+  const threshold = options.threshold ?? JUDGE_CALIBRATION_KAPPA_THRESHOLD;
+  const sliceSize = options.sliceSize ?? CALIBRATION_SLICE_SIZE;
+  const sliceIds = selectCalibrationSlice(
+    options.answers.map((answer) => answer.questionId),
+    sliceSize
+  );
+  const sliceIdSet = new Set(sliceIds);
+  const answerById = /* @__PURE__ */ new Map();
+  for (const answer of options.answers) {
+    if (sliceIdSet.has(answer.questionId) && !answerById.has(answer.questionId)) {
+      answerById.set(answer.questionId, answer);
+    }
+  }
+  const sliceAnswers = sliceIds.map((id) => answerById.get(id)).filter((answer) => answer !== void 0);
+  const localLabels = [];
+  const frontierLabels = [];
+  const verdicts = [];
+  for (const answer of sliceAnswers) {
+    const localScore = await options.localJudge.score(
+      answer.question,
+      answer.predicted,
+      answer.expected
+    );
+    const frontierScore = await options.frontierJudge.score(
+      answer.question,
+      answer.predicted,
+      answer.expected
+    );
+    const localCategory = binScore(localScore);
+    const frontierCategory = binScore(frontierScore);
+    localLabels.push(localCategory);
+    frontierLabels.push(frontierCategory);
+    verdicts.push({
+      questionId: answer.questionId,
+      localCategory,
+      frontierCategory
+    });
+  }
+  const kappaResult = computeCohensKappa(localLabels, frontierLabels);
+  const warning = kappaResult.kappa < threshold;
+  return {
+    ...kappaResult,
+    benchmarkId: options.benchmarkId,
+    sliceQuestionIds: sliceIds,
+    threshold,
+    warning,
+    verdicts
+  };
+}
+async function writeJudgeCalibrationState(result, calibrationDir, identities) {
+  await mkdir16(calibrationDir, { recursive: true });
+  const state = {
+    kappa: result.kappa,
+    sampleSize: result.sampleSize,
+    threshold: result.threshold,
+    warning: result.warning,
+    ...identities ? identities : {}
+  };
+  const filePath = path34.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
+  const tempPath = `${filePath}.${randomBytes3(6).toString("hex")}.tmp`;
+  await writeFile15(tempPath, `${JSON.stringify(state, null, 2)}
+`, "utf8");
+  try {
+    await rename3(tempPath, filePath);
+  } catch (error) {
+    await unlink3(tempPath).catch(() => void 0);
+    throw error;
+  }
+  return filePath;
+}
+async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
+  const filePath = path34.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
+  let raw;
+  try {
+    raw = await readFile21(filePath, "utf8");
+  } catch {
+    return void 0;
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    return void 0;
+  }
+  const record = parsed;
+  const kappa = record.kappa;
+  const sampleSize = record.sampleSize;
+  const threshold = record.threshold;
+  const warning = record.warning;
+  if (typeof kappa !== "number" || !Number.isFinite(kappa) || typeof sampleSize !== "number" || !Number.isFinite(sampleSize) || typeof threshold !== "number" || !Number.isFinite(threshold) || typeof warning !== "boolean") {
+    return void 0;
+  }
+  const loaded = { kappa, sampleSize, threshold, warning };
+  const identityKeys = [
+    "localJudgeProvider",
+    "localJudgeModel",
+    "frontierJudgeProvider",
+    "frontierJudgeModel"
+  ];
+  const identityValues = identityKeys.map((key) => record[key]);
+  if (identityValues.some((value) => value !== void 0)) {
+    if (identityValues.every((value) => typeof value === "string")) {
+      Object.assign(
+        loaded,
+        Object.fromEntries(identityKeys.map((key, index) => [key, identityValues[index]]))
+      );
+    }
+  }
+  return loaded;
+}
+function sanitizeCalibrationSegment(value) {
+  const lowered = value.trim().toLowerCase();
+  const chars = [];
+  let prevWasSeparator = false;
+  for (const ch of lowered) {
+    const code = ch.charCodeAt(0);
+    const isAllowed = code >= 97 && code <= 122 || // a-z
+    code >= 48 && code <= 57 || // 0-9
+    code === 95 || // _
+    code === 45;
+    if (isAllowed) {
+      const isSeparator = ch === "-";
+      if (isSeparator && prevWasSeparator) {
+        continue;
+      }
+      chars.push(ch);
+      prevWasSeparator = isSeparator;
+    } else if (!prevWasSeparator && chars.length > 0) {
+      chars.push("-");
+      prevWasSeparator = true;
+    }
+  }
+  while (chars.length > 0 && chars[0] === "-") {
+    chars.shift();
+  }
+  while (chars.length > 0 && chars[chars.length - 1] === "-") {
+    chars.pop();
+  }
+  return chars.length > 0 ? chars.join("") : "unknown";
+}
+
 // src/benchmarks/remnic/procedural-recall/ablation.ts
-import { mkdir as mkdir16, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile15, readFile as readFile21 } from "fs/promises";
+import { mkdir as mkdir17, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile16, readFile as readFile22 } from "fs/promises";
 import os9 from "os";
-import path34 from "path";
+import path35 from "path";
 import {
   StorageManager as StorageManager3,
   parseConfig as parseConfig5,
@@ -32071,7 +32339,7 @@ async function runSide(scenarios, proceduralEnabled) {
   const observed = [];
   for (const scenario of scenarios) {
     const dir = await mkdtemp11(
-      path34.join(os9.tmpdir(), "remnic-bench-proc-ablation-")
+      path35.join(os9.tmpdir(), "remnic-bench-proc-ablation-")
     );
     try {
       const storage = new StorageManager3(dir);
@@ -32086,7 +32354,7 @@ ${body}`,
       );
       const config = parseConfig5({
         memoryDir: dir,
-        workspaceDir: path34.join(dir, "ws"),
+        workspaceDir: path35.join(dir, "ws"),
         openaiApiKey: "bench-key",
         procedural: {
           enabled: proceduralEnabled,
@@ -32161,7 +32429,7 @@ async function runProceduralAblation(options) {
   };
 }
 async function loadAblationFixture(fixturePath) {
-  const raw = await readFile21(fixturePath, "utf8");
+  const raw = await readFile22(fixturePath, "utf8");
   let parsed;
   try {
     parsed = JSON.parse(raw);
@@ -32257,9 +32525,9 @@ async function runProceduralAblationCli(args) {
     random: args.random,
     seed: args.seed
   });
-  const outDir = path34.dirname(path34.resolve(args.outPath));
-  await mkdir16(outDir, { recursive: true });
-  await writeFile15(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
+  const outDir = path35.dirname(path35.resolve(args.outPath));
+  await mkdir17(outDir, { recursive: true });
+  await writeFile16(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
   return artifact;
 }
 
@@ -33344,21 +33612,25 @@ export {
   BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION,
   BENCHMARK_RESULT_SCHEMA,
   BENCHMARK_SPLIT_TYPES,
+  CALIBRATION_SLICE_SIZE,
   CANARY_FIXED_RECALL,
   CANARY_SCORE_FLOOR,
   DEFAULT_ABLATION_BOOTSTRAP_SEED,
   DEFAULT_ASSISTANT_RUBRIC_ID,
   DEFAULT_BASELINE_SCENARIOS,
+  DEFAULT_JUDGE_BINARIZATION_THRESHOLD,
   EMPTY_CONTAMINATION_MANIFEST,
   INTEGRITY_CIPHER_ALGORITHM,
   INTEGRITY_HASH_ALGORITHM,
   INTEGRITY_META_FIELDS,
+  JUDGE_CALIBRATION_KAPPA_THRESHOLD,
   LOCAL_LAB_PROVIDER_KINDS,
   LOCOMO_DATASET_FILENAMES,
   LONG_MEM_EVAL_DATASET_FILENAMES,
   LocalLabPreflightError,
   MEMORY_EVAL_DIMENSIONS,
   MEMORY_EVAL_PUBLIC_LINE,
+  MIN_CALIBRATION_SOURCE_TASKS,
   MITIGATED_BASELINE_SCENARIOS,
   OTHER_NAMESPACE_MEMORIES,
   PROCEDURAL_REAL_SCENARIOS,
@@ -33381,6 +33653,7 @@ export {
   assistantNextBestActionDefinition,
   assistantSynthesisDefinition,
   backlinkF1,
+  binarizeJudgeScore,
   bootstrapMeanConfidenceInterval,
   buildAmaBenchDiagnosticMatrixArtifact,
   buildAmaBenchDiagnosticVariantSummary,
@@ -33402,6 +33675,7 @@ export {
   clampScore,
   cohensD,
   compareResults,
+  computeCohensKappa,
   computeSealHash,
   containsAnswer,
   createSeededRng2 as createAdamSeededRng,
@@ -33475,6 +33749,7 @@ export {
   loadBenchmarkBaseline,
   loadBenchmarkResult,
   loadCustomBenchmarkFile,
+  loadJudgeCalibrationState,
   loadLoCoMo10,
   loadLocalLabManifest,
   loadLongMemEvalS,
@@ -33525,6 +33800,7 @@ export {
   runCustomBenchmarkFile,
   runExplain,
   runExtractionAttack,
+  runJudgeCalibration,
   runMitigatedBaseline,
   runProceduralAblation,
   runProceduralAblationCli,
@@ -33536,6 +33812,7 @@ export {
   schemaCompleteness,
   sealPayload,
   selectAmaBenchDiagnosticVariants,
+  selectCalibrationSlice,
   selectFixtureVariant,
   serializeBenchmarkArtifact,
   serializeJsonl,
@@ -33547,6 +33824,7 @@ export {
   writeBenchmarkPublishFeed,
   writeBenchmarkReproManifest,
   writeBenchmarkResult,
+  writeJudgeCalibrationState,
   writeLeaderboardArtifactsForResult,
   zeroScores
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/bench",
-  "version": "9.3.701",
+  "version": "9.3.703",
   "description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
   "type": "module",
   "main": "./dist/index.js",
@@ -36,7 +36,7 @@
   "dependencies": {
     "hyparquet": "^1.25.7",
     "yaml": "^2.4.2",
-    "@remnic/core": "^9.3.701"
+    "@remnic/core": "^9.3.703"
   },
   "devDependencies": {
     "tsup": "^8.5.1",