@remnic/bench 9.6.23 → 9.6.25

package/dist/index.d.ts

@@ -1785,6 +1785,10 @@ interface BenchmarkArtifactJudgeCalibration {
     sourceResultId?: string;
     /** Ordered, bounded task ids that make the pinned calibration slice auditable. */
     sliceQuestionIds?: readonly string[];
+    /** Sanitized full local judge provider configuration hash. */
+    localJudgeConfigHash?: string;
+    /** Sanitized full frontier judge provider configuration hash. */
+    frontierJudgeConfigHash?: string;
 }
 interface BenchmarkArtifactPerTaskScore {
     /** Runner-assigned task ID (stable across reruns). */
@@ -2330,6 +2334,8 @@ declare function createDeterministicSpotCheckLogger(options: {
 declare function zeroScores(): AssistantRubricScores;
 declare function clampScore(value: number): number;
 
+/** Hash of the exact rubric/prompt surface used by `createProviderBackedJudge`. */
+declare function getProviderBackedJudgePromptIdentity(config: ProviderFactoryConfig): string;
 interface GatewayResponderOptions {
     gatewayConfig?: GatewayConfig;
     agentId?: string;
@@ -2922,8 +2928,11 @@ interface ResolveBenchRuntimeProfileOptions {
     disableThinking?: boolean;
     /**
      * Path to a local-lab manifest JSON file (issue #1573 PR2). Required when
-     * `runtimeProfile: "local-lab"`. The manifest pins responder/judge/embedding
-     * to operator-hosted models with temperature=0 and a fixed seed.
+     * `runtimeProfile: "local-lab"`. For other profiles, an explicit manifest
+     * binds the benchmark judge to the manifest's normalized judge config while
+     * leaving the profile's responder unchanged. The manifest pins
+     * provider/model/baseUrl/temperature/seed; runtime options are overlaid by
+     * the same resolver used by `judge-calibrate`.
      */
     localLabManifestPath?: string;
 }
@@ -2949,6 +2958,15 @@ interface ResolvedBenchRuntimeProfile {
     localLab?: ResolvedLocalLabProfile;
 }
 declare function resolveBenchRuntimeProfile(options: ResolveBenchRuntimeProfileOptions): Promise<ResolvedBenchRuntimeProfile>;
+/**
+ * Resolve the local judge configuration that is shared by calibration and
+ * benchmark execution. Keeping manifest normalization and runtime overlays in
+ * one path prevents the persisted calibration hash from describing a subtly
+ * different judge (for example, a bare Ollama URL without temperature/seed).
+ */
+declare function resolveLocalLabJudgeProviderConfig(options: Pick<ResolveBenchRuntimeProfileOptions, "requestTimeout" | "max429WaitMs" | "disableThinking"> & {
+    localLabManifestPath: string;
+}): Promise<ProviderConfig>;
 
 /**
  * Published benchmark registry for @remnic/bench phase 1.
@@ -3998,6 +4016,9 @@ type LoadedJudgeCalibrationState = BenchmarkArtifactJudgeCalibration & Partial<J
     answerSetHash?: string;
     /** Question ids in the pinned calibration slice, in verdict order. */
     sliceQuestionIds?: readonly string[];
+    /** Full sanitized provider configuration hashes bound to this calibration. */
+    localJudgeConfigHash?: string;
+    frontierJudgeConfigHash?: string;
 };
 /**
  * Fixed slice size per benchmark. Issue #1877 raises the calibration target
@@ -4037,6 +4058,17 @@ interface CalibrationVerdictPair {
     localCategory: JudgeCategory;
     frontierCategory: JudgeCategory;
 }
+declare const JUDGE_CALIBRATION_PROTOCOL_VERSION = "judge-calibration-v3";
+interface JudgeCalibrationCheckpointProvenance {
+    dir: string;
+    sourceResultId: string;
+    sourceResultSha256: string;
+    orderedQuestionIdsHash: string;
+    localJudgePromptIdentity: string;
+    frontierJudgePromptIdentity: string;
+    localJudgeConfigHash: string;
+    frontierJudgeConfigHash: string;
+}
 interface RunJudgeCalibrationOptions {
     /** Benchmark id the calibration is scoped to (recorded in the result). */
     benchmarkId: string;
@@ -4052,6 +4084,8 @@ interface RunJudgeCalibrationOptions {
      * binning function so they are compared on the same scale.
      */
     binScore?: (score: number) => JudgeCategory;
+    /** Stable identity of `binScore`; required with a custom mapper and checkpointing. */
+    binningIdentity?: string;
     /** Override the slice size (default 200; mainly for tests). */
     sliceSize?: number;
     /** Override the warning threshold (default 0.7). */
@@ -4064,6 +4098,10 @@ interface RunJudgeCalibrationOptions {
     pinnedQuestionIds?: readonly string[];
     /** Fail before judge calls when the pinned answer payload changed. */
     expectedAnswerSetHash?: string;
+    /** Fail before judge calls when the full source task-id order changed. */
+    expectedOrderedQuestionIdsHash?: string;
+    /** Durable per-judge-side resume state. A mismatch or corrupt file fails closed. */
+    checkpoint?: JudgeCalibrationCheckpointProvenance;
 }
 interface JudgeCalibrationResult extends CohenKappaResult {
     benchmarkId: string;
@@ -4081,6 +4119,14 @@ interface JudgeCalibrationResult extends CohenKappaResult {
     answerSetHash: string;
     /** Per-question verdict pairs, in slice order. */
     verdicts: readonly CalibrationVerdictPair[];
+    /** Execution provenance distinguishes resumed outputs from fresh model calls. */
+    execution: {
+        localJudgeCalls: number;
+        frontierJudgeCalls: number;
+        resumedJudgeOutputs: number;
+        checkpointPath?: string;
+        checkpointContractHash?: string;
+    };
 }
 /**
  * Select the calibration slice from a universe of question ids.
@@ -4102,6 +4148,7 @@ declare function selectCalibrationSlice(questionIds: readonly string[], size?: n
  * subsequent local artifacts' `judgeCalibration.kappa`.
  */
 declare function runJudgeCalibration(options: RunJudgeCalibrationOptions): Promise<JudgeCalibrationResult>;
+declare function hashOrderedQuestionIds(questionIds: readonly string[]): string;
 /**
  * Persist a calibration result so subsequent local artifacts can carry the
  * kappa (issue #1573 done-when: "a kappa number that lands in subsequent
@@ -4118,6 +4165,8 @@ declare function runJudgeCalibration(options: RunJudgeCalibrationOptions): Promi
  */
 declare function writeJudgeCalibrationState(result: JudgeCalibrationResult, calibrationDir: string, identities?: JudgeCalibrationIdentities, provenance?: {
     sourceResultId: string;
+    localJudgeConfigHash?: string;
+    frontierJudgeConfigHash?: string;
 }): Promise<string>;
 /**
  * Load a previously persisted calibration result for a benchmark. Returns
@@ -5251,4 +5300,4 @@ declare function checkCodingGraphRegression(report: CodingGraphBenchReport, base
  */
 declare function buildBaselineFromReport(report: CodingGraphBenchReport, note: string): CodingGraphBaseline;
 
-export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildCodexCreditReceipt, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
+export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, JUDGE_CALIBRATION_PROTOCOL_VERSION, type JudgeCalibrationCheckpointProvenance, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildCodexCreditReceipt, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getProviderBackedJudgePromptIdentity, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashOrderedQuestionIds, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabJudgeProviderConfig, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };

package/dist/index.js

@@ -8303,6 +8303,8 @@ function readJudgeCalibrationFromBenchmarkOptions(value) {
   const answerSetHash = record.answerSetHash;
   const sourceResultId = record.sourceResultId;
   const sliceQuestionIds = readCalibrationQuestionIds(record.sliceQuestionIds);
+  const localJudgeConfigHash = record.localJudgeConfigHash;
+  const frontierJudgeConfigHash = record.frontierJudgeConfigHash;
   const hasCompleteProvenance = typeof answerSetHash === "string" && /^[0-9a-f]{64}$/.test(answerSetHash) && typeof sourceResultId === "string" && sourceResultId.length > 0 && sliceQuestionIds !== void 0 && sliceQuestionIds.length === sampleSize;
   return {
     kappa,
@@ -8311,7 +8313,8 @@ function readJudgeCalibrationFromBenchmarkOptions(value) {
     warning,
     ...confidenceInterval ? { confidenceInterval } : {},
     ...typeof bootstrapSamples === "number" && Number.isInteger(bootstrapSamples) && bootstrapSamples > 0 ? { bootstrapSamples } : {},
-    ...hasCompleteProvenance ? { answerSetHash, sourceResultId, sliceQuestionIds } : {}
+    ...hasCompleteProvenance ? { answerSetHash, sourceResultId, sliceQuestionIds } : {},
+    ...typeof localJudgeConfigHash === "string" && /^[0-9a-f]{64}$/.test(localJudgeConfigHash) && typeof frontierJudgeConfigHash === "string" && /^[0-9a-f]{64}$/.test(frontierJudgeConfigHash) ? { localJudgeConfigHash, frontierJudgeConfigHash } : {}
   };
 }
 function readCalibrationConfidenceInterval(value) {
@@ -8428,6 +8431,11 @@ function parseBenchmarkArtifact(raw) {
     if (calibration.sliceQuestionIds !== void 0 && !readCalibrationQuestionIds(calibration.sliceQuestionIds)) {
       throw new Error("BenchmarkArtifact judgeCalibration.sliceQuestionIds must contain 1 to 200 unique non-empty strings when provided.");
     }
+    for (const key of ["localJudgeConfigHash", "frontierJudgeConfigHash"]) {
+      if (calibration[key] !== void 0 && (typeof calibration[key] !== "string" || !/^[0-9a-f]{64}$/.test(calibration[key]))) {
+        throw new Error(`BenchmarkArtifact judgeCalibration.${key} must be a lowercase SHA-256 hex digest when provided.`);
+      }
+    }
     const hasAnyPinnedProvenance = calibration.answerSetHash !== void 0 || calibration.sourceResultId !== void 0 || calibration.sliceQuestionIds !== void 0;
     if (hasAnyPinnedProvenance && (typeof calibration.answerSetHash !== "string" || !/^[0-9a-f]{64}$/.test(calibration.answerSetHash) || typeof calibration.sourceResultId !== "string" || calibration.sourceResultId.length === 0 || !readCalibrationQuestionIds(calibration.sliceQuestionIds) || calibration.sliceQuestionIds.length !== calibration.sampleSize)) {
       throw new Error("BenchmarkArtifact judgeCalibration pinned provenance requires a sourceResultId, answerSetHash, and unique sliceQuestionIds matching sampleSize.");
@@ -12634,6 +12642,7 @@ function asStringArray(value) {
 }
 
 // src/responders.ts
+import { createHash as createHash7 } from "crypto";
 import { FallbackLlmClient } from "@remnic/core";
 
 // src/providers/openai-responses.ts
@@ -13082,6 +13091,32 @@ var DEFAULT_JUDGE_SYSTEM_PROMPT = [
   "Return only a numeric score from 0.00 to 1.00 inclusive.",
   "Use 1.00 for a fully correct answer, 0.00 for a fully incorrect answer, and fractional values for partial matches."
 ].join(" ");
+function buildDefaultJudgeUserPrompt(question, predicted, expected) {
+  return [
+    `QUESTION: ${question}`,
+    "",
+    `EXPECTED_ANSWER: ${expected}`,
+    "",
+    `PREDICTED_ANSWER: ${predicted}`,
+    "",
+    "Score the predicted answer against the expected answer."
+  ].join("\n");
+}
+function getProviderBackedJudgePromptIdentity(config) {
+  const contract = config.provider === "openai" ? {
+    kind: "structured",
+    rubric: GENERAL_ANSWER_JUDGE_RUBRIC,
+    rubricVersion: config.rubricVersion ?? OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION,
+    inputTemplate: ["QUESTION: {question}", "REFERENCE_ANSWER: {expected}", "PREDICTED_ANSWER: {predicted}"]
+  } : {
+    kind: "scalar",
+    systemPrompt: DEFAULT_JUDGE_SYSTEM_PROMPT,
+    userPrompt: buildDefaultJudgeUserPrompt("{question}", "{predicted}", "{expected}"),
+    temperature: 0,
+    maxTokens: 16
+  };
+  return `sha256:${createHash7("sha256").update(JSON.stringify(contract)).digest("hex")}`;
+}
 var AMA_BENCH_RECOMMENDED_JUDGE_SYSTEM_PROMPT = [
   "You are evaluating an AMA-Bench long-horizon memory question.",
   "Decide whether the predicted answer correctly answers the question using the reference answer as ground truth.",
@@ -13526,15 +13561,7 @@ function createJudgeFromProvider(provider) {
   }
   async function scoreWithMetrics(question, predicted, expected, control) {
     const completion = await provider.complete(
-      [
-        `QUESTION: ${question}`,
-        "",
-        `EXPECTED_ANSWER: ${expected}`,
-        "",
-        `PREDICTED_ANSWER: ${predicted}`,
-        "",
-        "Score the predicted answer against the expected answer."
-      ].join("\n"),
+      buildDefaultJudgeUserPrompt(question, predicted, expected),
       {
         systemPrompt: DEFAULT_JUDGE_SYSTEM_PROMPT,
         temperature: 0,
@@ -14753,7 +14780,7 @@ async function resolveBenchRuntimeProfile(options) {
     options.systemResponderContextBudgetChars,
     options.systemResponderPromptBudgetChars
   );
-  const judgeProvider = resolveProviderConfig(
+  const explicitJudgeProvider = resolveProviderConfig(
     "judge",
     options.judgeProvider,
     options.judgeModel,
@@ -14766,6 +14793,11 @@ async function resolveBenchRuntimeProfile(options) {
     void 0,
     void 0
   );
+  const judgeProvider = options.localLabManifestPath ? await resolveManifestBoundJudgeProvider(
+    options.localLabManifestPath,
+    options,
+    explicitJudgeProvider
+  ) : explicitJudgeProvider;
   const internalProvider = applyInternalProviderDefaults(
     resolveProviderConfig(
       "internal",
@@ -15185,6 +15217,56 @@ function applyLocalLabRuntimeOptions(config, options) {
     ...disableThinking ? { disableThinking: true } : {}
   };
 }
+async function resolveLocalLabJudgeProviderConfig(options) {
+  const manifest = await loadLocalLabManifest(options.localLabManifestPath);
+  const resolved = resolveLocalLabProfile(manifest);
+  return buildLocalLabJudgeProviderConfig(resolved, options);
+}
+function buildLocalLabJudgeProviderConfig(resolved, options) {
+  return applyLocalLabRuntimeOptions(
+    sanitizeProviderConfig(resolved.judge.providerConfig),
+    options
+  );
+}
+async function resolveManifestBoundJudgeProvider(localLabManifestPath, options, explicitJudgeProvider) {
+  const manifestJudge = await resolveLocalLabJudgeProviderConfig({
+    localLabManifestPath,
+    requestTimeout: options.requestTimeout,
+    max429WaitMs: options.max429WaitMs,
+    disableThinking: options.disableThinking
+  });
+  if (explicitJudgeProvider) {
+    assertManifestJudgeIdentity(explicitJudgeProvider, manifestJudge);
+  }
+  return manifestJudge;
+}
+function assertManifestJudgeIdentity(explicitJudge, manifestJudge) {
+  if (explicitJudge.provider !== manifestJudge.provider || explicitJudge.model !== manifestJudge.model) {
+    throw new Error(
+      `judge provider/model flags do not match the local-lab manifest judge: flags=${explicitJudge.provider}/${explicitJudge.model}, manifest=${manifestJudge.provider}/${manifestJudge.model}`
+    );
+  }
+  if (explicitJudge.baseUrl !== void 0 && normalizeProviderBaseUrl(explicitJudge.provider, explicitJudge.baseUrl) !== manifestJudge.baseUrl) {
+    throw new Error(
+      `judge baseUrl flag does not match the normalized local-lab manifest judge: flags=${normalizeProviderBaseUrl(explicitJudge.provider, explicitJudge.baseUrl)}, manifest=${String(manifestJudge.baseUrl)}`
+    );
+  }
+  if (explicitJudge.apiKey !== void 0) {
+    throw new Error(
+      "manifest-bound local judge does not accept --judge-api-key; keep local endpoint credentials out of the persisted manifest/hash contract"
+    );
+  }
+}
+function normalizeProviderBaseUrl(provider, rawBaseUrl) {
+  const trimmed = rawBaseUrl.trim().endsWith("/") ? rawBaseUrl.trim().slice(0, -1) : rawBaseUrl.trim();
+  if (provider === "ollama" && !trimmed.endsWith("/api")) {
+    return `${trimmed}/api`;
+  }
+  if (provider === "local-llm" && !trimmed.endsWith("/v1")) {
+    return `${trimmed}/v1`;
+  }
+  return trimmed;
+}
 function registerCodexCliFallbackRunnerIfNeeded(config) {
   if (!config || config.provider !== "codex-cli" || codexCliFallbackRegistered) {
     return;
@@ -15337,10 +15419,7 @@ async function resolveLocalLabRuntimeProfile(options) {
     sanitizeProviderConfig(resolved.responder.providerConfig),
     options
   );
-  const judgeProvider = applyLocalLabRuntimeOptions(
-    sanitizeProviderConfig(resolved.judge.providerConfig),
-    options
-  );
+  const judgeProvider = buildLocalLabJudgeProviderConfig(resolved, options);
   const judgeFactoryConfig = judgeProvider ? asProviderFactoryConfig(judgeProvider) : void 0;
   const judge = judgeFactoryConfig ? createProviderBackedJudge(judgeFactoryConfig) : void 0;
   const structuredJudge = judgeFactoryConfig ? createProviderBackedStructuredJudge(judgeFactoryConfig) : void 0;
@@ -15377,11 +15456,11 @@ async function resolveLocalLabRuntimeProfile(options) {
 // src/benchmark.ts
 import fs2 from "fs";
 import path35 from "path";
-import { createHash as createHash12 } from "crypto";
+import { createHash as createHash13 } from "crypto";
 import { expandTildePath as expandTildePath3 } from "@remnic/core";
 
 // src/judges/judge-cache.ts
-import { createHash as createHash7, randomBytes as randomBytes2 } from "crypto";
+import { createHash as createHash8, randomBytes as randomBytes2 } from "crypto";
 import {
   mkdir as mkdir9,
   readFile as readFile11,
@@ -15424,8 +15503,8 @@ var JudgeCache = class {
   }
   /** Compute the sha256-hex key for a set of parts. Pure, sync, side-effect-free. */
   computeKey(parts) {
-    const fieldDigest = (value) => createHash7("sha256").update(value).digest();
-    return createHash7("sha256").update(fieldDigest(parts.benchmarkId)).update(fieldDigest(parts.datasetVersion)).update(fieldDigest(parts.questionId)).update(fieldDigest(parts.answerText)).update(fieldDigest(parts.judgePromptHash)).update(fieldDigest(parts.judgeModelId)).update(fieldDigest(parts.judgeParamsHash)).digest("hex");
+    const fieldDigest = (value) => createHash8("sha256").update(value).digest();
+    return createHash8("sha256").update(fieldDigest(parts.benchmarkId)).update(fieldDigest(parts.datasetVersion)).update(fieldDigest(parts.questionId)).update(fieldDigest(parts.answerText)).update(fieldDigest(parts.judgePromptHash)).update(fieldDigest(parts.judgeModelId)).update(fieldDigest(parts.judgeParamsHash)).digest("hex");
   }
   /**
    * Read a previously-stored verdict. Returns `undefined` on miss, corrupted
@@ -15620,7 +15699,7 @@ function runJudgeWithCache(options) {
           // Binary prompts are content-sensitive: two distinct prompts of
           // the same character length would collide on the previous
           // `binary:N` key, so key on a sha256 prefix of the prompt body.
-          questionId: `binary:${createHash7("sha256").update(prompt).digest("hex").slice(0, 16)}`,
+          questionId: `binary:${createHash8("sha256").update(prompt).digest("hex").slice(0, 16)}`,
           answerText: prompt,
           judgePromptHash: keyExtras.judgePromptHash ?? "unknown-prompt",
           judgeModelId: keyExtras.judgeModelId ?? "unknown-judge",
@@ -23111,7 +23190,7 @@ var StructuredLiteralParser = class {
 };
 
 // src/benchmarks/published/personamem/runner.ts
-import { createHash as createHash8, randomUUID as randomUUID7 } from "crypto";
+import { createHash as createHash9, randomUUID as randomUUID7 } from "crypto";
 import { readFile as readFile16, realpath as realpath4 } from "fs/promises";
 import path19 from "path";
 
@@ -23716,7 +23795,7 @@ function buildMcqPrompt(sample, seed) {
 function deterministicShuffle(values, seedMaterial) {
   return values.map((value, index) => ({
     value,
-    key: createHash8("sha256").update(`${seedMaterial}:${index}:${value}`).digest("hex"),
+    key: createHash9("sha256").update(`${seedMaterial}:${index}:${value}`).digest("hex"),
     index
   })).sort((left, right) => {
     const byKey = left.key.localeCompare(right.key);
@@ -32175,7 +32254,7 @@ function pairedDeltaConfidenceInterval(candidateValues, baselineValues, options
 }
 
 // src/judges/sealed-rubric.ts
-import { createHash as createHash9 } from "crypto";
+import { createHash as createHash10 } from "crypto";
 import { appendFileSync, mkdirSync } from "fs";
 import path31 from "path";
 
@@ -32284,7 +32363,7 @@ function loadSealedRubric(id = DEFAULT_ASSISTANT_RUBRIC_ID, options = {}) {
   if (typeof prompt !== "string" || prompt.length === 0) {
     throw new Error(`sealed rubric not found in registry: ${id}`);
   }
-  const sha2563 = createHash9("sha256").update(prompt, "utf8").digest("hex");
+  const sha2563 = createHash10("sha256").update(prompt, "utf8").digest("hex");
   const version = parseVersionFromId(id);
   return { id, version, prompt, sha256: sha2563 };
 }
@@ -34487,7 +34566,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
 import { randomUUID as randomUUID31 } from "crypto";
 
 // src/benchmarks/remnic/memcorrect/generator.ts
-import { createHash as createHash10 } from "crypto";
+import { createHash as createHash11 } from "crypto";
 
 // src/benchmarks/remnic/memcorrect/token-pools.ts
 var PERSONAS = [
@@ -34811,7 +34890,7 @@ function corpusHash(corpus) {
     uptakeLatencyCap: corpus.options.uptakeLatencyCap,
     scenarios: corpus.scenarios
   });
-  return createHash10("sha256").update(canonical).digest("hex");
+  return createHash11("sha256").update(canonical).digest("hex");
 }
 
 // src/benchmarks/remnic/memcorrect/schema.ts
@@ -35766,7 +35845,7 @@ import { mkdir as mkdir17, writeFile as writeFile16 } from "fs/promises";
 import path34 from "path";
 
 // src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
-import { createHash as createHash11 } from "crypto";
+import { createHash as createHash12 } from "crypto";
 var SCOPE_ACME = "project:acme";
 var SCOPE_BETA = "project:beta";
 var SCOPE_ALICE = "user:alice";
@@ -36249,7 +36328,7 @@ var BOUNDED_MEMORY_SMOKE_FIXTURE = [
 function fixtureHash(tasks) {
   const source = tasks ?? BOUNDED_MEMORY_FIXTURE;
   const payload = JSON.stringify(source);
-  return createHash11("sha256").update(payload, "utf8").digest("hex");
+  return createHash12("sha256").update(payload, "utf8").digest("hex");
 }
 
 // src/benchmarks/remnic/bounded-memory-contracts/agent.ts
@@ -37509,13 +37588,13 @@ function wrapJudgeWithCache(args) {
       // differentiator is part of the prompt hash. Bumping
       // JUDGE_CACHE_PROTOCOL_VERSION invalidates verdicts when judge
       // prompt/parse semantics change (PR #1591, High).
-      judgePromptHash: createHash12("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("
").update(args.amaBenchJudgeProtocol).update("
").update(args.role).digest("hex"),
+      judgePromptHash: createHash13("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("
").update(args.amaBenchJudgeProtocol).update("
").update(args.role).digest("hex"),
       judgeModelId: args.provider?.model !== void 0 && args.provider.model.length > 0 ? `${args.provider.model}${crossJudgeIdSuffix}` : `unknown-${args.role}-judge`,
       // Full judge configuration, deterministically serialized (sorted
       // keys) so provider/base-url/retry changes produce fresh cache
       // keys. `role` is included so primary and cross judges never
       // share a paramsHash.
-      judgeParamsHash: createHash12("sha256").update(
+      judgeParamsHash: createHash13("sha256").update(
         stableStringify2({
           role: args.role,
           provider: args.provider
@@ -38221,7 +38300,7 @@ function formatSignedScore(value) {
 }
 
 // src/stats/locomo-recall-delta.ts
-import { createHash as createHash13 } from "crypto";
+import { createHash as createHash14 } from "crypto";
 import { basename } from "path";
 var LOCOMO_FULL_TASK_COUNT = 1986;
 var LOCOMO_RECALL_EXCERPT_CHARS = 240;
@@ -38702,7 +38781,7 @@ function normalizeText3(value) {
   return value.replaceAll("\r\n", "\n").replaceAll("\r", "\n");
 }
 function sha2562(value) {
-  return createHash13("sha256").update(value).digest("hex");
+  return createHash14("sha256").update(value).digest("hex");
 }
 function stableJson(value) {
   return JSON.stringify(value);
@@ -40140,8 +40219,8 @@ var chatFixture = {
 };
 
 // src/judges/calibration-slice.ts
-import { createHash as createHash14, randomBytes as randomBytes3 } from "crypto";
-import { mkdir as mkdir18, readFile as readFile22, rename as rename4, unlink as unlink4, writeFile as writeFile17 } from "fs/promises";
+import { createHash as createHash15, randomBytes as randomBytes3 } from "crypto";
+import { chmod as chmod2, lstat as lstat4, mkdir as mkdir18, open as open2, readFile as readFile22, rename as rename4, unlink as unlink4, writeFile as writeFile17 } from "fs/promises";
 import path37 from "path";
 
 // src/judges/cohen-kappa.ts
@@ -40272,6 +40351,8 @@ function binarizeJudgeScore(score, threshold = DEFAULT_JUDGE_BINARIZATION_THRESH
 var CALIBRATION_SLICE_SIZE = 200;
 var MIN_CALIBRATION_SOURCE_TASKS = 10;
 var JUDGE_CALIBRATION_KAPPA_THRESHOLD = 0.7;
+var JUDGE_CALIBRATION_PROTOCOL_VERSION = "judge-calibration-v3";
+var DEFAULT_JUDGE_BINNING_IDENTITY = "default-binary-score-v1:incorrect<0.5,correct>=0.5,nonfinite=incorrect";
 function selectCalibrationSlice(questionIds, size = CALIBRATION_SLICE_SIZE) {
   if (!Number.isInteger(size) || size <= 0) {
     throw new Error(`selectCalibrationSlice: size must be a positive integer; got ${String(size)}.`);
@@ -40287,7 +40368,7 @@ function selectCalibrationSlice(questionIds, size = CALIBRATION_SLICE_SIZE) {
       unique.push(id);
     }
   }
-  return unique.map((id) => ({ id, digest: createHash14("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
+  return unique.map((id) => ({ id, digest: createHash15("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
 }
 async function runJudgeCalibration(options) {
   const binScore = options.binScore ?? ((score) => binarizeJudgeScore(score));
@@ -40304,52 +40385,107 @@ async function runJudgeCalibration(options) {
   }
   const sliceAnswers = sliceIds.map((id) => answerById.get(id)).filter((answer) => answer !== void 0);
   const answerSetHash = hashCalibrationAnswerSet(sliceAnswers);
+  const orderedQuestionIdsHash = hashOrderedQuestionIds(options.answers.map((answer) => answer.questionId));
+  if (options.expectedOrderedQuestionIdsHash !== void 0 && orderedQuestionIdsHash !== options.expectedOrderedQuestionIdsHash) {
+    throw new Error(
+      `runJudgeCalibration: ordered question-id list changed (expected sha256:${options.expectedOrderedQuestionIdsHash}, got sha256:${orderedQuestionIdsHash}).`
+    );
+  }
   if (options.expectedAnswerSetHash !== void 0 && answerSetHash !== options.expectedAnswerSetHash) {
     throw new Error(
       `runJudgeCalibration: pinned answer set changed (expected sha256:${options.expectedAnswerSetHash}, got sha256:${answerSetHash}). Restore the original stored result or intentionally reset calibration state.`
     );
   }
+  if (options.checkpoint && options.binScore && !options.binningIdentity) {
+    throw new Error("runJudgeCalibration: checkpointed custom binScore requires an explicit binningIdentity.");
+  }
+  const binningIdentity = options.binningIdentity ?? DEFAULT_JUDGE_BINNING_IDENTITY;
+  const checkpoint = options.checkpoint ? await loadOrInitializeCheckpoint(
+    options.benchmarkId,
+    { ...options.checkpoint, binningIdentity },
+    sliceIds,
+    answerSetHash,
+    orderedQuestionIdsHash
+  ) : void 0;
   const localLabels = [];
   const frontierLabels = [];
   const verdicts = [];
-  for (const answer of sliceAnswers) {
-    const localScore = await options.localJudge.score(
-      answer.question,
-      answer.predicted,
-      answer.expected
-    );
-    const frontierScore = await options.frontierJudge.score(
-      answer.question,
-      answer.predicted,
-      answer.expected
-    );
-    const localCategory = binScore(localScore);
-    const frontierCategory = binScore(frontierScore);
-    localLabels.push(localCategory);
-    frontierLabels.push(frontierCategory);
-    verdicts.push({
-      questionId: answer.questionId,
-      localCategory,
-      frontierCategory
+  let localJudgeCalls = 0;
+  let frontierJudgeCalls = 0;
+  let resumedJudgeOutputs = 0;
+  try {
+    for (const answer of sliceAnswers) {
+      const saved = checkpoint?.state.completed[answer.questionId];
+      let localCategory = saved?.localCategory;
+      if (localCategory !== void 0) {
+        resumedJudgeOutputs += 1;
+      } else {
+        const localScore = await options.localJudge.score(answer.question, answer.predicted, answer.expected);
+        localJudgeCalls += 1;
+        localCategory = binScore(localScore);
+        if (checkpoint) {
+          checkpoint.state.completed[answer.questionId] = { ...saved, localCategory };
+          await writeCalibrationCheckpoint(checkpoint.path, checkpoint.state);
+        }
+      }
+      let frontierCategory = checkpoint?.state.completed[answer.questionId]?.frontierCategory;
+      if (frontierCategory !== void 0) {
+        resumedJudgeOutputs += 1;
+      } else {
+        const frontierScore = await options.frontierJudge.score(answer.question, answer.predicted, answer.expected);
+        frontierJudgeCalls += 1;
+        frontierCategory = binScore(frontierScore);
+        if (checkpoint) {
+          checkpoint.state.completed[answer.questionId] = {
+            ...checkpoint.state.completed[answer.questionId],
+            frontierCategory
+          };
+          await writeCalibrationCheckpoint(checkpoint.path, checkpoint.state);
+        }
+      }
+      localLabels.push(localCategory);
+      frontierLabels.push(frontierCategory);
+      verdicts.push({
+        questionId: answer.questionId,
+        localCategory,
+        frontierCategory
+      });
+    }
+    const kappaResult = computeCohensKappa(localLabels, frontierLabels);
+    const bootstrap = bootstrapCohensKappaConfidenceInterval(localLabels, frontierLabels, {
+      iterations: options.bootstrapSamples ?? DEFAULT_KAPPA_BOOTSTRAP_SAMPLES,
+      level: options.confidenceLevel
     });
+    const warning = kappaResult.kappa < threshold;
+    return {
+      ...kappaResult,
+      benchmarkId: options.benchmarkId,
+      sliceQuestionIds: sliceIds,
+      threshold,
+      warning,
+      confidenceInterval: bootstrap.confidenceInterval,
+      bootstrapSamples: bootstrap.bootstrapSamples,
+      answerSetHash,
+      verdicts,
+      execution: {
+        localJudgeCalls,
+        frontierJudgeCalls,
+        resumedJudgeOutputs,
+        ...checkpoint ? {
+          checkpointPath: checkpoint.path,
+          checkpointContractHash: checkpoint.state.contractHash
+        } : {}
+      }
+    };
+  } finally {
+    await checkpoint?.release();
   }
-  const kappaResult = computeCohensKappa(localLabels, frontierLabels);
-  const bootstrap = bootstrapCohensKappaConfidenceInterval(localLabels, frontierLabels, {
-    iterations: options.bootstrapSamples ?? DEFAULT_KAPPA_BOOTSTRAP_SAMPLES,
-    level: options.confidenceLevel
-  });
-  const warning = kappaResult.kappa < threshold;
-  return {
-    ...kappaResult,
-    benchmarkId: options.benchmarkId,
-    sliceQuestionIds: sliceIds,
-    threshold,
-    warning,
-    confidenceInterval: bootstrap.confidenceInterval,
-    bootstrapSamples: bootstrap.bootstrapSamples,
-    answerSetHash,
-    verdicts
-  };
+}
+function hashOrderedQuestionIds(questionIds) {
+  if (questionIds.some((id) => typeof id !== "string" || id.length === 0)) {
+    throw new Error("hashOrderedQuestionIds: question ids must be non-empty strings.");
+  }
+  return createHash15("sha256").update(JSON.stringify(questionIds), "utf8").digest("hex");
 }
 function validatePinnedQuestionIds(ids, availableIds) {
   if (ids.length === 0 || ids.length > CALIBRATION_SLICE_SIZE || ids.some((id) => typeof id !== "string" || id.length === 0) || new Set(ids).size !== ids.length) {
@@ -40362,15 +40498,141 @@ function validatePinnedQuestionIds(ids, availableIds) {
   return [...ids];
 }
 function hashCalibrationAnswerSet(answers) {
-  return createHash14("sha256").update(JSON.stringify(answers.map((answer) => [
+  return createHash15("sha256").update(JSON.stringify(answers.map((answer) => [
     answer.questionId,
     answer.question,
     answer.predicted,
     answer.expected
   ]))).digest("hex");
 }
+async function loadOrInitializeCheckpoint(benchmarkId, provenance, sliceQuestionIds, answerSetHash, orderedQuestionIdsHash) {
+  for (const [name, digest] of Object.entries({
+    sourceResultSha256: provenance.sourceResultSha256,
+    orderedQuestionIdsHash,
+    localJudgeConfigHash: provenance.localJudgeConfigHash,
+    frontierJudgeConfigHash: provenance.frontierJudgeConfigHash
+  })) {
+    if (!/^[0-9a-f]{64}$/.test(digest)) throw new Error(`runJudgeCalibration: ${name} must be a lowercase SHA-256 digest.`);
+  }
+  if (!provenance.sourceResultId || !provenance.localJudgePromptIdentity || !provenance.frontierJudgePromptIdentity || !provenance.binningIdentity) {
+    throw new Error("runJudgeCalibration: checkpoint sourceResultId, prompt identities, and binningIdentity are required.");
+  }
+  if (provenance.orderedQuestionIdsHash !== orderedQuestionIdsHash) {
+    throw new Error("runJudgeCalibration: checkpoint ordered-question-id hash does not match the validated source.");
+  }
+  await ensurePrivateDirectory(provenance.dir);
+  const checkpointPath = path37.join(provenance.dir, `${sanitizeCalibrationSegment(benchmarkId)}.checkpoint.json`);
+  const lockPath = `${checkpointPath}.lock`;
+  let lockHandle;
+  try {
+    lockHandle = await open2(lockPath, "wx", 384);
+    await lockHandle.writeFile(`${JSON.stringify({ pid: process.pid, acquiredAt: (/* @__PURE__ */ new Date()).toISOString() })}
+`, "utf8");
+    await chmod2(lockPath, 384);
+  } catch (error) {
+    if (lockHandle) {
+      await lockHandle.close().catch(() => void 0);
+      await unlink4(lockPath).catch(() => void 0);
+    }
+    if (error.code === "EEXIST") {
+      throw new Error(
+        `runJudgeCalibration: checkpoint is locked at ${lockPath}; refusing concurrent or stale-lock recovery to avoid duplicate paid judge calls.`
+      );
+    }
+    throw error;
+  }
+  let released = false;
+  const release = async () => {
+    if (released) return;
+    if (!lockHandle) throw new Error(`runJudgeCalibration: checkpoint lock handle was not acquired for ${lockPath}.`);
+    released = true;
+    await lockHandle.close();
+    await unlink4(lockPath);
+  };
+  try {
+    const contract = {
+      protocolVersion: JUDGE_CALIBRATION_PROTOCOL_VERSION,
+      benchmarkId,
+      sourceResultId: provenance.sourceResultId,
+      sourceResultSha256: provenance.sourceResultSha256,
+      orderedQuestionIdsHash: provenance.orderedQuestionIdsHash,
+      answerSetHash,
+      sliceQuestionIds,
+      localJudgePromptIdentity: provenance.localJudgePromptIdentity,
+      frontierJudgePromptIdentity: provenance.frontierJudgePromptIdentity,
+      localJudgeConfigHash: provenance.localJudgeConfigHash,
+      frontierJudgeConfigHash: provenance.frontierJudgeConfigHash,
+      binningIdentity: provenance.binningIdentity
+    };
+    const contractHash = createHash15("sha256").update(stableJson2(contract)).digest("hex");
+    let raw;
+    try {
+      const info = await lstat4(checkpointPath);
+      if (!info.isFile() || info.isSymbolicLink()) throw new Error("checkpoint path is not a regular file");
+      raw = await readFile22(checkpointPath, "utf8");
+    } catch (error) {
+      if (error.code !== "ENOENT") throw error;
+      const state = { schemaVersion: 2, contractHash, contract, completed: {} };
+      await writeCalibrationCheckpoint(checkpointPath, state);
+      return { path: checkpointPath, state, release };
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      throw new Error(`runJudgeCalibration: corrupt checkpoint at ${checkpointPath}; refusing to call judges.`);
+    }
+    if (!isValidCheckpoint(parsed, new Set(sliceQuestionIds))) {
+      throw new Error(`runJudgeCalibration: corrupt checkpoint at ${checkpointPath}; refusing to call judges.`);
+    }
+    if (parsed.contractHash !== contractHash || stableJson2(parsed.contract) !== stableJson2(contract)) {
+      throw new Error(`runJudgeCalibration: checkpoint contract mismatch at ${checkpointPath}; refusing to call judges.`);
+    }
+    await chmod2(checkpointPath, 384);
+    return { path: checkpointPath, state: parsed, release };
+  } catch (error) {
+    await release().catch(() => void 0);
+    throw error;
+  }
+}
+function isValidCheckpoint(value, sliceIds) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const record = value;
+  if (record.schemaVersion !== 2 || typeof record.contractHash !== "string" || !/^[0-9a-f]{64}$/.test(record.contractHash) || !record.contract || typeof record.contract !== "object" || Array.isArray(record.contract) || !record.completed || typeof record.completed !== "object" || Array.isArray(record.completed)) return false;
+  return Object.entries(record.completed).every(([id, output]) => {
+    if (!sliceIds.has(id) || !output || typeof output !== "object" || Array.isArray(output)) return false;
+    const fields = output;
+    return Object.keys(fields).every((key) => key === "localCategory" || key === "frontierCategory") && [fields.localCategory, fields.frontierCategory].every((category) => category === void 0 || typeof category === "string" && category.length > 0);
+  });
+}
+async function ensurePrivateDirectory(dir) {
+  await mkdir18(dir, { recursive: true, mode: 448 });
+  const info = await lstat4(dir);
+  if (!info.isDirectory() || info.isSymbolicLink()) throw new Error(`runJudgeCalibration: checkpoint directory must be a real directory: ${dir}`);
+  await chmod2(dir, 448);
+}
+async function writeCalibrationCheckpoint(filePath, state) {
+  const tempPath = `${filePath}.${randomBytes3(6).toString("hex")}.tmp`;
+  await writeFile17(tempPath, `${JSON.stringify(state, null, 2)}
+`, { encoding: "utf8", mode: 384 });
+  try {
+    await rename4(tempPath, filePath);
+    await chmod2(filePath, 384);
+  } catch (error) {
+    await unlink4(tempPath).catch(() => void 0);
+    throw error;
+  }
+}
+function stableJson2(value) {
+  if (Array.isArray(value)) return `[${value.map(stableJson2).join(",")}]`;
+  if (value !== null && typeof value === "object") {
+    const record = value;
+    return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableJson2(record[key])}`).join(",")}}`;
+  }
+  return JSON.stringify(value) ?? "null";
+}
 async function writeJudgeCalibrationState(result, calibrationDir, identities, provenance) {
-  await mkdir18(calibrationDir, { recursive: true });
+  await ensurePrivateDirectory(calibrationDir);
   const state = {
     kappa: result.kappa,
     sampleSize: result.sampleSize,
@@ -40386,9 +40648,10 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities, pr
   const filePath = path37.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
   const tempPath = `${filePath}.${randomBytes3(6).toString("hex")}.tmp`;
   await writeFile17(tempPath, `${JSON.stringify(state, null, 2)}
-`, "utf8");
+`, { encoding: "utf8", mode: 384 });
   try {
     await rename4(tempPath, filePath);
+    await chmod2(filePath, 384);
   } catch (error) {
     await unlink4(tempPath).catch(() => void 0);
     throw error;
@@ -40434,6 +40697,10 @@ async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
     loaded.answerSetHash = answerSetHash;
     loaded.sliceQuestionIds = record.sliceQuestionIds;
   }
+  if (typeof record.localJudgeConfigHash === "string" && /^[0-9a-f]{64}$/.test(record.localJudgeConfigHash) && typeof record.frontierJudgeConfigHash === "string" && /^[0-9a-f]{64}$/.test(record.frontierJudgeConfigHash)) {
+    loaded.localJudgeConfigHash = record.localJudgeConfigHash;
+    loaded.frontierJudgeConfigHash = record.frontierJudgeConfigHash;
+  }
   const identityKeys = [
     "localJudgeProvider",
     "localJudgeModel",
@@ -41818,7 +42085,7 @@ function createMitigatedTarget(config) {
 }
 
 // src/coding-graph/generator.ts
-import { createHash as createHash15 } from "crypto";
+import { createHash as createHash16 } from "crypto";
 function createSeededRng3(seed) {
   let state = seed >>> 0;
   return function rng() {
@@ -41847,7 +42114,7 @@ var EDGE_TYPE_WEIGHTS = [
 var PROVENANCE_VALUES = ["heuristic", "heuristic", "heuristic", "trace"];
 var AVG_BYTES_PER_LINE = 40;
 function hashContent(input) {
-  return createHash15("sha256").update(input).digest("hex").slice(0, 16);
+  return createHash16("sha256").update(input).digest("hex").slice(0, 16);
 }
 function generateSyntheticRepo(config) {
   const rng = createSeededRng3(config.seed);
@@ -42413,6 +42680,7 @@ export {
   INTEGRITY_HASH_ALGORITHM,
   INTEGRITY_META_FIELDS,
   JUDGE_CALIBRATION_KAPPA_THRESHOLD,
+  JUDGE_CALIBRATION_PROTOCOL_VERSION,
   LOCAL_LAB_PROVIDER_KINDS,
   LOCOMO_DATASET_FILENAMES,
   LOCOMO_FULL_TASK_COUNT,
@@ -42547,10 +42815,12 @@ export {
   getBenchmark,
   getBenchmarkLowerIsBetter,
   getMemoryEvalDimension,
+  getProviderBackedJudgePromptIdentity,
   getRemnicVersion,
   hashBenchmarkArtifact,
   hashBytes,
   hashCanonicalJson,
+  hashOrderedQuestionIds,
   hashString,
   integrityMetaIsComplete,
   interpretEffectSize,
@@ -42617,6 +42887,7 @@ export {
   resolveBenchmarkProgressLogging,
   resolveBenchmarkResultReference,
   resolveBenchmarkRunCount,
+  resolveLocalLabJudgeProviderConfig,
   resolveLocalLabProfile,
   resolveLocalLabRole,
   resolveStructuredJudge,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/bench",
-  "version": "9.6.23",
+  "version": "9.6.25",
   "description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
   "type": "module",
   "main": "./dist/index.js",
@@ -39,8 +39,8 @@
     "hyparquet": "^1.25.7",
     "yaml": "^2.4.2",
     "zod": "^3.24.0",
-    "@remnic/coding-graph": "^9.6.23",
-    "@remnic/core": "^9.6.23"
+    "@remnic/coding-graph": "^9.6.25",
+    "@remnic/core": "^9.6.25"
   },
   "devDependencies": {
     "tsup": "^8.5.1",