@remnic/cli 9.6.23 → 9.6.25

package/dist/index.js

@@ -782,6 +782,14 @@ var BENCH_VALUE_FLAGS = Object.freeze([
   "--provider",
   "--base-url",
   "--request-timeout",
+  "--local-judge-request-timeout",
+  "--frontier-judge-request-timeout",
+  "--calibration-dir",
+  "--calibration-local-config-sha256",
+  "--calibration-frontier-config-sha256",
+  "--source-result-id",
+  "--expected-answer-set-sha256",
+  "--expected-question-id-list-sha256",
   "--drain-timeout",
   "--max-429-wait",
   "--ama-bench-judge-protocol",
@@ -862,6 +870,9 @@ var RUN_VALUE_FLAGS = Object.freeze([
   "--provider",
   "--base-url",
   "--request-timeout",
+  "--calibration-dir",
+  "--calibration-local-config-sha256",
+  "--calibration-frontier-config-sha256",
   "--drain-timeout",
   "--max-429-wait",
   "--ama-bench-judge-protocol",
@@ -937,9 +948,16 @@ var BENCH_ACTION_FLAGS = {
       "--judge-provider",
       "--judge-model",
       "--judge-base-url",
-      "--judge-api-key"
+      "--judge-api-key",
+      "--local-judge-request-timeout",
+      "--frontier-judge-request-timeout",
+      "--max-429-wait",
+      "--calibration-dir",
+      "--source-result-id",
+      "--expected-answer-set-sha256",
+      "--expected-question-id-list-sha256"
     ],
-    boolean: ["--json", "--help", "-h"]
+    boolean: ["--disable-thinking", "--json", "--help", "-h"]
   },
   published: {
     value: PUBLISHED_VALUE_FLAGS,
@@ -1121,6 +1139,14 @@ function parseBenchArgs(argv) {
   const output = readBenchOptionValue(args, "--output");
   const targetRaw = readBenchOptionValue(args, "--target");
   const requestTimeoutRaw = readBenchOptionValue(args, "--request-timeout");
+  const localJudgeRequestTimeoutRaw = readBenchOptionValue(args, "--local-judge-request-timeout");
+  const frontierJudgeRequestTimeoutRaw = readBenchOptionValue(args, "--frontier-judge-request-timeout");
+  const calibrationDirRaw = readBenchOptionValue(args, "--calibration-dir");
+  const calibrationLocalConfigSha256 = readBenchOptionValue(args, "--calibration-local-config-sha256");
+  const calibrationFrontierConfigSha256 = readBenchOptionValue(args, "--calibration-frontier-config-sha256");
+  const sourceResultId = readBenchOptionValue(args, "--source-result-id");
+  const expectedAnswerSetSha256 = readBenchOptionValue(args, "--expected-answer-set-sha256");
+  const expectedQuestionIdListSha256 = readBenchOptionValue(args, "--expected-question-id-list-sha256");
   const drainTimeoutRaw = readBenchOptionValue(args, "--drain-timeout");
   const judgeCacheDirRaw = readBenchOptionValue(args, "--judge-cache-dir");
   const localLabManifestRaw = readBenchOptionValue(args, "--local-lab-manifest");
@@ -1296,6 +1322,26 @@ function parseBenchArgs(argv) {
       );
     }
   }
+  const parseJudgeTimeout = (raw, flag) => {
+    if (raw === void 0) return void 0;
+    const value = Number(raw);
+    if (!Number.isInteger(value) || value <= 0 || value > 36e5) {
+      throw new Error(`ERROR: ${flag} must be a positive integer no greater than 3,600,000 ms.`);
+    }
+    return value;
+  };
+  const localJudgeRequestTimeout = parseJudgeTimeout(localJudgeRequestTimeoutRaw, "--local-judge-request-timeout");
+  const frontierJudgeRequestTimeout = parseJudgeTimeout(frontierJudgeRequestTimeoutRaw, "--frontier-judge-request-timeout");
+  for (const [flag, digest] of [
+    ["--expected-answer-set-sha256", expectedAnswerSetSha256],
+    ["--expected-question-id-list-sha256", expectedQuestionIdListSha256],
+    ["--calibration-local-config-sha256", calibrationLocalConfigSha256],
+    ["--calibration-frontier-config-sha256", calibrationFrontierConfigSha256]
+  ]) {
+    if (digest !== void 0 && !/^[0-9a-f]{64}$/.test(digest)) {
+      throw new Error(`ERROR: ${flag} must be a lowercase SHA-256 hex digest.`);
+    }
+  }
   let drainTimeout;
   if (drainTimeoutRaw !== void 0) {
     drainTimeout = Number(drainTimeoutRaw);
@@ -1604,6 +1650,14 @@ function parseBenchArgs(argv) {
     publishedOut: publishedOutRaw ? path4.resolve(expandTilde(publishedOutRaw)) : void 0,
     publishedDryRun: args.includes("--dry-run"),
     requestTimeout,
+    localJudgeRequestTimeout,
+    frontierJudgeRequestTimeout,
+    calibrationDir: calibrationDirRaw ? path4.resolve(expandTilde(calibrationDirRaw)) : void 0,
+    calibrationLocalConfigSha256,
+    calibrationFrontierConfigSha256,
+    sourceResultId,
+    expectedAnswerSetSha256,
+    expectedQuestionIdListSha256,
     drainTimeout,
     // Issue #1573 PR1: surface judge-cache flags into the runner options.
     noJudgeCache: args.includes("--no-judge-cache"),
@@ -3449,7 +3503,9 @@ Commands:
                            Generate the Remnic.ai benchmark feed from stored runs
   ui                       Launch the local benchmark overview UI
   providers discover       Auto-detect available local provider backends
-  judge-calibrate --benchmark <id> --local-lab-manifest <path> --judge-provider <p> --judge-model <m>
+  judge-calibrate --benchmark <id> --source-result-id <id> --expected-answer-set-sha256 <sha256>
+                           --expected-question-id-list-sha256 <sha256>
+                           --local-lab-manifest <path> --judge-provider <p> --judge-model <m>
                            Cross-tier judge calibration (issue #1573): runs the
                            local + frontier judges over a benchmark's cached
                            answers, reports Cohen's kappa, and persists it so
@@ -3510,6 +3566,7 @@ Options:
                            Codex CLI reasoning effort for Remnic's internal LLM
   --ama-bench-judge-protocol <default|recommended>
                            For ama-bench, use the recommended binary LLM-judge protocol
+                           (default-protocol calibration is not attached to recommended runs)
   --ama-bench-cross-judge-model <model>
                            For ama-bench, add a second recommended-protocol judge for agreement checks
   --ama-bench-cross-judge-provider <provider>
@@ -3522,6 +3579,22 @@ Options:
   --results-dir <path>     Override the stored benchmark results directory
   --baselines-dir <path>   Override the named baseline directory
   --request-timeout <ms>   Provider request timeout in milliseconds (codex-cli default: 180000)
+  --local-judge-request-timeout <ms>
+                           Calibration-only timeout for each local judge call
+  --frontier-judge-request-timeout <ms>
+                           Calibration-only timeout for each frontier judge call
+  --max-429-wait <ms>      Maximum cumulative 429 retry wait for provider calls
+  --disable-thinking       Disable thinking for supported provider-backed models
+  --source-result-id <id>  Exact full stored result used by judge-calibrate
+  --expected-answer-set-sha256 <sha256>
+                           Expected deterministic calibration-slice payload hash
+  --expected-question-id-list-sha256 <sha256>
+                           Expected ordered source task-ID list hash
+  --calibration-dir <path> Private final-state and resumable-checkpoint directory
+  --calibration-local-config-sha256 <sha256>
+                           Required on later runs that attach calibration state
+  --calibration-frontier-config-sha256 <sha256>
+                           Required on later runs that attach calibration state
   --drain-timeout <ms>     Memory drain timeout in milliseconds
                            (defaults to --request-timeout; implicit codex-cli default: 600000)
   --local-lab-manifest <path>
@@ -3596,7 +3669,7 @@ function buildBenchRuntimeProfileRequest(parsed, runtimeProfile) {
     max429WaitMs: parsed.max429WaitMs,
     disableThinking: parsed.disableThinking,
     lcmObserveConcurrency: parsed.publishedIngestConcurrency,
-    localLabManifestPath: runtimeProfile === "local-lab" ? parsed.localLabManifestPath : void 0
+    localLabManifestPath: parsed.localLabManifestPath
   };
 }
 var BENCH_STDOUT_REDACTED_SECRET = "[REDACTED]";
@@ -4673,12 +4746,19 @@ async function calibrateBenchJudges(parsed, rawArgs) {
     );
     process.exit(1);
   }
+  if (!parsed.sourceResultId || !parsed.expectedAnswerSetSha256 || !parsed.expectedQuestionIdListSha256) {
+    console.error(
+      "ERROR: judge-calibrate requires --source-result-id, --expected-answer-set-sha256, and --expected-question-id-list-sha256 so all source drift is rejected before judge calls."
+    );
+    process.exit(1);
+  }
   const bench = await loadBenchModule();
   const resultsDir = expandTilde(
     parsed.resultsDir ?? path11.join(resolveHomeDir(), ".remnic", "bench", "results")
   );
-  const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
-  const previousCalibration = await bench.loadJudgeCalibrationState?.(benchmarkId, calibrationDir);
+  const calibrationDir = expandTilde(
+    parsed.calibrationDir ?? path11.join(resolveHomeDir(), ".remnic", "bench", "calibration")
+  );
   const stored = await bench.listBenchmarkResults(resultsDir);
   const allForBenchmark = stored.filter((entry) => entry.benchmark === benchmarkId);
   const candidates = allForBenchmark.filter((entry) => entry.mode === "full").sort((a, b) => {
@@ -4687,32 +4767,16 @@ async function calibrateBenchJudges(parsed, rawArgs) {
     }
     return a.id < b.id ? 1 : a.id > b.id ? -1 : 0;
   });
-  const pinnedSourceId = previousCalibration?.sourceResultId;
-  const latest = pinnedSourceId ? candidates.find((entry) => entry.id === pinnedSourceId) : candidates[0];
+  const pinnedSourceId = parsed.sourceResultId;
+  const latest = candidates.find((entry) => entry.id === pinnedSourceId);
   if (!latest) {
-    if (pinnedSourceId) {
-      console.error(
-        `ERROR: pinned calibration source ${pinnedSourceId} for "${benchmarkId}" is no longer present in ${resultsDir}. Restore that stored result or remove the calibration state intentionally before selecting a new answer set.`
-      );
-      process.exit(1);
-    }
-    const quickCount = allForBenchmark.filter((entry) => entry.mode === "quick").length;
     console.error(
-      quickCount > 0 ? `ERROR: no full stored results for "${benchmarkId}" in ${resultsDir} (found ${quickCount} quick run(s); a 1-task quick sample cannot calibrate the judge). Run a full benchmark first (remnic bench run ${benchmarkId}).` : `ERROR: no stored benchmark results for "${benchmarkId}" in ${resultsDir}. Run the benchmark first (remnic bench run ${benchmarkId}) so cached answers exist to calibrate against.`
+      `ERROR: explicitly pinned full calibration source ${pinnedSourceId} for "${benchmarkId}" is not present in ${resultsDir}. Restore that exact stored result.`
     );
     process.exit(1);
   }
   let loaded = await bench.loadBenchmarkResult(latest.path);
-  if (!pinnedSourceId && loaded.meta.status === "partial") {
-    for (const candidate of candidates.slice(1)) {
-      const candidateResult = await bench.loadBenchmarkResult(candidate.path);
-      if (candidateResult.meta.status !== "partial") {
-        loaded = candidateResult;
-        break;
-      }
-    }
-  }
-  if (pinnedSourceId && loaded.meta.status === "partial") {
+  if (loaded.meta.status === "partial") {
     console.error(
       `ERROR: pinned calibration source ${pinnedSourceId} is partial. Restore a complete copy or remove the calibration state intentionally before selecting a new answer set.`
     );
@@ -4732,23 +4796,53 @@ async function calibrateBenchJudges(parsed, rawArgs) {
     predicted: task.actual,
     expected: task.expected
   }));
-  const manifest = await bench.loadLocalLabManifest(expandTilde(manifestPath));
-  const resolvedProfile = bench.resolveLocalLabProfile(manifest);
-  const localJudgeConfig = resolvedProfile.judge.providerConfig;
-  const localJudge = bench.createProviderBackedJudge(localJudgeConfig);
-  const frontierJudge = bench.createProviderBackedJudge({
-    provider: parsed.judgeProvider,
-    model: parsed.judgeModel,
-    ...parsed.judgeBaseUrl ? { baseUrl: parsed.judgeBaseUrl } : {},
-    ...parsed.judgeApiKey ? { apiKey: parsed.judgeApiKey } : {}
+  const orderedQuestionIdsHash = bench.hashOrderedQuestionIds(
+    loaded.results.tasks.map((task) => task.taskId)
+  );
+  if (orderedQuestionIdsHash !== parsed.expectedQuestionIdListSha256) {
+    console.error(
+      `ERROR: ordered question-id list changed for ${loaded.meta.id} (expected sha256:${parsed.expectedQuestionIdListSha256}, got sha256:${orderedQuestionIdsHash}); refusing to call judges.`
+    );
+    process.exit(1);
+  }
+  const sourceResultSha256 = createHash("sha256").update(fs7.readFileSync(latest.path)).digest("hex");
+  const expandedManifestPath = expandTilde(manifestPath);
+  if (!bench.resolveLocalLabJudgeProviderConfig) {
+    console.error(
+      "ERROR: installed @remnic/bench version does not export resolveLocalLabJudgeProviderConfig; rebuild or upgrade @remnic/bench before calibrating."
+    );
+    process.exit(1);
+  }
+  const localJudgeConfig = await bench.resolveLocalLabJudgeProviderConfig({
+    localLabManifestPath: expandedManifestPath,
+    ...parsed.localJudgeRequestTimeout ? { requestTimeout: parsed.localJudgeRequestTimeout } : {},
+    max429WaitMs: parsed.max429WaitMs,
+    disableThinking: parsed.disableThinking
   });
+  const localJudge = bench.createProviderBackedJudge(localJudgeConfig);
+  const frontierJudgeConfig = buildCalibrationFrontierJudgeConfig(
+    parsed
+  );
+  const frontierJudge = bench.createProviderBackedJudge(frontierJudgeConfig);
+  const localJudgeConfigHash = hashCalibrationProviderConfig(localJudgeConfig);
+  const frontierJudgeConfigHash = hashCalibrationProviderConfig(frontierJudgeConfig);
   const result = await bench.runJudgeCalibration({
     benchmarkId,
     localJudge,
     frontierJudge,
     answers,
-    ...previousCalibration?.sliceQuestionIds ? { pinnedQuestionIds: previousCalibration.sliceQuestionIds } : {},
-    ...previousCalibration?.answerSetHash ? { expectedAnswerSetHash: previousCalibration.answerSetHash } : {}
+    expectedAnswerSetHash: parsed.expectedAnswerSetSha256,
+    expectedOrderedQuestionIdsHash: parsed.expectedQuestionIdListSha256,
+    checkpoint: {
+      dir: calibrationDir,
+      sourceResultId: loaded.meta.id,
+      sourceResultSha256,
+      orderedQuestionIdsHash,
+      localJudgePromptIdentity: bench.getProviderBackedJudgePromptIdentity(localJudgeConfig),
+      frontierJudgePromptIdentity: bench.getProviderBackedJudgePromptIdentity(frontierJudgeConfig),
+      localJudgeConfigHash,
+      frontierJudgeConfigHash
+    }
   });
   const calibrationIdentities = {
     localJudgeProvider: String(localJudgeConfig.provider),
@@ -4760,10 +4854,10 @@ async function calibrateBenchJudges(parsed, rawArgs) {
     result,
     calibrationDir,
     calibrationIdentities,
-    { sourceResultId: loaded.meta.id }
+    { sourceResultId: loaded.meta.id, localJudgeConfigHash, frontierJudgeConfigHash }
   );
   const persisted = await bench.loadJudgeCalibrationState(benchmarkId, calibrationDir);
-  if (!persisted || persisted.kappa !== result.kappa || persisted.warning !== result.warning) {
+  if (!persisted || persisted.kappa !== result.kappa || persisted.warning !== result.warning || persisted.localJudgeConfigHash !== localJudgeConfigHash || persisted.frontierJudgeConfigHash !== frontierJudgeConfigHash) {
     console.error(
       `ERROR: calibration state round-trip failed for ${benchmarkId} (wrote kappa ${result.kappa}, read back ${persisted ? persisted.kappa : "nothing"}). Re-run judge-calibrate.`
     );
@@ -4785,6 +4879,11 @@ async function calibrateBenchJudges(parsed, rawArgs) {
           answerSetHash: result.answerSetHash,
           sourceResultId: loaded.meta.id,
           categories: result.categories,
+          orderedQuestionIdsHash,
+          sourceResultSha256,
+          localJudgeConfigHash,
+          frontierJudgeConfigHash,
+          execution: result.execution,
           statePath
         },
         null,
@@ -4800,6 +4899,8 @@ async function calibrateBenchJudges(parsed, rawArgs) {
     `  ${(result.confidenceInterval.level * 100).toFixed(0)}% bootstrap CI: [${result.confidenceInterval.lower.toFixed(4)}, ${result.confidenceInterval.upper.toFixed(4)}] (${result.bootstrapSamples} paired resamples)`
   );
   console.log(`  Pinned source: ${loaded.meta.id} (answer set sha256:${result.answerSetHash})`);
+  console.log(`  Ordered question ids: sha256:${orderedQuestionIdsHash}`);
+  console.log(`  Judge calls: local ${result.execution.localJudgeCalls}, frontier ${result.execution.frontierJudgeCalls}; resumed outputs ${result.execution.resumedJudgeOutputs}`);
   console.log(`  Observed agreement: ${result.observedAgreement.toFixed(4)}`);
   console.log(`  Expected agreement: ${result.expectedAgreement.toFixed(4)}`);
   if (result.warning) {
@@ -4812,6 +4913,26 @@ async function calibrateBenchJudges(parsed, rawArgs) {
   console.log(`  Calibration state written + verified (round-trip ok): ${statePath}`);
   console.log(`  Subsequent local artifacts for ${benchmarkId} will carry kappa ${persisted.kappa.toFixed(4)}.`);
 }
+function buildCalibrationFrontierJudgeConfig(parsed) {
+  if (!parsed.judgeProvider || !parsed.judgeModel) {
+    throw new Error(
+      "Calibration frontier judge requires both --judge-provider and --judge-model."
+    );
+  }
+  return {
+    provider: parsed.judgeProvider,
+    model: parsed.judgeModel,
+    ...parsed.judgeBaseUrl ? { baseUrl: parsed.judgeBaseUrl } : {},
+    ...parsed.judgeApiKey ? { apiKey: parsed.judgeApiKey } : {},
+    ...parsed.frontierJudgeRequestTimeout !== void 0 || parsed.max429WaitMs !== void 0 ? {
+      retryOptions: {
+        ...parsed.frontierJudgeRequestTimeout !== void 0 ? { timeoutMs: parsed.frontierJudgeRequestTimeout } : {},
+        ...parsed.max429WaitMs !== void 0 ? { max429WaitMs: parsed.max429WaitMs } : {}
+      }
+    } : {},
+    ...parsed.disableThinking ? { disableThinking: true } : {}
+  };
+}
 async function publishBenchPackageResults(parsed) {
   if (parsed.benchmarks.length > 0) {
     console.error(
@@ -5200,6 +5321,12 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
   if (!plan) {
     return { ok: false };
   }
+  const judgeCalibration = await preparePersistedJudgeCalibrationAttachment(
+    benchModule,
+    benchmarkId,
+    plan.runtime.judgeProvider,
+    parsed
+  );
   await preflightLocalLabEndpointsIfNeeded(benchModule, plan);
   const outputDir = parsed.resultsDir ?? resolveBenchOutputDir();
   const datasetDir = resolveBenchDatasetDir(
@@ -5283,7 +5410,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
     });
     result.config.remnicConfig = plan.runtime.remnicConfig;
     result.config.internalProvider = plan.runtime.internalProvider;
-    await attachPersistedJudgeCalibration(benchModule, benchmarkId, result);
+    attachPreparedJudgeCalibration(result, judgeCalibration);
     const writtenPath = await benchModule.writeBenchmarkResult(result, outputDir);
     if (parsed.json) {
       console.log(JSON.stringify(redactBenchResultForStdout(benchModule, result), null, 2));
@@ -5304,7 +5431,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
         err instanceof Error ? err.message : String(err),
         parsed.quick ? "quick" : "full"
       );
-      await attachPersistedJudgeCalibration(benchModule, benchmarkId, partialResult);
+      attachPreparedJudgeCalibration(partialResult, judgeCalibration);
       try {
         const partialPath = await benchModule.writeBenchmarkResult(partialResult, outputDir);
         console.error(`  Partial results (${partialTasks.length} tasks) written to ${partialPath}`);
@@ -5328,32 +5455,111 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
     }
   }
 }
-async function attachPersistedJudgeCalibration(benchModule, benchmarkId, result) {
-  const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
+async function preparePersistedJudgeCalibrationAttachment(benchModule, benchmarkId, runJudgeProvider, calibrationBinding) {
+  const hasLocalPin = Boolean(calibrationBinding.calibrationLocalConfigSha256);
+  const hasFrontierPin = Boolean(calibrationBinding.calibrationFrontierConfigSha256);
+  const hasAnyPin = hasLocalPin || hasFrontierPin;
+  const hasBothPins = hasLocalPin && hasFrontierPin;
+  if (benchmarkId === "ama-bench" && calibrationBinding.amaBenchJudgeProtocol === "recommended") {
+    if (hasAnyPin) {
+      throw new Error(
+        "AMA-Bench recommended-protocol runs cannot attach default-protocol judge calibration; remove both calibration pins or calibrate the recommended prompt contract separately."
+      );
+    }
+    return void 0;
+  }
+  if (hasAnyPin && !hasBothPins) {
+    throw new Error(
+      "--calibration-local-config-sha256 and --calibration-frontier-config-sha256 must be supplied together."
+    );
+  }
+  const calibrationDir = expandTilde(
+    calibrationBinding.calibrationDir ?? path11.join(resolveHomeDir(), ".remnic", "bench", "calibration")
+  );
   const state = await benchModule.loadJudgeCalibrationState?.(benchmarkId, calibrationDir);
-  if (!state) return;
-  if (state.localJudgeModel !== void 0 && state.frontierJudgeModel !== void 0) {
-    const runJudgeProvider = result.config.judgeProvider?.provider;
-    const runJudgeModel = result.config.judgeProvider?.model;
-    const matchesLocal = runJudgeProvider === state.localJudgeProvider && runJudgeModel === state.localJudgeModel;
+  if (!state) {
+    if (hasAnyPin) {
+      throw new Error(
+        `Calibration binding pins were supplied for ${benchmarkId}, but no valid calibration state could be loaded from ${calibrationDir}; rerun judge-calibrate or remove both pins.`
+      );
+    }
+    return void 0;
+  }
+  if (state.localJudgeProvider !== void 0 && state.localJudgeModel !== void 0) {
+    const matchesLocal = runJudgeProvider?.provider === state.localJudgeProvider && runJudgeProvider.model === state.localJudgeModel;
     if (!matchesLocal) {
-      return;
+      if (hasBothPins) {
+        throw new Error(
+          `Calibration binding pins for ${benchmarkId} target the calibrated local judge ${state.localJudgeProvider}/${state.localJudgeModel}, but the run judge is ${runJudgeProvider?.provider ?? "unset"}/${runJudgeProvider?.model ?? "unset"}; refusing to ignore explicit pins.`
+        );
+      }
+      return void 0;
     }
   }
+  if (!state.localJudgeConfigHash || !state.frontierJudgeConfigHash) {
+    throw new Error(
+      `Calibration state for ${benchmarkId} is missing bound judge configuration hashes; recalibrate before attaching it.`
+    );
+  }
+  const resolvedRunJudgeConfigHash = hashCalibrationProviderConfig(runJudgeProvider);
+  const hasCompleteLocalIdentity = state.localJudgeProvider !== void 0 && state.localJudgeModel !== void 0;
+  if (!hasCompleteLocalIdentity && resolvedRunJudgeConfigHash !== state.localJudgeConfigHash) {
+    if (hasBothPins) {
+      throw new Error(
+        `Calibration binding pins for ${benchmarkId} do not match the resolved run judge configuration; refusing to ignore explicit pins.`
+      );
+    }
+    return void 0;
+  }
+  if (!calibrationBinding.calibrationLocalConfigSha256 || !calibrationBinding.calibrationFrontierConfigSha256) {
+    throw new Error(
+      `Calibration state exists for ${benchmarkId}; --calibration-local-config-sha256 and --calibration-frontier-config-sha256 are required to attach it.`
+    );
+  }
+  if (calibrationBinding.calibrationLocalConfigSha256 !== state.localJudgeConfigHash || calibrationBinding.calibrationFrontierConfigSha256 !== state.frontierJudgeConfigHash) {
+    throw new Error(`Calibration configuration hash mismatch for ${benchmarkId}; refusing to attach stale kappa.`);
+  }
+  if (resolvedRunJudgeConfigHash !== state.localJudgeConfigHash) {
+    throw new Error(
+      `Resolved run judge configuration hash mismatch for ${benchmarkId}; expected sha256:${state.localJudgeConfigHash}, got sha256:${resolvedRunJudgeConfigHash}. Refusing to attach stale kappa before benchmark dispatch.`
+    );
+  }
+  return {
+    kappa: state.kappa,
+    sampleSize: state.sampleSize,
+    threshold: state.threshold,
+    warning: state.warning,
+    ...state.confidenceInterval ? { confidenceInterval: state.confidenceInterval } : {},
+    ...state.bootstrapSamples ? { bootstrapSamples: state.bootstrapSamples } : {},
+    ...state.answerSetHash ? { answerSetHash: state.answerSetHash } : {},
+    ...state.sourceResultId ? { sourceResultId: state.sourceResultId } : {},
+    ...state.sliceQuestionIds ? { sliceQuestionIds: state.sliceQuestionIds } : {},
+    localJudgeConfigHash: state.localJudgeConfigHash,
+    frontierJudgeConfigHash: state.frontierJudgeConfigHash
+  };
+}
+function attachPreparedJudgeCalibration(result, judgeCalibration) {
+  if (!judgeCalibration) return;
   result.config.benchmarkOptions = {
     ...result.config.benchmarkOptions ?? {},
-    judgeCalibration: {
-      kappa: state.kappa,
-      sampleSize: state.sampleSize,
-      threshold: state.threshold,
-      warning: state.warning,
-      ...state.confidenceInterval ? { confidenceInterval: state.confidenceInterval } : {},
-      ...state.bootstrapSamples ? { bootstrapSamples: state.bootstrapSamples } : {},
-      ...state.answerSetHash ? { answerSetHash: state.answerSetHash } : {},
-      ...state.sourceResultId ? { sourceResultId: state.sourceResultId } : {},
-      ...state.sliceQuestionIds ? { sliceQuestionIds: state.sliceQuestionIds } : {}
+    judgeCalibration
+  };
+}
+function hashCalibrationProviderConfig(config) {
+  const canonicalize = (value, key = "") => {
+    if (typeof value === "string" && /(?:api.?key|authorization|token|secret)/i.test(key)) {
+      return { secretSha256: createHash("sha256").update(value).digest("hex") };
     }
+    if (Array.isArray(value)) return value.map((item) => canonicalize(item));
+    if (value && typeof value === "object") {
+      return Object.fromEntries(Object.keys(value).sort().map((childKey) => [
+        childKey,
+        canonicalize(value[childKey], childKey)
+      ]));
+    }
+    return value;
   };
+  return createHash("sha256").update(JSON.stringify(canonicalize(config))).digest("hex");
 }
 function restoreOptionalEnv(key, previousValue) {
   if (previousValue === void 0) {
@@ -12920,7 +13126,9 @@ export {
   __benchDatasetTestHooks,
   advanceOfflineBaseFilesForSuccessfulPush,
   advanceOfflineLargeFileFailureCounts,
+  attachPreparedJudgeCalibration,
   buildBenchRuntimeProfileRequest,
+  buildCalibrationFrontierJudgeConfig,
   buildPackageBenchExecutionPlans,
   buildQueryRecallRequest,
   chunkOfflineChangesetApplyBatches,
@@ -12932,6 +13140,7 @@ export {
   formatOfflineRequestForError,
   getBenchUsageText,
   hasFlag,
+  hashCalibrationProviderConfig,
   hydrateOfflineSnapshotContent,
   isOfflineMissingContentDeferrablePath,
   isOfflineSnapshotPostFallbackError,
@@ -12947,6 +13156,7 @@ export {
   parseOfflineSyncRequestTimeoutMs,
   parseTaxonomyResolveArgs,
   parseTrainingExportArgs,
+  preparePersistedJudgeCalibrationAttachment,
   pushOfflineFileContent,
   pushOfflineFileContentFromChunkReader,
   renderQueryTextLines,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/cli",
-  "version": "9.6.23",
+  "version": "9.6.25",
   "description": "CLI for Remnic memory — init, query, doctor, daemon management",
   "type": "module",
   "main": "dist/index.js",
@@ -26,23 +26,23 @@
   },
   "dependencies": {
     "yaml": "^2.4.2",
-    "@remnic/plugin-pi": "^9.6.23",
-    "@remnic/core": "^9.6.23",
-    "@remnic/server": "^9.6.23"
+    "@remnic/plugin-pi": "^9.6.25",
+    "@remnic/core": "^9.6.25",
+    "@remnic/server": "^9.6.25"
   },
   "peerDependencies": {
-    "@remnic/bench": "^9.6.23",
-    "@remnic/export-weclone": "^9.6.23",
-    "@remnic/import-weclone": "^9.6.23",
-    "@remnic/import-chatgpt": "^9.6.23",
-    "@remnic/import-claude": "^9.6.23",
-    "@remnic/import-gemini": "^9.6.23",
-    "@remnic/import-lossless-claw": "^9.6.23",
-    "@remnic/import-mem0": "^9.6.23",
-    "@remnic/import-supermemory": "^9.6.23",
-    "@remnic/connector-limitless": "^9.6.23",
-    "@remnic/connector-bee": "^9.6.23",
-    "@remnic/connector-omi": "^9.6.23"
+    "@remnic/bench": "^9.6.25",
+    "@remnic/export-weclone": "^9.6.25",
+    "@remnic/import-weclone": "^9.6.25",
+    "@remnic/import-chatgpt": "^9.6.25",
+    "@remnic/import-claude": "^9.6.25",
+    "@remnic/import-gemini": "^9.6.25",
+    "@remnic/import-lossless-claw": "^9.6.25",
+    "@remnic/import-mem0": "^9.6.25",
+    "@remnic/import-supermemory": "^9.6.25",
+    "@remnic/connector-limitless": "^9.6.25",
+    "@remnic/connector-bee": "^9.6.25",
+    "@remnic/connector-omi": "^9.6.25"
   },
   "peerDependenciesMeta": {
     "@remnic/bench": {
@@ -85,18 +85,18 @@
   "devDependencies": {
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
-    "@remnic/bench": "9.6.23",
-    "@remnic/export-weclone": "9.6.23",
-    "@remnic/import-weclone": "9.6.23",
-    "@remnic/import-chatgpt": "9.6.23",
-    "@remnic/import-claude": "9.6.23",
-    "@remnic/import-gemini": "9.6.23",
-    "@remnic/import-lossless-claw": "9.6.23",
-    "@remnic/import-mem0": "9.6.23",
-    "@remnic/connector-limitless": "9.6.23",
-    "@remnic/connector-bee": "9.6.23",
-    "@remnic/import-supermemory": "9.6.23",
-    "@remnic/connector-omi": "9.6.23"
+    "@remnic/bench": "9.6.25",
+    "@remnic/export-weclone": "9.6.25",
+    "@remnic/import-weclone": "9.6.25",
+    "@remnic/import-chatgpt": "9.6.25",
+    "@remnic/import-claude": "9.6.25",
+    "@remnic/import-mem0": "9.6.25",
+    "@remnic/import-supermemory": "9.6.25",
+    "@remnic/import-gemini": "9.6.25",
+    "@remnic/import-lossless-claw": "9.6.25",
+    "@remnic/connector-limitless": "9.6.25",
+    "@remnic/connector-bee": "9.6.25",
+    "@remnic/connector-omi": "9.6.25"
   },
   "license": "MIT",
   "repository": {