@remnic/cli 9.3.701 → 9.3.702

package/dist/index.js

@@ -643,6 +643,7 @@ function readBenchOptionValue(argv, flag) {
 }
 var BENCH_VALUE_FLAGS = Object.freeze([
   "--dataset-dir",
+  "--benchmark",
   "--results-dir",
   "--baselines-dir",
   "--runtime-profile",
@@ -826,6 +827,18 @@ var BENCH_ACTION_FLAGS = {
     value: ["--results-dir", "--target", "--output"],
     boolean: ["--json", "--help", "-h"]
   },
+  "judge-calibrate": {
+    value: [
+      "--results-dir",
+      "--benchmark",
+      "--local-lab-manifest",
+      "--judge-provider",
+      "--judge-model",
+      "--judge-base-url",
+      "--judge-api-key"
+    ],
+    boolean: ["--json", "--help", "-h"]
+  },
   published: {
     value: PUBLISHED_VALUE_FLAGS,
     boolean: PUBLISHED_BOOLEAN_FLAGS
@@ -915,7 +928,7 @@ function collectBenchmarks(argv) {
 }
 function parseBenchActionArgs(argv) {
   const [first, ...rest] = argv;
-  const action = first === "list" || first === "run" || first === "datasets" || first === "runs" || first === "compare" || first === "ui" || first === "results" || first === "baseline" || first === "export" || first === "providers" || first === "publish" || first === "published" || first === "check" || first === "report" ? first : first === void 0 || first === "--help" || first === "-h" ? "help" : "run";
+  const action = first === "list" || first === "run" || first === "datasets" || first === "runs" || first === "compare" || first === "ui" || first === "results" || first === "baseline" || first === "export" || first === "providers" || first === "publish" || first === "published" || first === "judge-calibrate" || first === "check" || first === "report" ? first : first === void 0 || first === "--help" || first === "-h" ? "help" : "run";
   return {
     action,
     args: action === "run" && action !== first ? argv : rest
@@ -3219,8 +3232,8 @@ async function loadTrainingExportCoreRuntime() {
   return await import("@remnic/core");
 }
 function getBenchUsageText() {
-  return `Usage: remnic bench <list|run|published|datasets|runs|compare|results|baseline|export|publish|ui|providers> [options] [benchmark...]
-       remnic benchmark <list|run|published|datasets|runs|compare|results|baseline|export|publish|ui|providers|check|report> [options] [benchmark...]
+  return `Usage: remnic bench <list|run|published|datasets|runs|compare|results|baseline|export|publish|ui|providers|judge-calibrate> [options] [benchmark...]
+       remnic benchmark <list|run|published|datasets|runs|compare|results|baseline|export|publish|ui|providers|judge-calibrate|check|report> [options] [benchmark...]
 
 Commands:
   list                     List published benchmark packs
@@ -3248,6 +3261,11 @@ Commands:
                            Generate the Remnic.ai benchmark feed from stored runs
   ui                       Launch the local benchmark overview UI
   providers discover       Auto-detect available local provider backends
+  judge-calibrate --benchmark <id> --local-lab-manifest <path> --judge-provider <p> --judge-model <m>
+                           Cross-tier judge calibration (issue #1573): runs the
+                           local + frontier judges over a benchmark's cached
+                           answers, reports Cohen's kappa, and persists it so
+                           subsequent local artifacts carry the kappa + warning.
   check                    Legacy latency regression gate (compatibility)
   report                   Legacy latency report generator (compatibility)
   procedural-ablation --out <path> [--fixture <path>]
@@ -4425,6 +4443,144 @@ async function discoverBenchProviders(parsed) {
     }
   }
 }
+async function calibrateBenchJudges(parsed, rawArgs) {
+  const benchmarkId = readBenchOptionValue(rawArgs, "--benchmark") ?? parsed.benchmarks[0];
+  if (!benchmarkId) {
+    console.error(
+      "ERROR: judge-calibrate requires a benchmark. Usage: remnic bench judge-calibrate --benchmark <id> [--local-lab-manifest <path>] [--judge-provider <p> --judge-model <m>] [--results-dir <path>] [--json]"
+    );
+    process.exit(1);
+  }
+  const knownBenchmarkIds = await resolveKnownBenchmarkIds();
+  if (!knownBenchmarkIds.has(benchmarkId)) {
+    console.error(
+      `ERROR: unknown benchmark "${benchmarkId}". Known: ${[...knownBenchmarkIds].sort().join(", ")}.`
+    );
+    process.exit(1);
+  }
+  const manifestPath = parsed.localLabManifestPath;
+  if (!manifestPath) {
+    console.error(
+      "ERROR: judge-calibrate requires --local-lab-manifest <path> (the Tier L judge source). Provide a local-lab manifest whose judge role names the local model."
+    );
+    process.exit(1);
+  }
+  if (!parsed.judgeProvider || !parsed.judgeModel) {
+    console.error(
+      "ERROR: judge-calibrate requires --judge-provider <p> and --judge-model <m> (the Tier F gold-standard judge)."
+    );
+    process.exit(1);
+  }
+  const bench = await loadBenchModule();
+  const resultsDir = expandTilde(
+    parsed.resultsDir ?? path11.join(resolveHomeDir(), ".remnic", "bench", "results")
+  );
+  const stored = await bench.listBenchmarkResults(resultsDir);
+  const allForBenchmark = stored.filter((entry) => entry.benchmark === benchmarkId);
+  const candidates = allForBenchmark.filter((entry) => entry.mode === "full").sort((a, b) => {
+    if (a.timestamp !== b.timestamp) {
+      return a.timestamp < b.timestamp ? 1 : -1;
+    }
+    return a.id < b.id ? 1 : a.id > b.id ? -1 : 0;
+  });
+  const latest = candidates[0];
+  if (!latest) {
+    const quickCount = allForBenchmark.filter((entry) => entry.mode === "quick").length;
+    console.error(
+      quickCount > 0 ? `ERROR: no full stored results for "${benchmarkId}" in ${resultsDir} (found ${quickCount} quick run(s); a 1-task quick sample cannot calibrate the judge). Run a full benchmark first (remnic bench run ${benchmarkId}).` : `ERROR: no stored benchmark results for "${benchmarkId}" in ${resultsDir}. Run the benchmark first (remnic bench run ${benchmarkId}) so cached answers exist to calibrate against.`
+    );
+    process.exit(1);
+  }
+  let loaded = await bench.loadBenchmarkResult(latest.path);
+  if (loaded.meta.status === "partial") {
+    for (const candidate of candidates.slice(1)) {
+      const candidateResult = await bench.loadBenchmarkResult(candidate.path);
+      if (candidateResult.meta.status !== "partial") {
+        loaded = candidateResult;
+        break;
+      }
+    }
+  }
+  const uniqueTaskIds = new Set(loaded.results.tasks.map((task) => task.taskId));
+  const sourceTaskCount = uniqueTaskIds.size;
+  if (sourceTaskCount < bench.MIN_CALIBRATION_SOURCE_TASKS) {
+    console.error(
+      `ERROR: stored result for "${benchmarkId}" has only ${sourceTaskCount} task(s) \u2014 too few for a meaningful calibration (minimum ${bench.MIN_CALIBRATION_SOURCE_TASKS}). Run a full uncapped benchmark first (remnic bench run ${benchmarkId}).`
+    );
+    process.exit(1);
+  }
+  const answers = loaded.results.tasks.map((task) => ({
+    questionId: task.taskId,
+    question: task.question,
+    predicted: task.actual,
+    expected: task.expected
+  }));
+  const manifest = await bench.loadLocalLabManifest(expandTilde(manifestPath));
+  const resolvedProfile = bench.resolveLocalLabProfile(manifest);
+  const localJudgeConfig = resolvedProfile.judge.providerConfig;
+  const localJudge = bench.createProviderBackedJudge(localJudgeConfig);
+  const frontierJudge = bench.createProviderBackedJudge({
+    provider: parsed.judgeProvider,
+    model: parsed.judgeModel,
+    ...parsed.judgeBaseUrl ? { baseUrl: parsed.judgeBaseUrl } : {},
+    ...parsed.judgeApiKey ? { apiKey: parsed.judgeApiKey } : {}
+  });
+  const result = await bench.runJudgeCalibration({
+    benchmarkId,
+    localJudge,
+    frontierJudge,
+    answers
+  });
+  const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
+  const calibrationIdentities = {
+    localJudgeProvider: String(localJudgeConfig.provider),
+    localJudgeModel: String(localJudgeConfig.model),
+    frontierJudgeProvider: parsed.judgeProvider,
+    frontierJudgeModel: parsed.judgeModel
+  };
+  const statePath = await bench.writeJudgeCalibrationState(result, calibrationDir, calibrationIdentities);
+  const persisted = await bench.loadJudgeCalibrationState(benchmarkId, calibrationDir);
+  if (!persisted || persisted.kappa !== result.kappa || persisted.warning !== result.warning) {
+    console.error(
+      `ERROR: calibration state round-trip failed for ${benchmarkId} (wrote kappa ${result.kappa}, read back ${persisted ? persisted.kappa : "nothing"}). Re-run judge-calibrate.`
+    );
+    process.exit(1);
+  }
+  if (parsed.json) {
+    console.log(
+      JSON.stringify(
+        {
+          benchmarkId: result.benchmarkId,
+          kappa: result.kappa,
+          observedAgreement: result.observedAgreement,
+          expectedAgreement: result.expectedAgreement,
+          sampleSize: result.sampleSize,
+          threshold: result.threshold,
+          warning: result.warning,
+          categories: result.categories,
+          statePath
+        },
+        null,
+        2
+      )
+    );
+    return;
+  }
+  console.log(`Judge calibration: ${benchmarkId}`);
+  console.log(`  Cohen's kappa: ${result.kappa.toFixed(4)} (threshold ${result.threshold})`);
+  console.log(`  Sample size:   ${result.sampleSize}`);
+  console.log(`  Observed agreement: ${result.observedAgreement.toFixed(4)}`);
+  console.log(`  Expected agreement: ${result.expectedAgreement.toFixed(4)}`);
+  if (result.warning) {
+    console.log(
+      `  WARNING: local judge unreliable for ${benchmarkId} (kappa ${result.kappa.toFixed(4)} < ${result.threshold}). Tier L numbers for this benchmark should not be trusted for regression until the judge improves.`
+    );
+  } else {
+    console.log(`  OK: local judge agrees with frontier above threshold.`);
+  }
+  console.log(`  Calibration state written + verified (round-trip ok): ${statePath}`);
+  console.log(`  Subsequent local artifacts for ${benchmarkId} will carry kappa ${persisted.kappa.toFixed(4)}.`);
+}
 async function publishBenchPackageResults(parsed) {
   if (parsed.benchmarks.length > 0) {
     console.error(
@@ -4889,6 +5045,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
     });
     result.config.remnicConfig = plan.runtime.remnicConfig;
     result.config.internalProvider = plan.runtime.internalProvider;
+    await attachPersistedJudgeCalibration(benchModule, benchmarkId, result);
     const writtenPath = await benchModule.writeBenchmarkResult(result, outputDir);
     if (parsed.json) {
       console.log(JSON.stringify(redactBenchResultForStdout(benchModule, result), null, 2));
@@ -4909,6 +5066,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
         err instanceof Error ? err.message : String(err),
         parsed.quick ? "quick" : "full"
       );
+      await attachPersistedJudgeCalibration(benchModule, benchmarkId, partialResult);
       try {
         const partialPath = await benchModule.writeBenchmarkResult(partialResult, outputDir);
         console.error(`  Partial results (${partialTasks.length} tasks) written to ${partialPath}`);
@@ -4932,6 +5090,28 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
     }
   }
 }
+async function attachPersistedJudgeCalibration(benchModule, benchmarkId, result) {
+  const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
+  const state = await benchModule.loadJudgeCalibrationState?.(benchmarkId, calibrationDir);
+  if (!state) return;
+  if (state.localJudgeModel !== void 0 && state.frontierJudgeModel !== void 0) {
+    const runJudgeProvider = result.config.judgeProvider?.provider;
+    const runJudgeModel = result.config.judgeProvider?.model;
+    const matchesLocal = runJudgeProvider === state.localJudgeProvider && runJudgeModel === state.localJudgeModel;
+    if (!matchesLocal) {
+      return;
+    }
+  }
+  result.config.benchmarkOptions = {
+    ...result.config.benchmarkOptions ?? {},
+    judgeCalibration: {
+      kappa: state.kappa,
+      sampleSize: state.sampleSize,
+      threshold: state.threshold,
+      warning: state.warning
+    }
+  };
+}
 function restoreOptionalEnv(key, previousValue) {
   if (previousValue === void 0) {
     delete process.env[key];
@@ -9666,6 +9846,10 @@ async function cmdBench(rest) {
     await exportBenchPackageResult(parsed);
     return;
   }
+  if (parsed.action === "judge-calibrate") {
+    await calibrateBenchJudges(parsed, benchAction.args);
+    return;
+  }
   if (parsed.action === "datasets") {
     await manageBenchDatasets(parsed);
     return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/cli",
-  "version": "9.3.701",
+  "version": "9.3.702",
   "description": "CLI for Remnic memory — init, query, doctor, daemon management",
   "type": "module",
   "main": "dist/index.js",
@@ -26,23 +26,23 @@
   },
   "dependencies": {
     "yaml": "^2.4.2",
-    "@remnic/plugin-pi": "^9.3.701",
-    "@remnic/server": "^9.3.701",
-    "@remnic/core": "^9.3.701"
+    "@remnic/plugin-pi": "^9.3.702",
+    "@remnic/core": "^9.3.702",
+    "@remnic/server": "^9.3.702"
   },
   "peerDependencies": {
-    "@remnic/bench": "^9.3.701",
-    "@remnic/export-weclone": "^9.3.701",
-    "@remnic/import-weclone": "^9.3.701",
-    "@remnic/import-chatgpt": "^9.3.701",
-    "@remnic/import-claude": "^9.3.701",
-    "@remnic/import-gemini": "^9.3.701",
-    "@remnic/import-lossless-claw": "^9.3.701",
-    "@remnic/import-mem0": "^9.3.701",
-    "@remnic/import-supermemory": "^9.3.701",
-    "@remnic/connector-limitless": "^9.3.701",
-    "@remnic/connector-bee": "^9.3.701",
-    "@remnic/connector-omi": "^9.3.701"
+    "@remnic/bench": "^9.3.702",
+    "@remnic/export-weclone": "^9.3.702",
+    "@remnic/import-weclone": "^9.3.702",
+    "@remnic/import-chatgpt": "^9.3.702",
+    "@remnic/import-claude": "^9.3.702",
+    "@remnic/import-gemini": "^9.3.702",
+    "@remnic/import-lossless-claw": "^9.3.702",
+    "@remnic/import-mem0": "^9.3.702",
+    "@remnic/import-supermemory": "^9.3.702",
+    "@remnic/connector-limitless": "^9.3.702",
+    "@remnic/connector-bee": "^9.3.702",
+    "@remnic/connector-omi": "^9.3.702"
   },
   "peerDependenciesMeta": {
     "@remnic/bench": {
@@ -85,18 +85,18 @@
   "devDependencies": {
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
-    "@remnic/bench": "9.3.701",
-    "@remnic/export-weclone": "9.3.701",
-    "@remnic/import-weclone": "9.3.701",
-    "@remnic/import-chatgpt": "9.3.701",
-    "@remnic/import-gemini": "9.3.701",
-    "@remnic/import-mem0": "9.3.701",
-    "@remnic/import-claude": "9.3.701",
-    "@remnic/import-lossless-claw": "9.3.701",
-    "@remnic/import-supermemory": "9.3.701",
-    "@remnic/connector-omi": "9.3.701",
-    "@remnic/connector-bee": "9.3.701",
-    "@remnic/connector-limitless": "9.3.701"
+    "@remnic/bench": "9.3.702",
+    "@remnic/export-weclone": "9.3.702",
+    "@remnic/import-weclone": "9.3.702",
+    "@remnic/import-chatgpt": "9.3.702",
+    "@remnic/import-claude": "9.3.702",
+    "@remnic/import-gemini": "9.3.702",
+    "@remnic/import-mem0": "9.3.702",
+    "@remnic/import-lossless-claw": "9.3.702",
+    "@remnic/import-supermemory": "9.3.702",
+    "@remnic/connector-omi": "9.3.702",
+    "@remnic/connector-limitless": "9.3.702",
+    "@remnic/connector-bee": "9.3.702"
   },
   "license": "MIT",
   "repository": {