@remnic/cli 9.7.0 → 9.7.1

package/dist/index.js

@@ -790,6 +790,8 @@ var BENCH_VALUE_FLAGS = Object.freeze([
   "--source-result-id",
   "--expected-answer-set-sha256",
   "--expected-question-id-list-sha256",
+  "--task-ids-file",
+  "--expected-task-id-list-sha256",
   "--drain-timeout",
   "--max-429-wait",
   "--ama-bench-judge-protocol",
@@ -873,6 +875,8 @@ var RUN_VALUE_FLAGS = Object.freeze([
   "--calibration-dir",
   "--calibration-local-config-sha256",
   "--calibration-frontier-config-sha256",
+  "--task-ids-file",
+  "--expected-task-id-list-sha256",
   "--drain-timeout",
   "--max-429-wait",
   "--ama-bench-judge-protocol",
@@ -1147,6 +1151,8 @@ function parseBenchArgs(argv) {
   const sourceResultId = readBenchOptionValue(args, "--source-result-id");
   const expectedAnswerSetSha256 = readBenchOptionValue(args, "--expected-answer-set-sha256");
   const expectedQuestionIdListSha256 = readBenchOptionValue(args, "--expected-question-id-list-sha256");
+  const taskIdsFileRaw = readBenchOptionValue(args, "--task-ids-file");
+  const expectedTaskIdListSha256 = readBenchOptionValue(args, "--expected-task-id-list-sha256");
   const drainTimeoutRaw = readBenchOptionValue(args, "--drain-timeout");
   const judgeCacheDirRaw = readBenchOptionValue(args, "--judge-cache-dir");
   const localLabManifestRaw = readBenchOptionValue(args, "--local-lab-manifest");
@@ -1335,6 +1341,7 @@ function parseBenchArgs(argv) {
   for (const [flag, digest] of [
     ["--expected-answer-set-sha256", expectedAnswerSetSha256],
     ["--expected-question-id-list-sha256", expectedQuestionIdListSha256],
+    ["--expected-task-id-list-sha256", expectedTaskIdListSha256],
     ["--calibration-local-config-sha256", calibrationLocalConfigSha256],
     ["--calibration-frontier-config-sha256", calibrationFrontierConfigSha256]
   ]) {
@@ -1450,6 +1457,29 @@ function parseBenchArgs(argv) {
   if (publishedTrialLimit !== void 0 && !trialLimitTargetsSupportedBenchmark) {
     throw new Error("ERROR: --trial-limit is currently supported only for LoCoMo and MemoryAgentBench.");
   }
+  const hasTaskIdsFile = taskIdsFileRaw !== void 0;
+  const hasExpectedTaskIdListSha256 = expectedTaskIdListSha256 !== void 0;
+  if (hasTaskIdsFile !== hasExpectedTaskIdListSha256) {
+    throw new Error(
+      "ERROR: --task-ids-file and --expected-task-id-list-sha256 must be supplied together."
+    );
+  }
+  if (hasTaskIdsFile) {
+    const targetsSingleLoCoMo = action === "published" ? publishedName === "locomo" && benchmarks.length === 0 : action === "run" && !args.includes("--all") && benchmarks.length === 1 && benchmarks[0] === "locomo";
+    if (!targetsSingleLoCoMo) {
+      throw new Error(
+        "ERROR: --task-ids-file is supported only for a single LoCoMo run."
+      );
+    }
+    if (args.includes("--quick")) {
+      throw new Error("ERROR: --task-ids-file requires a full LoCoMo run; remove --quick.");
+    }
+    if (publishedLimit !== void 0 || publishedTrialLimit !== void 0) {
+      throw new Error(
+        "ERROR: --task-ids-file cannot be combined with --limit or --trial-limit."
+      );
+    }
+  }
   let publishedTrialConcurrency;
   if (publishedTrialConcurrencyRaw !== void 0) {
     const parsed = Number(publishedTrialConcurrencyRaw);
@@ -1658,6 +1688,8 @@ function parseBenchArgs(argv) {
     sourceResultId,
     expectedAnswerSetSha256,
     expectedQuestionIdListSha256,
+    taskIdsFile: taskIdsFileRaw ? path4.resolve(expandTilde(taskIdsFileRaw)) : void 0,
+    expectedTaskIdListSha256,
     drainTimeout,
     // Issue #1573 PR1: surface judge-cache flags into the runner options.
     noJudgeCache: args.includes("--no-judge-cache"),
@@ -3601,6 +3633,9 @@ Options:
                            Path to a local-lab manifest JSON file (required for --runtime-profile local-lab)
   --threshold <value>      Regression threshold for compare (default: 0.05)
   --trial-limit <n>        Cap scored LoCoMo or MemoryAgentBench QA trials for staged published runs
+  --task-ids-file <path>   Select an explicit JSON array of LoCoMo task IDs
+  --expected-task-id-list-sha256 <sha256>
+                           Pin the selected LoCoMo task-ID list
   --task-filter <pattern>  BEAM diagnostic filter; match task id, ability, or question text
   --detail                 Include per-task details for bench results
   --format <json|csv|html> Output format for bench export
@@ -3773,6 +3808,11 @@ async function resolveKnownBenchmarkIds() {
   return knownIds;
 }
 async function runBenchViaFallback(parsed, benchmarkId, runtimeProfile) {
+  if (parsed.taskIdsFile) {
+    throw new Error(
+      "Fallback benchmark runner does not support hash-pinned LoCoMo task selection. Build/install @remnic/bench to use --task-ids-file."
+    );
+  }
   if (runtimeProfile === "real" && parsed.remnicConfigPath) {
     resolveExistingBenchRemnicConfigPath(parsed.remnicConfigPath);
   }
@@ -4270,9 +4310,11 @@ var __benchDatasetTestHooks = {
   isDatasetDownloaded,
   resolveBenchDatasetDir,
   resolveDownloadedBenchDatasetDir,
-  buildPublishedBenchmarkOptionsForTest(benchmarkId, args) {
-    return buildPublishedBenchmarkOptions(benchmarkId, args);
+  buildPublishedBenchmarkOptionsForTest(benchmarkId, args, taskSelector) {
+    return buildPublishedBenchmarkOptions(benchmarkId, args, taskSelector);
   },
+  loadPinnedLoCoMoTaskSelectorForTest: loadPinnedLoCoMoTaskSelector,
+  redactBenchTaskIdsFilePathForTest: redactBenchTaskIdsFilePath,
   validateRunnerManagedPublishedDryRunDatasetForTest,
   validateRunnerManagedPublishedDryRunDatasetWithModuleForTest(benchModule, benchmarkId, mode, datasetDir, limit, seed, benchmarkOptions) {
     return validateRunnerManagedPublishedDryRunDataset(
@@ -4285,6 +4327,17 @@ var __benchDatasetTestHooks = {
       benchmarkOptions
     );
   },
+  validatePinnedLoCoMoPublishedDryRunSelectorWithModuleForTest(benchModule, mode, datasetDir, limit, seed, benchmarkOptions, taskSelector) {
+    return validatePinnedLoCoMoPublishedDryRunSelector(
+      benchModule,
+      mode,
+      datasetDir,
+      limit,
+      seed,
+      benchmarkOptions,
+      taskSelector
+    );
+  },
   printBenchStatusLineForTest: printBenchStatusLine
 };
 function printBenchPackageSummary(result, outputPath, outputLabel = "Results saved") {
@@ -4997,6 +5050,7 @@ async function runBenchPublished(parsed) {
     );
     process.exit(1);
   }
+  const taskSelector = loadPinnedLoCoMoTaskSelector(parsed);
   if (parsed.publishedDryRun) {
     const loaded = await tryLoadBenchModule();
     if (!loaded) {
@@ -5012,7 +5066,8 @@ async function runBenchPublished(parsed) {
     const effectiveLimit = parsed.publishedLimit ?? (parsed.quick ? 1 : void 0);
     const benchmarkOptions = buildPublishedBenchmarkOptions(
       benchmarkId2,
-      parsed
+      parsed,
+      taskSelector
     );
     let itemCount;
     let loadResult;
@@ -5092,6 +5147,24 @@ async function runBenchPublished(parsed) {
       );
       process.exit(1);
     }
+    if (benchmarkId2 === "locomo") {
+      try {
+        await validatePinnedLoCoMoPublishedDryRunSelector(
+          benchModule,
+          mode,
+          parsed.datasetDir,
+          effectiveLimit,
+          parsed.publishedSeed,
+          benchmarkOptions,
+          taskSelector
+        );
+      } catch (error) {
+        console.error(
+          `ERROR: [dry-run] ${benchmarkId2}: ${error instanceof Error ? error.message : String(error)}`
+        );
+        process.exit(1);
+      }
+    }
     return;
   }
   const runtimeProfiles = resolveBenchRunProfiles(parsed);
@@ -5101,7 +5174,9 @@ async function runBenchPublished(parsed) {
     const result = await runBenchViaPackage(
       parsed,
       benchmarkId,
-      runtimeProfile
+      runtimeProfile,
+      void 0,
+      taskSelector
     );
     if (!result.ok) {
       console.error(
@@ -5157,14 +5232,15 @@ function createDryRunDatasetValidationAdapter(benchmarkId) {
     }
   };
 }
-function buildPublishedBenchmarkOptions(benchmarkId, args) {
+function buildPublishedBenchmarkOptions(benchmarkId, args, taskSelector) {
   const trialLimitOptions = args.publishedTrialLimit !== void 0 ? { trialLimit: args.publishedTrialLimit } : void 0;
   const trialConcurrencyOptions = args.publishedTrialConcurrency !== void 0 ? { trialConcurrency: args.publishedTrialConcurrency } : void 0;
   if (benchmarkId === "locomo") {
     return {
       ...trialLimitOptions ?? {},
       ...trialConcurrencyOptions ?? {},
-      replayExtractionMode: "skip"
+      replayExtractionMode: "skip",
+      ...taskSelector ? { taskSelector } : {}
     };
   }
   if (benchmarkId === "ama-bench") {
@@ -5181,6 +5257,47 @@ function buildPublishedBenchmarkOptions(benchmarkId, args) {
   }
   return void 0;
 }
+function loadPinnedLoCoMoTaskSelector(parsed) {
+  if (!parsed.taskIdsFile) return void 0;
+  if (!parsed.expectedTaskIdListSha256) {
+    throw new Error(
+      "--task-ids-file requires --expected-task-id-list-sha256."
+    );
+  }
+  let decoded;
+  try {
+    decoded = JSON.parse(fs7.readFileSync(parsed.taskIdsFile, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Unable to read --task-ids-file ${parsed.taskIdsFile}: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+  if (!Array.isArray(decoded) || decoded.length === 0) {
+    throw new Error("--task-ids-file JSON must be a non-empty array of strings.");
+  }
+  if (decoded.some((taskId) => typeof taskId !== "string" || taskId.length === 0)) {
+    throw new Error("--task-ids-file JSON must contain only non-empty strings.");
+  }
+  const taskIds = decoded;
+  if (new Set(taskIds).size !== taskIds.length) {
+    throw new Error("--task-ids-file must not contain duplicate task IDs.");
+  }
+  return {
+    kind: "explicit-task-ids",
+    taskIds,
+    expectedSelectedTaskIdsSha256: parsed.expectedTaskIdListSha256
+  };
+}
+function redactBenchTaskIdsFilePath(argv) {
+  const redacted = [...argv];
+  for (let index = 0; index < redacted.length; index += 1) {
+    if (redacted[index] === "--task-ids-file" && index + 1 < redacted.length) {
+      redacted[index + 1] = "<task-ids-file>";
+      index += 1;
+    }
+  }
+  return redacted;
+}
 async function validateRunnerManagedPublishedDryRunDataset(benchModule, benchmarkId, mode, datasetDir, limit, seed, benchmarkOptions) {
   if (!benchModule.runBenchmark) {
     throw new Error(
@@ -5212,6 +5329,20 @@ async function validateRunnerManagedPublishedDryRunDataset(benchModule, benchmar
     throw error;
   }
 }
+async function validatePinnedLoCoMoPublishedDryRunSelector(benchModule, mode, datasetDir, limit, seed, benchmarkOptions, taskSelector) {
+  if (taskSelector === void 0) {
+    return;
+  }
+  await validateRunnerManagedPublishedDryRunDataset(
+    benchModule,
+    "locomo",
+    mode,
+    datasetDir,
+    limit,
+    seed,
+    benchmarkOptions
+  );
+}
 async function validateRunnerManagedPublishedDryRunDatasetForTest(benchmarkId, mode, datasetDir, limit, benchmarkOptions) {
   const benchModule = await tryLoadBenchModule();
   if (!benchModule) {
@@ -5300,7 +5431,7 @@ async function preflightLocalLabEndpointsIfNeeded(benchModule, plan) {
     );
   }
 }
-async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStatusPath) {
+async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStatusPath, taskSelector) {
   const loaded = await tryLoadBenchModule();
   if (!loaded) return { ok: false };
   assertBenchModuleFreshForDevelopment();
@@ -5350,7 +5481,11 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
   }
   const effectiveLimit = parsed.publishedLimit ?? (parsed.quick ? 1 : void 0);
   const effectiveSeed = parsed.publishedSeed;
-  let benchmarkOptions = buildPublishedBenchmarkOptions(benchmarkId, parsed);
+  let benchmarkOptions = buildPublishedBenchmarkOptions(
+    benchmarkId,
+    parsed,
+    taskSelector
+  );
   try {
     const amaBenchProtocol = buildAmaBenchProtocolOptions(
       benchModule,
@@ -5802,7 +5937,7 @@ async function writeBenchReproManifestForPackageRun(args) {
       datasetDirs: resolveBenchReproDatasetDirs(args.parsed, args.benchmarkIds),
       command: {
         cwd: process.cwd(),
-        argv: process.argv.slice(2),
+        argv: redactBenchTaskIdsFilePath(process.argv.slice(2)),
         env: process.env,
         envKeys: resolveBenchReproEnvKeys()
       },
@@ -10897,6 +11032,7 @@ async function cmdBench(rest) {
     );
     process.exit(1);
   }
+  const taskSelector = loadPinnedLoCoMoTaskSelector(parsed);
   const runtimeProfiles = resolveBenchRunProfiles(parsed);
   let selectedWorkItems = createBenchWorkItems(selectedBenchmarks, runtimeProfiles);
   const knownBenchmarkIds = await resolveKnownBenchmarkIds();
@@ -10981,7 +11117,8 @@ async function cmdBench(rest) {
           parsed,
           benchmarkId,
           runtimeProfile,
-          benchStatusPath
+          benchStatusPath,
+          taskSelector
         );
         if (handledByPackage.ok) {
           if (handledByPackage.writtenPath) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/cli",
-  "version": "9.7.0",
+  "version": "9.7.1",
   "description": "CLI for Remnic memory — init, query, doctor, daemon management",
   "type": "module",
   "main": "dist/index.js",
@@ -26,23 +26,23 @@
   },
   "dependencies": {
     "yaml": "^2.4.2",
-    "@remnic/plugin-pi": "^9.7.0",
-    "@remnic/core": "^9.7.0",
-    "@remnic/server": "^9.7.0"
+    "@remnic/server": "^9.7.1",
+    "@remnic/core": "^9.7.1",
+    "@remnic/plugin-pi": "^9.7.1"
   },
   "peerDependencies": {
-    "@remnic/bench": "^9.7.0",
-    "@remnic/export-weclone": "^9.7.0",
-    "@remnic/import-weclone": "^9.7.0",
-    "@remnic/import-chatgpt": "^9.7.0",
-    "@remnic/import-claude": "^9.7.0",
-    "@remnic/import-gemini": "^9.7.0",
-    "@remnic/import-lossless-claw": "^9.7.0",
-    "@remnic/import-mem0": "^9.7.0",
-    "@remnic/import-supermemory": "^9.7.0",
-    "@remnic/connector-limitless": "^9.7.0",
-    "@remnic/connector-bee": "^9.7.0",
-    "@remnic/connector-omi": "^9.7.0"
+    "@remnic/bench": "^9.7.1",
+    "@remnic/export-weclone": "^9.7.1",
+    "@remnic/import-weclone": "^9.7.1",
+    "@remnic/import-chatgpt": "^9.7.1",
+    "@remnic/import-claude": "^9.7.1",
+    "@remnic/import-gemini": "^9.7.1",
+    "@remnic/import-lossless-claw": "^9.7.1",
+    "@remnic/import-mem0": "^9.7.1",
+    "@remnic/import-supermemory": "^9.7.1",
+    "@remnic/connector-limitless": "^9.7.1",
+    "@remnic/connector-bee": "^9.7.1",
+    "@remnic/connector-omi": "^9.7.1"
   },
   "peerDependenciesMeta": {
     "@remnic/bench": {
@@ -85,18 +85,18 @@
   "devDependencies": {
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
-    "@remnic/export-weclone": "9.7.0",
-    "@remnic/bench": "9.7.0",
-    "@remnic/import-weclone": "9.7.0",
-    "@remnic/import-chatgpt": "9.7.0",
-    "@remnic/import-claude": "9.7.0",
-    "@remnic/import-gemini": "9.7.0",
-    "@remnic/import-lossless-claw": "9.7.0",
-    "@remnic/import-mem0": "9.7.0",
-    "@remnic/import-supermemory": "9.7.0",
-    "@remnic/connector-bee": "9.7.0",
-    "@remnic/connector-limitless": "9.7.0",
-    "@remnic/connector-omi": "9.7.0"
+    "@remnic/bench": "9.7.1",
+    "@remnic/export-weclone": "9.7.1",
+    "@remnic/import-weclone": "9.7.1",
+    "@remnic/import-chatgpt": "9.7.1",
+    "@remnic/import-claude": "9.7.1",
+    "@remnic/import-gemini": "9.7.1",
+    "@remnic/import-mem0": "9.7.1",
+    "@remnic/import-lossless-claw": "9.7.1",
+    "@remnic/connector-bee": "9.7.1",
+    "@remnic/connector-limitless": "9.7.1",
+    "@remnic/import-supermemory": "9.7.1",
+    "@remnic/connector-omi": "9.7.1"
   },
   "license": "MIT",
   "repository": {