@remnic/cli 9.6.17 → 9.6.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +256 -14
  2. package/package.json +28 -28
package/dist/index.js CHANGED
@@ -654,6 +654,70 @@ function parseCodexReasoningEffort(raw, flag) {
654
654
  }
655
655
  return raw;
656
656
  }
657
+ var MCP_TOOL_OPERATIONS = /* @__PURE__ */ new Set(["store", "recall", "correct", "reset"]);
658
+ var MCP_ARGUMENT_SEMANTICS = /* @__PURE__ */ new Set([
659
+ "namespace",
660
+ "sessionId",
661
+ "content",
662
+ "role",
663
+ "timestamp",
664
+ "query",
665
+ "limit"
666
+ ]);
667
+ function parseMcpToolMapping(value) {
668
+ if (!isPlainObject(value)) throw new Error("expected a JSON object");
669
+ for (const operation of Object.keys(value).sort()) {
670
+ if (!MCP_TOOL_OPERATIONS.has(operation)) {
671
+ throw new Error(`unknown operation ${operation}`);
672
+ }
673
+ const entry = value[operation];
674
+ if (typeof entry === "string") {
675
+ if (entry.trim().length === 0) throw new Error(`${operation} tool name must not be empty`);
676
+ continue;
677
+ }
678
+ if (!isPlainObject(entry)) {
679
+ throw new Error(`${operation} must be a tool-name string or mapping object`);
680
+ }
681
+ for (const field of Object.keys(entry).sort()) {
682
+ if (field !== "name" && field !== "arguments" && field !== "resultPath") {
683
+ throw new Error(`${operation} contains unknown field ${field}`);
684
+ }
685
+ }
686
+ if (typeof entry.name !== "string" || entry.name.trim().length === 0) {
687
+ throw new Error(`${operation}.name must be a non-empty string`);
688
+ }
689
+ if (entry.resultPath !== void 0) {
690
+ if (typeof entry.resultPath !== "string" || !isSafeMcpResultPath(entry.resultPath)) {
691
+ throw new Error(`${operation}.resultPath must be a non-empty safe dot path`);
692
+ }
693
+ }
694
+ if (entry.arguments !== void 0) {
695
+ if (!isPlainObject(entry.arguments)) {
696
+ throw new Error(`${operation}.arguments must be an object`);
697
+ }
698
+ for (const semantic of Object.keys(entry.arguments).sort()) {
699
+ if (!MCP_ARGUMENT_SEMANTICS.has(semantic)) {
700
+ throw new Error(`${operation}.arguments contains unknown semantic ${semantic}`);
701
+ }
702
+ const argumentName = entry.arguments[semantic];
703
+ if (typeof argumentName !== "string" || argumentName.trim().length === 0) {
704
+ throw new Error(`${operation}.arguments.${semantic} must be a non-empty string`);
705
+ }
706
+ }
707
+ }
708
+ }
709
+ return value;
710
+ }
711
+ function isPlainObject(value) {
712
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
713
+ const prototype = Object.getPrototypeOf(value);
714
+ return prototype === Object.prototype || prototype === null;
715
+ }
716
+ function isSafeMcpResultPath(value) {
717
+ return value.split(".").every(
718
+ (segment) => segment.length > 0 && segment !== "__proto__" && segment !== "prototype" && segment !== "constructor" && /^[A-Za-z0-9_-]+$/.test(segment)
719
+ );
720
+ }
657
721
  function readBenchOptionValue(argv, flag) {
658
722
  const index = argv.indexOf(flag);
659
723
  if (index === -1) {
@@ -666,6 +730,11 @@ function readBenchOptionValue(argv, flag) {
666
730
  return value;
667
731
  }
668
732
  var BENCH_VALUE_FLAGS = Object.freeze([
733
+ "--adapter",
734
+ "--mcp-command",
735
+ "--mcp-args",
736
+ "--mcp-url",
737
+ "--mcp-tool-map",
669
738
  "--dataset-dir",
670
739
  "--benchmark",
671
740
  "--results-dir",
@@ -725,6 +794,7 @@ var BENCH_VALUE_FLAGS = Object.freeze([
725
794
  "--memcorrect-adapter"
726
795
  ]);
727
796
  var BENCH_BOOLEAN_FLAGS = Object.freeze([
797
+ "--mcp-demo",
728
798
  "--quick",
729
799
  "--all",
730
800
  "--json",
@@ -748,6 +818,11 @@ function isBenchBooleanFlag(arg) {
748
818
  return BENCH_BOOLEAN_FLAG_SET.has(arg);
749
819
  }
750
820
  var RUN_VALUE_FLAGS = Object.freeze([
821
+ "--adapter",
822
+ "--mcp-command",
823
+ "--mcp-args",
824
+ "--mcp-url",
825
+ "--mcp-tool-map",
751
826
  "--dataset-dir",
752
827
  "--results-dir",
753
828
  "--runtime-profile",
@@ -799,6 +874,7 @@ var RUN_VALUE_FLAGS = Object.freeze([
799
874
  "--memcorrect-adapter"
800
875
  ]);
801
876
  var RUN_BOOLEAN_FLAGS = Object.freeze([
877
+ "--mcp-demo",
802
878
  "--quick",
803
879
  "--all",
804
880
  "--json",
@@ -992,6 +1068,12 @@ function parseBenchArgs(argv) {
992
1068
  const benchmarkArgs = action === "baseline" || action === "datasets" || action === "providers" || action === "runs" ? args.slice(1) : args;
993
1069
  const benchmarks = collectBenchmarks(benchmarkArgs);
994
1070
  const datasetDir = readBenchOptionValue(args, "--dataset-dir") ?? readBenchOptionValue(args, "--dataset");
1071
+ const adapterRaw = readBenchOptionValue(args, "--adapter");
1072
+ const mcpCommand = readBenchOptionValue(args, "--mcp-command");
1073
+ const mcpArgsRaw = readBenchOptionValue(args, "--mcp-args");
1074
+ const mcpUrl = readBenchOptionValue(args, "--mcp-url");
1075
+ const mcpToolMapRaw = readBenchOptionValue(args, "--mcp-tool-map");
1076
+ const mcpDemo = args.includes("--mcp-demo");
995
1077
  const resultsDir = readBenchOptionValue(args, "--results-dir");
996
1078
  const baselinesDir = readBenchOptionValue(args, "--baselines-dir");
997
1079
  const runtimeProfileRaw = readBenchOptionValue(args, "--runtime-profile");
@@ -1048,6 +1130,63 @@ function parseBenchArgs(argv) {
1048
1130
  const amaBenchCrossJudgeModel = readBenchOptionValue(args, "--ama-bench-cross-judge-model");
1049
1131
  const amaBenchCrossJudgeBaseUrl = readBenchOptionValue(args, "--ama-bench-cross-judge-base-url");
1050
1132
  const amaBenchCrossJudgeApiKey = readBenchOptionValue(args, "--ama-bench-cross-judge-api-key");
1133
+ let adapter;
1134
+ if (adapterRaw !== void 0) {
1135
+ if (adapterRaw !== "remnic" && adapterRaw !== "mcp") {
1136
+ throw new Error('ERROR: --adapter must be "remnic" or "mcp".');
1137
+ }
1138
+ adapter = adapterRaw;
1139
+ }
1140
+ const hasMcpOptions = Boolean(
1141
+ mcpCommand || mcpArgsRaw || mcpUrl || mcpToolMapRaw || mcpDemo
1142
+ );
1143
+ if (hasMcpOptions && adapter !== "mcp") {
1144
+ throw new Error("ERROR: MCP options require --adapter mcp.");
1145
+ }
1146
+ if (adapter === "mcp") {
1147
+ const transportCount = Number(Boolean(mcpCommand)) + Number(Boolean(mcpUrl)) + Number(mcpDemo);
1148
+ if (transportCount !== 1) {
1149
+ throw new Error(
1150
+ "ERROR: --adapter mcp requires exactly one of --mcp-command, --mcp-url, or --mcp-demo."
1151
+ );
1152
+ }
1153
+ }
1154
+ if (mcpArgsRaw && !mcpCommand) {
1155
+ throw new Error("ERROR: --mcp-args requires --mcp-command.");
1156
+ }
1157
+ if (mcpUrl) {
1158
+ let parsedUrl;
1159
+ try {
1160
+ parsedUrl = new URL(mcpUrl);
1161
+ } catch {
1162
+ throw new Error("ERROR: --mcp-url must be a valid HTTP(S) URL.");
1163
+ }
1164
+ if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
1165
+ throw new Error("ERROR: --mcp-url must use http or https.");
1166
+ }
1167
+ }
1168
+ let mcpArgs;
1169
+ if (mcpArgsRaw) {
1170
+ try {
1171
+ const parsed = JSON.parse(mcpArgsRaw);
1172
+ if (!Array.isArray(parsed) || !parsed.every((value) => typeof value === "string")) {
1173
+ throw new Error("shape");
1174
+ }
1175
+ mcpArgs = parsed;
1176
+ } catch {
1177
+ throw new Error('ERROR: --mcp-args must be a JSON array of strings, for example ["server.js"].');
1178
+ }
1179
+ }
1180
+ let mcpToolMap;
1181
+ if (mcpToolMapRaw) {
1182
+ try {
1183
+ const parsed = JSON.parse(mcpToolMapRaw);
1184
+ mcpToolMap = parseMcpToolMapping(parsed);
1185
+ } catch (error) {
1186
+ const detail = error instanceof Error ? error.message : String(error);
1187
+ throw new Error(`ERROR: --mcp-tool-map is invalid: ${detail}`);
1188
+ }
1189
+ }
1051
1190
  const amaBenchCrossJudgeCodexReasoningEffortRaw = readBenchOptionValue(
1052
1191
  args,
1053
1192
  "--ama-bench-cross-judge-codex-reasoning-effort"
@@ -1411,6 +1550,12 @@ function parseBenchArgs(argv) {
1411
1550
  all: args.includes("--all"),
1412
1551
  json: args.includes("--json"),
1413
1552
  detail: args.includes("--detail"),
1553
+ adapter,
1554
+ mcpCommand,
1555
+ mcpArgs,
1556
+ mcpUrl,
1557
+ mcpToolMap,
1558
+ mcpDemo,
1414
1559
  datasetDir: datasetDir ? path4.resolve(expandTilde(datasetDir)) : void 0,
1415
1560
  resultsDir: resultsDir ? path4.resolve(expandTilde(resultsDir)) : void 0,
1416
1561
  baselinesDir: baselinesDir ? path4.resolve(expandTilde(baselinesDir)) : void 0,
@@ -3317,6 +3462,14 @@ Commands:
3317
3462
  Options:
3318
3463
  --quick Run a lightweight quick pass (maps to --lightweight --limit 1)
3319
3464
  --all Run every published benchmark
3465
+ --adapter <remnic|mcp> Memory backend adapter (default: remnic)
3466
+ --mcp-demo Use the packaged keyless stdio MCP demo server
3467
+ --mcp-command <command> Spawn an MCP stdio server
3468
+ --mcp-args <json> JSON string array passed to --mcp-command
3469
+ --mcp-url <url> Connect to a Streamable HTTP MCP server
3470
+ REMNIC_BENCH_MCP_BEARER_TOKEN
3471
+ Optional bearer token for --mcp-url (environment only)
3472
+ --mcp-tool-map <json> Explicit store/recall/correct/reset tool mapping
3320
3473
  --runtime-profile <baseline|real|openclaw-chain|local-lab>
3321
3474
  Choose the benchmark runtime profile
3322
3475
  --matrix <profiles> Run a benchmark across a comma-separated profile matrix
@@ -3429,7 +3582,7 @@ function buildBenchRuntimeProfileRequest(parsed, runtimeProfile) {
3429
3582
  judgeProvider: parsed.judgeProvider,
3430
3583
  judgeModel: parsed.judgeModel,
3431
3584
  judgeBaseUrl: parsed.judgeBaseUrl,
3432
- judgeApiKey: parsed.judgeApiKey,
3585
+ judgeApiKey: parsed.judgeApiKey ?? (parsed.judgeProvider === "openai" ? process.env.OPENAI_API_KEY : void 0),
3433
3586
  judgeCodexReasoningEffort: parsed.judgeCodexReasoningEffort,
3434
3587
  internalProvider: parsed.internalProvider,
3435
3588
  internalModel: parsed.internalModel,
@@ -4317,6 +4470,7 @@ async function exportBenchPackageResult(parsed) {
4317
4470
  const {
4318
4471
  resolveBenchmarkResultReference,
4319
4472
  loadBenchmarkResult,
4473
+ loadBenchmarkReportCardProvenance,
4320
4474
  renderBenchmarkResultExport
4321
4475
  } = await loadBenchModule();
4322
4476
  const reference = parsed.benchmarks[0];
@@ -4326,7 +4480,10 @@ async function exportBenchPackageResult(parsed) {
4326
4480
  process.exit(1);
4327
4481
  }
4328
4482
  const result = await loadBenchmarkResult(summary.path);
4329
- const rendered = renderBenchmarkResultExport(result, parsed.format);
4483
+ const reportCardProvenance = parsed.format === "html" ? await loadBenchmarkReportCardProvenance(path11.dirname(summary.path), result.meta.id) : void 0;
4484
+ const rendered = renderBenchmarkResultExport(result, parsed.format, {
4485
+ ...reportCardProvenance ? { reportCardProvenance } : {}
4486
+ });
4330
4487
  if (parsed.output) {
4331
4488
  fs7.mkdirSync(path11.dirname(parsed.output), { recursive: true });
4332
4489
  fs7.writeFileSync(parsed.output, rendered);
@@ -4519,6 +4676,8 @@ async function calibrateBenchJudges(parsed, rawArgs) {
4519
4676
  const resultsDir = expandTilde(
4520
4677
  parsed.resultsDir ?? path11.join(resolveHomeDir(), ".remnic", "bench", "results")
4521
4678
  );
4679
+ const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
4680
+ const previousCalibration = await bench.loadJudgeCalibrationState?.(benchmarkId, calibrationDir);
4522
4681
  const stored = await bench.listBenchmarkResults(resultsDir);
4523
4682
  const allForBenchmark = stored.filter((entry) => entry.benchmark === benchmarkId);
4524
4683
  const candidates = allForBenchmark.filter((entry) => entry.mode === "full").sort((a, b) => {
@@ -4527,8 +4686,15 @@ async function calibrateBenchJudges(parsed, rawArgs) {
4527
4686
  }
4528
4687
  return a.id < b.id ? 1 : a.id > b.id ? -1 : 0;
4529
4688
  });
4530
- const latest = candidates[0];
4689
+ const pinnedSourceId = previousCalibration?.sourceResultId;
4690
+ const latest = pinnedSourceId ? candidates.find((entry) => entry.id === pinnedSourceId) : candidates[0];
4531
4691
  if (!latest) {
4692
+ if (pinnedSourceId) {
4693
+ console.error(
4694
+ `ERROR: pinned calibration source ${pinnedSourceId} for "${benchmarkId}" is no longer present in ${resultsDir}. Restore that stored result or remove the calibration state intentionally before selecting a new answer set.`
4695
+ );
4696
+ process.exit(1);
4697
+ }
4532
4698
  const quickCount = allForBenchmark.filter((entry) => entry.mode === "quick").length;
4533
4699
  console.error(
4534
4700
  quickCount > 0 ? `ERROR: no full stored results for "${benchmarkId}" in ${resultsDir} (found ${quickCount} quick run(s); a 1-task quick sample cannot calibrate the judge). Run a full benchmark first (remnic bench run ${benchmarkId}).` : `ERROR: no stored benchmark results for "${benchmarkId}" in ${resultsDir}. Run the benchmark first (remnic bench run ${benchmarkId}) so cached answers exist to calibrate against.`
@@ -4536,7 +4702,7 @@ async function calibrateBenchJudges(parsed, rawArgs) {
4536
4702
  process.exit(1);
4537
4703
  }
4538
4704
  let loaded = await bench.loadBenchmarkResult(latest.path);
4539
- if (loaded.meta.status === "partial") {
4705
+ if (!pinnedSourceId && loaded.meta.status === "partial") {
4540
4706
  for (const candidate of candidates.slice(1)) {
4541
4707
  const candidateResult = await bench.loadBenchmarkResult(candidate.path);
4542
4708
  if (candidateResult.meta.status !== "partial") {
@@ -4545,6 +4711,12 @@ async function calibrateBenchJudges(parsed, rawArgs) {
4545
4711
  }
4546
4712
  }
4547
4713
  }
4714
+ if (pinnedSourceId && loaded.meta.status === "partial") {
4715
+ console.error(
4716
+ `ERROR: pinned calibration source ${pinnedSourceId} is partial. Restore a complete copy or remove the calibration state intentionally before selecting a new answer set.`
4717
+ );
4718
+ process.exit(1);
4719
+ }
4548
4720
  const uniqueTaskIds = new Set(loaded.results.tasks.map((task) => task.taskId));
4549
4721
  const sourceTaskCount = uniqueTaskIds.size;
4550
4722
  if (sourceTaskCount < bench.MIN_CALIBRATION_SOURCE_TASKS) {
@@ -4573,16 +4745,22 @@ async function calibrateBenchJudges(parsed, rawArgs) {
4573
4745
  benchmarkId,
4574
4746
  localJudge,
4575
4747
  frontierJudge,
4576
- answers
4748
+ answers,
4749
+ ...previousCalibration?.sliceQuestionIds ? { pinnedQuestionIds: previousCalibration.sliceQuestionIds } : {},
4750
+ ...previousCalibration?.answerSetHash ? { expectedAnswerSetHash: previousCalibration.answerSetHash } : {}
4577
4751
  });
4578
- const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
4579
4752
  const calibrationIdentities = {
4580
4753
  localJudgeProvider: String(localJudgeConfig.provider),
4581
4754
  localJudgeModel: String(localJudgeConfig.model),
4582
4755
  frontierJudgeProvider: parsed.judgeProvider,
4583
4756
  frontierJudgeModel: parsed.judgeModel
4584
4757
  };
4585
- const statePath = await bench.writeJudgeCalibrationState(result, calibrationDir, calibrationIdentities);
4758
+ const statePath = await bench.writeJudgeCalibrationState(
4759
+ result,
4760
+ calibrationDir,
4761
+ calibrationIdentities,
4762
+ { sourceResultId: loaded.meta.id }
4763
+ );
4586
4764
  const persisted = await bench.loadJudgeCalibrationState(benchmarkId, calibrationDir);
4587
4765
  if (!persisted || persisted.kappa !== result.kappa || persisted.warning !== result.warning) {
4588
4766
  console.error(
@@ -4601,6 +4779,10 @@ async function calibrateBenchJudges(parsed, rawArgs) {
4601
4779
  sampleSize: result.sampleSize,
4602
4780
  threshold: result.threshold,
4603
4781
  warning: result.warning,
4782
+ confidenceInterval: result.confidenceInterval,
4783
+ bootstrapSamples: result.bootstrapSamples,
4784
+ answerSetHash: result.answerSetHash,
4785
+ sourceResultId: loaded.meta.id,
4604
4786
  categories: result.categories,
4605
4787
  statePath
4606
4788
  },
@@ -4613,6 +4795,10 @@ async function calibrateBenchJudges(parsed, rawArgs) {
4613
4795
  console.log(`Judge calibration: ${benchmarkId}`);
4614
4796
  console.log(` Cohen's kappa: ${result.kappa.toFixed(4)} (threshold ${result.threshold})`);
4615
4797
  console.log(` Sample size: ${result.sampleSize}`);
4798
+ console.log(
4799
+ ` ${(result.confidenceInterval.level * 100).toFixed(0)}% bootstrap CI: [${result.confidenceInterval.lower.toFixed(4)}, ${result.confidenceInterval.upper.toFixed(4)}] (${result.bootstrapSamples} paired resamples)`
4800
+ );
4801
+ console.log(` Pinned source: ${loaded.meta.id} (answer set sha256:${result.answerSetHash})`);
4616
4802
  console.log(` Observed agreement: ${result.observedAgreement.toFixed(4)}`);
4617
4803
  console.log(` Expected agreement: ${result.expectedAgreement.toFixed(4)}`);
4618
4804
  if (result.warning) {
@@ -5036,7 +5222,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
5036
5222
  }
5037
5223
  const effectiveLimit = parsed.publishedLimit ?? (parsed.quick ? 1 : void 0);
5038
5224
  const effectiveSeed = parsed.publishedSeed;
5039
- const benchmarkOptions = buildPublishedBenchmarkOptions(benchmarkId, parsed);
5225
+ let benchmarkOptions = buildPublishedBenchmarkOptions(benchmarkId, parsed);
5040
5226
  try {
5041
5227
  const amaBenchProtocol = buildAmaBenchProtocolOptions(
5042
5228
  benchModule,
@@ -5044,11 +5230,14 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
5044
5230
  benchmarkId,
5045
5231
  plan.runtime
5046
5232
  );
5047
- system = await plan.createAdapter({
5233
+ system = benchmarkId === "memcorrect-v1" && parsed.adapter === "mcp" ? await createPackageMcpMemCorrectAdapter(benchModule, parsed) : await plan.createAdapter({
5048
5234
  ...plan.runtime.adapterOptions,
5049
5235
  ...benchmarkId === "locomo" ? { replayExtractionMode: "skip" } : {},
5050
5236
  ...amaBenchProtocol.primaryJudge ? { judge: amaBenchProtocol.primaryJudge } : {}
5051
5237
  });
5238
+ if (benchmarkId === "memcorrect-v1" && parsed.adapter === "mcp") {
5239
+ benchmarkOptions = { ...benchmarkOptions ?? {}, adapter: system };
5240
+ }
5052
5241
  const result = await benchModule.runBenchmark(benchmarkId, {
5053
5242
  mode: parsed.quick ? "quick" : "full",
5054
5243
  datasetDir,
@@ -5069,6 +5258,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
5069
5258
  ...amaBenchProtocol.judgeProtocol ? { amaBenchJudgeProtocol: amaBenchProtocol.judgeProtocol } : {},
5070
5259
  ...amaBenchProtocol.crossJudge ? { amaBenchCrossJudge: amaBenchProtocol.crossJudge } : {},
5071
5260
  ...amaBenchProtocol.crossJudgeProvider ? { amaBenchCrossJudgeProvider: amaBenchProtocol.crossJudgeProvider } : {},
5261
+ ...plan.runtime.adapterOptions.judge ? { memCorrectJudge: plan.runtime.adapterOptions.judge } : {},
5072
5262
  system,
5073
5263
  onTaskComplete: (task, completed, total) => {
5074
5264
  partialTasks.push(task);
@@ -5155,7 +5345,12 @@ async function attachPersistedJudgeCalibration(benchModule, benchmarkId, result)
5155
5345
  kappa: state.kappa,
5156
5346
  sampleSize: state.sampleSize,
5157
5347
  threshold: state.threshold,
5158
- warning: state.warning
5348
+ warning: state.warning,
5349
+ ...state.confidenceInterval ? { confidenceInterval: state.confidenceInterval } : {},
5350
+ ...state.bootstrapSamples ? { bootstrapSamples: state.bootstrapSamples } : {},
5351
+ ...state.answerSetHash ? { answerSetHash: state.answerSetHash } : {},
5352
+ ...state.sourceResultId ? { sourceResultId: state.sourceResultId } : {},
5353
+ ...state.sliceQuestionIds ? { sliceQuestionIds: state.sliceQuestionIds } : {}
5159
5354
  }
5160
5355
  };
5161
5356
  }
@@ -5342,6 +5537,7 @@ var BENCH_REPRO_ENV_KEYS = [
5342
5537
  "REMNIC_BENCH_IDS",
5343
5538
  "REMNIC_BENCH_LIMIT",
5344
5539
  "REMNIC_BENCH_MODE",
5540
+ "REMNIC_BENCH_MCP_BEARER_TOKEN",
5345
5541
  "REMNIC_BENCH_PHASE_TIMEOUT_MS",
5346
5542
  "REMNIC_BENCH_CODEX_CLI_EXECUTABLE",
5347
5543
  "REMNIC_BENCH_CODEX_CLI_TRANSPORT",
@@ -5460,11 +5656,56 @@ function resolveExistingBenchOpenclawConfigPath(cliPath) {
5460
5656
  function resolveBenchRunProfiles(parsed) {
5461
5657
  return parsed.matrixProfiles ?? [parsed.runtimeProfile ?? "baseline"];
5462
5658
  }
5463
- function resolvePackageBenchAdapterMode(quick, runtimeProfile) {
5659
+ function resolvePackageBenchAdapterMode(parsed, quick, runtimeProfile) {
5660
+ if (parsed.adapter === "mcp") return "mcp";
5464
5661
  return quick && runtimeProfile === "baseline" ? "lightweight" : "direct";
5465
5662
  }
5466
- function resolvePackageBenchAdapterFactory(benchModule, quick, runtimeProfile) {
5467
- return resolvePackageBenchAdapterMode(quick, runtimeProfile) === "lightweight" ? benchModule.createLightweightAdapter : benchModule.createRemnicAdapter;
5663
+ function resolvePackageBenchAdapterFactory(benchModule, parsed, quick, runtimeProfile) {
5664
+ if (parsed.adapter === "mcp") {
5665
+ if (parsed.mcpDemo) {
5666
+ if (!benchModule.createMcpDemoMemoryAdapter) return void 0;
5667
+ return async () => benchModule.createMcpDemoMemoryAdapter({
5668
+ ...parsed.requestTimeout ? { timeoutMs: parsed.requestTimeout } : {}
5669
+ });
5670
+ }
5671
+ if (!benchModule.createMcpMemoryAdapter) return void 0;
5672
+ return async () => benchModule.createMcpMemoryAdapter({
5673
+ ...buildPackageMcpAdapterOptions(parsed)
5674
+ });
5675
+ }
5676
+ return resolvePackageBenchAdapterMode(parsed, quick, runtimeProfile) === "lightweight" ? benchModule.createLightweightAdapter : benchModule.createRemnicAdapter;
5677
+ }
5678
+ function buildPackageMcpAdapterOptions(parsed) {
5679
+ const transport = parsed.mcpCommand ? { type: "stdio", command: parsed.mcpCommand, args: parsed.mcpArgs } : {
5680
+ type: "http",
5681
+ url: parsed.mcpUrl,
5682
+ bearerToken: process.env.REMNIC_BENCH_MCP_BEARER_TOKEN
5683
+ };
5684
+ return {
5685
+ transport,
5686
+ ...parsed.mcpToolMap ? { tools: parsed.mcpToolMap } : {},
5687
+ ...parsed.requestTimeout ? { timeoutMs: parsed.requestTimeout } : {}
5688
+ };
5689
+ }
5690
+ async function createPackageMcpMemCorrectAdapter(benchModule, parsed) {
5691
+ if (parsed.mcpDemo) {
5692
+ if (!benchModule.createMcpDemoMemCorrectAdapter) {
5693
+ throw new Error(
5694
+ "Installed @remnic/bench does not export createMcpDemoMemCorrectAdapter()."
5695
+ );
5696
+ }
5697
+ return benchModule.createMcpDemoMemCorrectAdapter({
5698
+ ...parsed.requestTimeout ? { timeoutMs: parsed.requestTimeout } : {}
5699
+ });
5700
+ }
5701
+ if (!benchModule.createMcpMemCorrectAdapter) {
5702
+ throw new Error(
5703
+ "Installed @remnic/bench does not export createMcpMemCorrectAdapter()."
5704
+ );
5705
+ }
5706
+ return benchModule.createMcpMemCorrectAdapter(
5707
+ buildPackageMcpAdapterOptions(parsed)
5708
+ );
5468
5709
  }
5469
5710
  async function buildPackageBenchExecutionPlans(benchModule, parsed, runtimeProfiles) {
5470
5711
  const plans = [];
@@ -5476,6 +5717,7 @@ async function buildPackageBenchExecutionPlans(benchModule, parsed, runtimeProfi
5476
5717
  );
5477
5718
  const createAdapter = resolvePackageBenchAdapterFactory(
5478
5719
  benchModule,
5720
+ parsed,
5479
5721
  parsed.quick,
5480
5722
  runtime.profile
5481
5723
  );
@@ -5485,7 +5727,7 @@ async function buildPackageBenchExecutionPlans(benchModule, parsed, runtimeProfi
5485
5727
  plans.push({
5486
5728
  runtime,
5487
5729
  createAdapter,
5488
- adapterMode: resolvePackageBenchAdapterMode(parsed.quick, runtime.profile)
5730
+ adapterMode: resolvePackageBenchAdapterMode(parsed, parsed.quick, runtime.profile)
5489
5731
  });
5490
5732
  }
5491
5733
  return plans;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remnic/cli",
3
- "version": "9.6.17",
3
+ "version": "9.6.19",
4
4
  "description": "CLI for Remnic memory — init, query, doctor, daemon management",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -26,23 +26,23 @@
26
26
  },
27
27
  "dependencies": {
28
28
  "yaml": "^2.4.2",
29
- "@remnic/core": "^9.6.17",
30
- "@remnic/plugin-pi": "^9.6.17",
31
- "@remnic/server": "^9.6.17"
29
+ "@remnic/server": "^9.6.19",
30
+ "@remnic/plugin-pi": "^9.6.19",
31
+ "@remnic/core": "^9.6.19"
32
32
  },
33
33
  "peerDependencies": {
34
- "@remnic/bench": "^9.6.17",
35
- "@remnic/export-weclone": "^9.6.17",
36
- "@remnic/import-weclone": "^9.6.17",
37
- "@remnic/import-chatgpt": "^9.6.17",
38
- "@remnic/import-claude": "^9.6.17",
39
- "@remnic/import-gemini": "^9.6.17",
40
- "@remnic/import-lossless-claw": "^9.6.17",
41
- "@remnic/import-mem0": "^9.6.17",
42
- "@remnic/import-supermemory": "^9.6.17",
43
- "@remnic/connector-limitless": "^9.6.17",
44
- "@remnic/connector-bee": "^9.6.17",
45
- "@remnic/connector-omi": "^9.6.17"
34
+ "@remnic/bench": "^9.6.19",
35
+ "@remnic/export-weclone": "^9.6.19",
36
+ "@remnic/import-weclone": "^9.6.19",
37
+ "@remnic/import-chatgpt": "^9.6.19",
38
+ "@remnic/import-claude": "^9.6.19",
39
+ "@remnic/import-gemini": "^9.6.19",
40
+ "@remnic/import-lossless-claw": "^9.6.19",
41
+ "@remnic/import-mem0": "^9.6.19",
42
+ "@remnic/import-supermemory": "^9.6.19",
43
+ "@remnic/connector-limitless": "^9.6.19",
44
+ "@remnic/connector-bee": "^9.6.19",
45
+ "@remnic/connector-omi": "^9.6.19"
46
46
  },
47
47
  "peerDependenciesMeta": {
48
48
  "@remnic/bench": {
@@ -85,18 +85,18 @@
85
85
  "devDependencies": {
86
86
  "tsup": "^8.5.1",
87
87
  "typescript": "^5.9.3",
88
- "@remnic/bench": "9.6.17",
89
- "@remnic/export-weclone": "9.6.17",
90
- "@remnic/import-weclone": "9.6.17",
91
- "@remnic/import-gemini": "9.6.17",
92
- "@remnic/import-chatgpt": "9.6.17",
93
- "@remnic/import-claude": "9.6.17",
94
- "@remnic/import-supermemory": "9.6.17",
95
- "@remnic/connector-bee": "9.6.17",
96
- "@remnic/connector-limitless": "9.6.17",
97
- "@remnic/connector-omi": "9.6.17",
98
- "@remnic/import-mem0": "9.6.17",
99
- "@remnic/import-lossless-claw": "9.6.17"
88
+ "@remnic/bench": "9.6.19",
89
+ "@remnic/export-weclone": "9.6.19",
90
+ "@remnic/import-weclone": "9.6.19",
91
+ "@remnic/import-chatgpt": "9.6.19",
92
+ "@remnic/import-gemini": "9.6.19",
93
+ "@remnic/import-lossless-claw": "9.6.19",
94
+ "@remnic/import-claude": "9.6.19",
95
+ "@remnic/import-mem0": "9.6.19",
96
+ "@remnic/connector-omi": "9.6.19",
97
+ "@remnic/import-supermemory": "9.6.19",
98
+ "@remnic/connector-bee": "9.6.19",
99
+ "@remnic/connector-limitless": "9.6.19"
100
100
  },
101
101
  "license": "MIT",
102
102
  "repository": {