@remnic/cli 9.6.17 → 9.6.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +256 -14
- package/package.json +28 -28
package/dist/index.js
CHANGED
|
@@ -654,6 +654,70 @@ function parseCodexReasoningEffort(raw, flag) {
|
|
|
654
654
|
}
|
|
655
655
|
return raw;
|
|
656
656
|
}
|
|
657
|
+
var MCP_TOOL_OPERATIONS = /* @__PURE__ */ new Set(["store", "recall", "correct", "reset"]);
|
|
658
|
+
var MCP_ARGUMENT_SEMANTICS = /* @__PURE__ */ new Set([
|
|
659
|
+
"namespace",
|
|
660
|
+
"sessionId",
|
|
661
|
+
"content",
|
|
662
|
+
"role",
|
|
663
|
+
"timestamp",
|
|
664
|
+
"query",
|
|
665
|
+
"limit"
|
|
666
|
+
]);
|
|
667
|
+
function parseMcpToolMapping(value) {
|
|
668
|
+
if (!isPlainObject(value)) throw new Error("expected a JSON object");
|
|
669
|
+
for (const operation of Object.keys(value).sort()) {
|
|
670
|
+
if (!MCP_TOOL_OPERATIONS.has(operation)) {
|
|
671
|
+
throw new Error(`unknown operation ${operation}`);
|
|
672
|
+
}
|
|
673
|
+
const entry = value[operation];
|
|
674
|
+
if (typeof entry === "string") {
|
|
675
|
+
if (entry.trim().length === 0) throw new Error(`${operation} tool name must not be empty`);
|
|
676
|
+
continue;
|
|
677
|
+
}
|
|
678
|
+
if (!isPlainObject(entry)) {
|
|
679
|
+
throw new Error(`${operation} must be a tool-name string or mapping object`);
|
|
680
|
+
}
|
|
681
|
+
for (const field of Object.keys(entry).sort()) {
|
|
682
|
+
if (field !== "name" && field !== "arguments" && field !== "resultPath") {
|
|
683
|
+
throw new Error(`${operation} contains unknown field ${field}`);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
if (typeof entry.name !== "string" || entry.name.trim().length === 0) {
|
|
687
|
+
throw new Error(`${operation}.name must be a non-empty string`);
|
|
688
|
+
}
|
|
689
|
+
if (entry.resultPath !== void 0) {
|
|
690
|
+
if (typeof entry.resultPath !== "string" || !isSafeMcpResultPath(entry.resultPath)) {
|
|
691
|
+
throw new Error(`${operation}.resultPath must be a non-empty safe dot path`);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
if (entry.arguments !== void 0) {
|
|
695
|
+
if (!isPlainObject(entry.arguments)) {
|
|
696
|
+
throw new Error(`${operation}.arguments must be an object`);
|
|
697
|
+
}
|
|
698
|
+
for (const semantic of Object.keys(entry.arguments).sort()) {
|
|
699
|
+
if (!MCP_ARGUMENT_SEMANTICS.has(semantic)) {
|
|
700
|
+
throw new Error(`${operation}.arguments contains unknown semantic ${semantic}`);
|
|
701
|
+
}
|
|
702
|
+
const argumentName = entry.arguments[semantic];
|
|
703
|
+
if (typeof argumentName !== "string" || argumentName.trim().length === 0) {
|
|
704
|
+
throw new Error(`${operation}.arguments.${semantic} must be a non-empty string`);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
return value;
|
|
710
|
+
}
|
|
711
|
+
function isPlainObject(value) {
|
|
712
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
713
|
+
const prototype = Object.getPrototypeOf(value);
|
|
714
|
+
return prototype === Object.prototype || prototype === null;
|
|
715
|
+
}
|
|
716
|
+
function isSafeMcpResultPath(value) {
|
|
717
|
+
return value.split(".").every(
|
|
718
|
+
(segment) => segment.length > 0 && segment !== "__proto__" && segment !== "prototype" && segment !== "constructor" && /^[A-Za-z0-9_-]+$/.test(segment)
|
|
719
|
+
);
|
|
720
|
+
}
|
|
657
721
|
function readBenchOptionValue(argv, flag) {
|
|
658
722
|
const index = argv.indexOf(flag);
|
|
659
723
|
if (index === -1) {
|
|
@@ -666,6 +730,11 @@ function readBenchOptionValue(argv, flag) {
|
|
|
666
730
|
return value;
|
|
667
731
|
}
|
|
668
732
|
var BENCH_VALUE_FLAGS = Object.freeze([
|
|
733
|
+
"--adapter",
|
|
734
|
+
"--mcp-command",
|
|
735
|
+
"--mcp-args",
|
|
736
|
+
"--mcp-url",
|
|
737
|
+
"--mcp-tool-map",
|
|
669
738
|
"--dataset-dir",
|
|
670
739
|
"--benchmark",
|
|
671
740
|
"--results-dir",
|
|
@@ -725,6 +794,7 @@ var BENCH_VALUE_FLAGS = Object.freeze([
|
|
|
725
794
|
"--memcorrect-adapter"
|
|
726
795
|
]);
|
|
727
796
|
var BENCH_BOOLEAN_FLAGS = Object.freeze([
|
|
797
|
+
"--mcp-demo",
|
|
728
798
|
"--quick",
|
|
729
799
|
"--all",
|
|
730
800
|
"--json",
|
|
@@ -748,6 +818,11 @@ function isBenchBooleanFlag(arg) {
|
|
|
748
818
|
return BENCH_BOOLEAN_FLAG_SET.has(arg);
|
|
749
819
|
}
|
|
750
820
|
var RUN_VALUE_FLAGS = Object.freeze([
|
|
821
|
+
"--adapter",
|
|
822
|
+
"--mcp-command",
|
|
823
|
+
"--mcp-args",
|
|
824
|
+
"--mcp-url",
|
|
825
|
+
"--mcp-tool-map",
|
|
751
826
|
"--dataset-dir",
|
|
752
827
|
"--results-dir",
|
|
753
828
|
"--runtime-profile",
|
|
@@ -799,6 +874,7 @@ var RUN_VALUE_FLAGS = Object.freeze([
|
|
|
799
874
|
"--memcorrect-adapter"
|
|
800
875
|
]);
|
|
801
876
|
var RUN_BOOLEAN_FLAGS = Object.freeze([
|
|
877
|
+
"--mcp-demo",
|
|
802
878
|
"--quick",
|
|
803
879
|
"--all",
|
|
804
880
|
"--json",
|
|
@@ -992,6 +1068,12 @@ function parseBenchArgs(argv) {
|
|
|
992
1068
|
const benchmarkArgs = action === "baseline" || action === "datasets" || action === "providers" || action === "runs" ? args.slice(1) : args;
|
|
993
1069
|
const benchmarks = collectBenchmarks(benchmarkArgs);
|
|
994
1070
|
const datasetDir = readBenchOptionValue(args, "--dataset-dir") ?? readBenchOptionValue(args, "--dataset");
|
|
1071
|
+
const adapterRaw = readBenchOptionValue(args, "--adapter");
|
|
1072
|
+
const mcpCommand = readBenchOptionValue(args, "--mcp-command");
|
|
1073
|
+
const mcpArgsRaw = readBenchOptionValue(args, "--mcp-args");
|
|
1074
|
+
const mcpUrl = readBenchOptionValue(args, "--mcp-url");
|
|
1075
|
+
const mcpToolMapRaw = readBenchOptionValue(args, "--mcp-tool-map");
|
|
1076
|
+
const mcpDemo = args.includes("--mcp-demo");
|
|
995
1077
|
const resultsDir = readBenchOptionValue(args, "--results-dir");
|
|
996
1078
|
const baselinesDir = readBenchOptionValue(args, "--baselines-dir");
|
|
997
1079
|
const runtimeProfileRaw = readBenchOptionValue(args, "--runtime-profile");
|
|
@@ -1048,6 +1130,63 @@ function parseBenchArgs(argv) {
|
|
|
1048
1130
|
const amaBenchCrossJudgeModel = readBenchOptionValue(args, "--ama-bench-cross-judge-model");
|
|
1049
1131
|
const amaBenchCrossJudgeBaseUrl = readBenchOptionValue(args, "--ama-bench-cross-judge-base-url");
|
|
1050
1132
|
const amaBenchCrossJudgeApiKey = readBenchOptionValue(args, "--ama-bench-cross-judge-api-key");
|
|
1133
|
+
let adapter;
|
|
1134
|
+
if (adapterRaw !== void 0) {
|
|
1135
|
+
if (adapterRaw !== "remnic" && adapterRaw !== "mcp") {
|
|
1136
|
+
throw new Error('ERROR: --adapter must be "remnic" or "mcp".');
|
|
1137
|
+
}
|
|
1138
|
+
adapter = adapterRaw;
|
|
1139
|
+
}
|
|
1140
|
+
const hasMcpOptions = Boolean(
|
|
1141
|
+
mcpCommand || mcpArgsRaw || mcpUrl || mcpToolMapRaw || mcpDemo
|
|
1142
|
+
);
|
|
1143
|
+
if (hasMcpOptions && adapter !== "mcp") {
|
|
1144
|
+
throw new Error("ERROR: MCP options require --adapter mcp.");
|
|
1145
|
+
}
|
|
1146
|
+
if (adapter === "mcp") {
|
|
1147
|
+
const transportCount = Number(Boolean(mcpCommand)) + Number(Boolean(mcpUrl)) + Number(mcpDemo);
|
|
1148
|
+
if (transportCount !== 1) {
|
|
1149
|
+
throw new Error(
|
|
1150
|
+
"ERROR: --adapter mcp requires exactly one of --mcp-command, --mcp-url, or --mcp-demo."
|
|
1151
|
+
);
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
if (mcpArgsRaw && !mcpCommand) {
|
|
1155
|
+
throw new Error("ERROR: --mcp-args requires --mcp-command.");
|
|
1156
|
+
}
|
|
1157
|
+
if (mcpUrl) {
|
|
1158
|
+
let parsedUrl;
|
|
1159
|
+
try {
|
|
1160
|
+
parsedUrl = new URL(mcpUrl);
|
|
1161
|
+
} catch {
|
|
1162
|
+
throw new Error("ERROR: --mcp-url must be a valid HTTP(S) URL.");
|
|
1163
|
+
}
|
|
1164
|
+
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
|
|
1165
|
+
throw new Error("ERROR: --mcp-url must use http or https.");
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
let mcpArgs;
|
|
1169
|
+
if (mcpArgsRaw) {
|
|
1170
|
+
try {
|
|
1171
|
+
const parsed = JSON.parse(mcpArgsRaw);
|
|
1172
|
+
if (!Array.isArray(parsed) || !parsed.every((value) => typeof value === "string")) {
|
|
1173
|
+
throw new Error("shape");
|
|
1174
|
+
}
|
|
1175
|
+
mcpArgs = parsed;
|
|
1176
|
+
} catch {
|
|
1177
|
+
throw new Error('ERROR: --mcp-args must be a JSON array of strings, for example ["server.js"].');
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
let mcpToolMap;
|
|
1181
|
+
if (mcpToolMapRaw) {
|
|
1182
|
+
try {
|
|
1183
|
+
const parsed = JSON.parse(mcpToolMapRaw);
|
|
1184
|
+
mcpToolMap = parseMcpToolMapping(parsed);
|
|
1185
|
+
} catch (error) {
|
|
1186
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1187
|
+
throw new Error(`ERROR: --mcp-tool-map is invalid: ${detail}`);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1051
1190
|
const amaBenchCrossJudgeCodexReasoningEffortRaw = readBenchOptionValue(
|
|
1052
1191
|
args,
|
|
1053
1192
|
"--ama-bench-cross-judge-codex-reasoning-effort"
|
|
@@ -1411,6 +1550,12 @@ function parseBenchArgs(argv) {
|
|
|
1411
1550
|
all: args.includes("--all"),
|
|
1412
1551
|
json: args.includes("--json"),
|
|
1413
1552
|
detail: args.includes("--detail"),
|
|
1553
|
+
adapter,
|
|
1554
|
+
mcpCommand,
|
|
1555
|
+
mcpArgs,
|
|
1556
|
+
mcpUrl,
|
|
1557
|
+
mcpToolMap,
|
|
1558
|
+
mcpDemo,
|
|
1414
1559
|
datasetDir: datasetDir ? path4.resolve(expandTilde(datasetDir)) : void 0,
|
|
1415
1560
|
resultsDir: resultsDir ? path4.resolve(expandTilde(resultsDir)) : void 0,
|
|
1416
1561
|
baselinesDir: baselinesDir ? path4.resolve(expandTilde(baselinesDir)) : void 0,
|
|
@@ -3317,6 +3462,14 @@ Commands:
|
|
|
3317
3462
|
Options:
|
|
3318
3463
|
--quick Run a lightweight quick pass (maps to --lightweight --limit 1)
|
|
3319
3464
|
--all Run every published benchmark
|
|
3465
|
+
--adapter <remnic|mcp> Memory backend adapter (default: remnic)
|
|
3466
|
+
--mcp-demo Use the packaged keyless stdio MCP demo server
|
|
3467
|
+
--mcp-command <command> Spawn an MCP stdio server
|
|
3468
|
+
--mcp-args <json> JSON string array passed to --mcp-command
|
|
3469
|
+
--mcp-url <url> Connect to a Streamable HTTP MCP server
|
|
3470
|
+
REMNIC_BENCH_MCP_BEARER_TOKEN
|
|
3471
|
+
Optional bearer token for --mcp-url (environment only)
|
|
3472
|
+
--mcp-tool-map <json> Explicit store/recall/correct/reset tool mapping
|
|
3320
3473
|
--runtime-profile <baseline|real|openclaw-chain|local-lab>
|
|
3321
3474
|
Choose the benchmark runtime profile
|
|
3322
3475
|
--matrix <profiles> Run a benchmark across a comma-separated profile matrix
|
|
@@ -3429,7 +3582,7 @@ function buildBenchRuntimeProfileRequest(parsed, runtimeProfile) {
|
|
|
3429
3582
|
judgeProvider: parsed.judgeProvider,
|
|
3430
3583
|
judgeModel: parsed.judgeModel,
|
|
3431
3584
|
judgeBaseUrl: parsed.judgeBaseUrl,
|
|
3432
|
-
judgeApiKey: parsed.judgeApiKey,
|
|
3585
|
+
judgeApiKey: parsed.judgeApiKey ?? (parsed.judgeProvider === "openai" ? process.env.OPENAI_API_KEY : void 0),
|
|
3433
3586
|
judgeCodexReasoningEffort: parsed.judgeCodexReasoningEffort,
|
|
3434
3587
|
internalProvider: parsed.internalProvider,
|
|
3435
3588
|
internalModel: parsed.internalModel,
|
|
@@ -4317,6 +4470,7 @@ async function exportBenchPackageResult(parsed) {
|
|
|
4317
4470
|
const {
|
|
4318
4471
|
resolveBenchmarkResultReference,
|
|
4319
4472
|
loadBenchmarkResult,
|
|
4473
|
+
loadBenchmarkReportCardProvenance,
|
|
4320
4474
|
renderBenchmarkResultExport
|
|
4321
4475
|
} = await loadBenchModule();
|
|
4322
4476
|
const reference = parsed.benchmarks[0];
|
|
@@ -4326,7 +4480,10 @@ async function exportBenchPackageResult(parsed) {
|
|
|
4326
4480
|
process.exit(1);
|
|
4327
4481
|
}
|
|
4328
4482
|
const result = await loadBenchmarkResult(summary.path);
|
|
4329
|
-
const
|
|
4483
|
+
const reportCardProvenance = parsed.format === "html" ? await loadBenchmarkReportCardProvenance(path11.dirname(summary.path), result.meta.id) : void 0;
|
|
4484
|
+
const rendered = renderBenchmarkResultExport(result, parsed.format, {
|
|
4485
|
+
...reportCardProvenance ? { reportCardProvenance } : {}
|
|
4486
|
+
});
|
|
4330
4487
|
if (parsed.output) {
|
|
4331
4488
|
fs7.mkdirSync(path11.dirname(parsed.output), { recursive: true });
|
|
4332
4489
|
fs7.writeFileSync(parsed.output, rendered);
|
|
@@ -4519,6 +4676,8 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
4519
4676
|
const resultsDir = expandTilde(
|
|
4520
4677
|
parsed.resultsDir ?? path11.join(resolveHomeDir(), ".remnic", "bench", "results")
|
|
4521
4678
|
);
|
|
4679
|
+
const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
|
|
4680
|
+
const previousCalibration = await bench.loadJudgeCalibrationState?.(benchmarkId, calibrationDir);
|
|
4522
4681
|
const stored = await bench.listBenchmarkResults(resultsDir);
|
|
4523
4682
|
const allForBenchmark = stored.filter((entry) => entry.benchmark === benchmarkId);
|
|
4524
4683
|
const candidates = allForBenchmark.filter((entry) => entry.mode === "full").sort((a, b) => {
|
|
@@ -4527,8 +4686,15 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
4527
4686
|
}
|
|
4528
4687
|
return a.id < b.id ? 1 : a.id > b.id ? -1 : 0;
|
|
4529
4688
|
});
|
|
4530
|
-
const
|
|
4689
|
+
const pinnedSourceId = previousCalibration?.sourceResultId;
|
|
4690
|
+
const latest = pinnedSourceId ? candidates.find((entry) => entry.id === pinnedSourceId) : candidates[0];
|
|
4531
4691
|
if (!latest) {
|
|
4692
|
+
if (pinnedSourceId) {
|
|
4693
|
+
console.error(
|
|
4694
|
+
`ERROR: pinned calibration source ${pinnedSourceId} for "${benchmarkId}" is no longer present in ${resultsDir}. Restore that stored result or remove the calibration state intentionally before selecting a new answer set.`
|
|
4695
|
+
);
|
|
4696
|
+
process.exit(1);
|
|
4697
|
+
}
|
|
4532
4698
|
const quickCount = allForBenchmark.filter((entry) => entry.mode === "quick").length;
|
|
4533
4699
|
console.error(
|
|
4534
4700
|
quickCount > 0 ? `ERROR: no full stored results for "${benchmarkId}" in ${resultsDir} (found ${quickCount} quick run(s); a 1-task quick sample cannot calibrate the judge). Run a full benchmark first (remnic bench run ${benchmarkId}).` : `ERROR: no stored benchmark results for "${benchmarkId}" in ${resultsDir}. Run the benchmark first (remnic bench run ${benchmarkId}) so cached answers exist to calibrate against.`
|
|
@@ -4536,7 +4702,7 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
4536
4702
|
process.exit(1);
|
|
4537
4703
|
}
|
|
4538
4704
|
let loaded = await bench.loadBenchmarkResult(latest.path);
|
|
4539
|
-
if (loaded.meta.status === "partial") {
|
|
4705
|
+
if (!pinnedSourceId && loaded.meta.status === "partial") {
|
|
4540
4706
|
for (const candidate of candidates.slice(1)) {
|
|
4541
4707
|
const candidateResult = await bench.loadBenchmarkResult(candidate.path);
|
|
4542
4708
|
if (candidateResult.meta.status !== "partial") {
|
|
@@ -4545,6 +4711,12 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
4545
4711
|
}
|
|
4546
4712
|
}
|
|
4547
4713
|
}
|
|
4714
|
+
if (pinnedSourceId && loaded.meta.status === "partial") {
|
|
4715
|
+
console.error(
|
|
4716
|
+
`ERROR: pinned calibration source ${pinnedSourceId} is partial. Restore a complete copy or remove the calibration state intentionally before selecting a new answer set.`
|
|
4717
|
+
);
|
|
4718
|
+
process.exit(1);
|
|
4719
|
+
}
|
|
4548
4720
|
const uniqueTaskIds = new Set(loaded.results.tasks.map((task) => task.taskId));
|
|
4549
4721
|
const sourceTaskCount = uniqueTaskIds.size;
|
|
4550
4722
|
if (sourceTaskCount < bench.MIN_CALIBRATION_SOURCE_TASKS) {
|
|
@@ -4573,16 +4745,22 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
4573
4745
|
benchmarkId,
|
|
4574
4746
|
localJudge,
|
|
4575
4747
|
frontierJudge,
|
|
4576
|
-
answers
|
|
4748
|
+
answers,
|
|
4749
|
+
...previousCalibration?.sliceQuestionIds ? { pinnedQuestionIds: previousCalibration.sliceQuestionIds } : {},
|
|
4750
|
+
...previousCalibration?.answerSetHash ? { expectedAnswerSetHash: previousCalibration.answerSetHash } : {}
|
|
4577
4751
|
});
|
|
4578
|
-
const calibrationDir = path11.join(resolveHomeDir(), ".remnic", "bench", "calibration");
|
|
4579
4752
|
const calibrationIdentities = {
|
|
4580
4753
|
localJudgeProvider: String(localJudgeConfig.provider),
|
|
4581
4754
|
localJudgeModel: String(localJudgeConfig.model),
|
|
4582
4755
|
frontierJudgeProvider: parsed.judgeProvider,
|
|
4583
4756
|
frontierJudgeModel: parsed.judgeModel
|
|
4584
4757
|
};
|
|
4585
|
-
const statePath = await bench.writeJudgeCalibrationState(
|
|
4758
|
+
const statePath = await bench.writeJudgeCalibrationState(
|
|
4759
|
+
result,
|
|
4760
|
+
calibrationDir,
|
|
4761
|
+
calibrationIdentities,
|
|
4762
|
+
{ sourceResultId: loaded.meta.id }
|
|
4763
|
+
);
|
|
4586
4764
|
const persisted = await bench.loadJudgeCalibrationState(benchmarkId, calibrationDir);
|
|
4587
4765
|
if (!persisted || persisted.kappa !== result.kappa || persisted.warning !== result.warning) {
|
|
4588
4766
|
console.error(
|
|
@@ -4601,6 +4779,10 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
4601
4779
|
sampleSize: result.sampleSize,
|
|
4602
4780
|
threshold: result.threshold,
|
|
4603
4781
|
warning: result.warning,
|
|
4782
|
+
confidenceInterval: result.confidenceInterval,
|
|
4783
|
+
bootstrapSamples: result.bootstrapSamples,
|
|
4784
|
+
answerSetHash: result.answerSetHash,
|
|
4785
|
+
sourceResultId: loaded.meta.id,
|
|
4604
4786
|
categories: result.categories,
|
|
4605
4787
|
statePath
|
|
4606
4788
|
},
|
|
@@ -4613,6 +4795,10 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
4613
4795
|
console.log(`Judge calibration: ${benchmarkId}`);
|
|
4614
4796
|
console.log(` Cohen's kappa: ${result.kappa.toFixed(4)} (threshold ${result.threshold})`);
|
|
4615
4797
|
console.log(` Sample size: ${result.sampleSize}`);
|
|
4798
|
+
console.log(
|
|
4799
|
+
` ${(result.confidenceInterval.level * 100).toFixed(0)}% bootstrap CI: [${result.confidenceInterval.lower.toFixed(4)}, ${result.confidenceInterval.upper.toFixed(4)}] (${result.bootstrapSamples} paired resamples)`
|
|
4800
|
+
);
|
|
4801
|
+
console.log(` Pinned source: ${loaded.meta.id} (answer set sha256:${result.answerSetHash})`);
|
|
4616
4802
|
console.log(` Observed agreement: ${result.observedAgreement.toFixed(4)}`);
|
|
4617
4803
|
console.log(` Expected agreement: ${result.expectedAgreement.toFixed(4)}`);
|
|
4618
4804
|
if (result.warning) {
|
|
@@ -5036,7 +5222,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
|
|
|
5036
5222
|
}
|
|
5037
5223
|
const effectiveLimit = parsed.publishedLimit ?? (parsed.quick ? 1 : void 0);
|
|
5038
5224
|
const effectiveSeed = parsed.publishedSeed;
|
|
5039
|
-
|
|
5225
|
+
let benchmarkOptions = buildPublishedBenchmarkOptions(benchmarkId, parsed);
|
|
5040
5226
|
try {
|
|
5041
5227
|
const amaBenchProtocol = buildAmaBenchProtocolOptions(
|
|
5042
5228
|
benchModule,
|
|
@@ -5044,11 +5230,14 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
|
|
|
5044
5230
|
benchmarkId,
|
|
5045
5231
|
plan.runtime
|
|
5046
5232
|
);
|
|
5047
|
-
system = await plan.createAdapter({
|
|
5233
|
+
system = benchmarkId === "memcorrect-v1" && parsed.adapter === "mcp" ? await createPackageMcpMemCorrectAdapter(benchModule, parsed) : await plan.createAdapter({
|
|
5048
5234
|
...plan.runtime.adapterOptions,
|
|
5049
5235
|
...benchmarkId === "locomo" ? { replayExtractionMode: "skip" } : {},
|
|
5050
5236
|
...amaBenchProtocol.primaryJudge ? { judge: amaBenchProtocol.primaryJudge } : {}
|
|
5051
5237
|
});
|
|
5238
|
+
if (benchmarkId === "memcorrect-v1" && parsed.adapter === "mcp") {
|
|
5239
|
+
benchmarkOptions = { ...benchmarkOptions ?? {}, adapter: system };
|
|
5240
|
+
}
|
|
5052
5241
|
const result = await benchModule.runBenchmark(benchmarkId, {
|
|
5053
5242
|
mode: parsed.quick ? "quick" : "full",
|
|
5054
5243
|
datasetDir,
|
|
@@ -5069,6 +5258,7 @@ async function runBenchViaPackage(parsed, benchmarkId, runtimeProfile, benchStat
|
|
|
5069
5258
|
...amaBenchProtocol.judgeProtocol ? { amaBenchJudgeProtocol: amaBenchProtocol.judgeProtocol } : {},
|
|
5070
5259
|
...amaBenchProtocol.crossJudge ? { amaBenchCrossJudge: amaBenchProtocol.crossJudge } : {},
|
|
5071
5260
|
...amaBenchProtocol.crossJudgeProvider ? { amaBenchCrossJudgeProvider: amaBenchProtocol.crossJudgeProvider } : {},
|
|
5261
|
+
...plan.runtime.adapterOptions.judge ? { memCorrectJudge: plan.runtime.adapterOptions.judge } : {},
|
|
5072
5262
|
system,
|
|
5073
5263
|
onTaskComplete: (task, completed, total) => {
|
|
5074
5264
|
partialTasks.push(task);
|
|
@@ -5155,7 +5345,12 @@ async function attachPersistedJudgeCalibration(benchModule, benchmarkId, result)
|
|
|
5155
5345
|
kappa: state.kappa,
|
|
5156
5346
|
sampleSize: state.sampleSize,
|
|
5157
5347
|
threshold: state.threshold,
|
|
5158
|
-
warning: state.warning
|
|
5348
|
+
warning: state.warning,
|
|
5349
|
+
...state.confidenceInterval ? { confidenceInterval: state.confidenceInterval } : {},
|
|
5350
|
+
...state.bootstrapSamples ? { bootstrapSamples: state.bootstrapSamples } : {},
|
|
5351
|
+
...state.answerSetHash ? { answerSetHash: state.answerSetHash } : {},
|
|
5352
|
+
...state.sourceResultId ? { sourceResultId: state.sourceResultId } : {},
|
|
5353
|
+
...state.sliceQuestionIds ? { sliceQuestionIds: state.sliceQuestionIds } : {}
|
|
5159
5354
|
}
|
|
5160
5355
|
};
|
|
5161
5356
|
}
|
|
@@ -5342,6 +5537,7 @@ var BENCH_REPRO_ENV_KEYS = [
|
|
|
5342
5537
|
"REMNIC_BENCH_IDS",
|
|
5343
5538
|
"REMNIC_BENCH_LIMIT",
|
|
5344
5539
|
"REMNIC_BENCH_MODE",
|
|
5540
|
+
"REMNIC_BENCH_MCP_BEARER_TOKEN",
|
|
5345
5541
|
"REMNIC_BENCH_PHASE_TIMEOUT_MS",
|
|
5346
5542
|
"REMNIC_BENCH_CODEX_CLI_EXECUTABLE",
|
|
5347
5543
|
"REMNIC_BENCH_CODEX_CLI_TRANSPORT",
|
|
@@ -5460,11 +5656,56 @@ function resolveExistingBenchOpenclawConfigPath(cliPath) {
|
|
|
5460
5656
|
function resolveBenchRunProfiles(parsed) {
|
|
5461
5657
|
return parsed.matrixProfiles ?? [parsed.runtimeProfile ?? "baseline"];
|
|
5462
5658
|
}
|
|
5463
|
-
function resolvePackageBenchAdapterMode(quick, runtimeProfile) {
|
|
5659
|
+
function resolvePackageBenchAdapterMode(parsed, quick, runtimeProfile) {
|
|
5660
|
+
if (parsed.adapter === "mcp") return "mcp";
|
|
5464
5661
|
return quick && runtimeProfile === "baseline" ? "lightweight" : "direct";
|
|
5465
5662
|
}
|
|
5466
|
-
function resolvePackageBenchAdapterFactory(benchModule, quick, runtimeProfile) {
|
|
5467
|
-
|
|
5663
|
+
function resolvePackageBenchAdapterFactory(benchModule, parsed, quick, runtimeProfile) {
|
|
5664
|
+
if (parsed.adapter === "mcp") {
|
|
5665
|
+
if (parsed.mcpDemo) {
|
|
5666
|
+
if (!benchModule.createMcpDemoMemoryAdapter) return void 0;
|
|
5667
|
+
return async () => benchModule.createMcpDemoMemoryAdapter({
|
|
5668
|
+
...parsed.requestTimeout ? { timeoutMs: parsed.requestTimeout } : {}
|
|
5669
|
+
});
|
|
5670
|
+
}
|
|
5671
|
+
if (!benchModule.createMcpMemoryAdapter) return void 0;
|
|
5672
|
+
return async () => benchModule.createMcpMemoryAdapter({
|
|
5673
|
+
...buildPackageMcpAdapterOptions(parsed)
|
|
5674
|
+
});
|
|
5675
|
+
}
|
|
5676
|
+
return resolvePackageBenchAdapterMode(parsed, quick, runtimeProfile) === "lightweight" ? benchModule.createLightweightAdapter : benchModule.createRemnicAdapter;
|
|
5677
|
+
}
|
|
5678
|
+
function buildPackageMcpAdapterOptions(parsed) {
|
|
5679
|
+
const transport = parsed.mcpCommand ? { type: "stdio", command: parsed.mcpCommand, args: parsed.mcpArgs } : {
|
|
5680
|
+
type: "http",
|
|
5681
|
+
url: parsed.mcpUrl,
|
|
5682
|
+
bearerToken: process.env.REMNIC_BENCH_MCP_BEARER_TOKEN
|
|
5683
|
+
};
|
|
5684
|
+
return {
|
|
5685
|
+
transport,
|
|
5686
|
+
...parsed.mcpToolMap ? { tools: parsed.mcpToolMap } : {},
|
|
5687
|
+
...parsed.requestTimeout ? { timeoutMs: parsed.requestTimeout } : {}
|
|
5688
|
+
};
|
|
5689
|
+
}
|
|
5690
|
+
async function createPackageMcpMemCorrectAdapter(benchModule, parsed) {
|
|
5691
|
+
if (parsed.mcpDemo) {
|
|
5692
|
+
if (!benchModule.createMcpDemoMemCorrectAdapter) {
|
|
5693
|
+
throw new Error(
|
|
5694
|
+
"Installed @remnic/bench does not export createMcpDemoMemCorrectAdapter()."
|
|
5695
|
+
);
|
|
5696
|
+
}
|
|
5697
|
+
return benchModule.createMcpDemoMemCorrectAdapter({
|
|
5698
|
+
...parsed.requestTimeout ? { timeoutMs: parsed.requestTimeout } : {}
|
|
5699
|
+
});
|
|
5700
|
+
}
|
|
5701
|
+
if (!benchModule.createMcpMemCorrectAdapter) {
|
|
5702
|
+
throw new Error(
|
|
5703
|
+
"Installed @remnic/bench does not export createMcpMemCorrectAdapter()."
|
|
5704
|
+
);
|
|
5705
|
+
}
|
|
5706
|
+
return benchModule.createMcpMemCorrectAdapter(
|
|
5707
|
+
buildPackageMcpAdapterOptions(parsed)
|
|
5708
|
+
);
|
|
5468
5709
|
}
|
|
5469
5710
|
async function buildPackageBenchExecutionPlans(benchModule, parsed, runtimeProfiles) {
|
|
5470
5711
|
const plans = [];
|
|
@@ -5476,6 +5717,7 @@ async function buildPackageBenchExecutionPlans(benchModule, parsed, runtimeProfi
|
|
|
5476
5717
|
);
|
|
5477
5718
|
const createAdapter = resolvePackageBenchAdapterFactory(
|
|
5478
5719
|
benchModule,
|
|
5720
|
+
parsed,
|
|
5479
5721
|
parsed.quick,
|
|
5480
5722
|
runtime.profile
|
|
5481
5723
|
);
|
|
@@ -5485,7 +5727,7 @@ async function buildPackageBenchExecutionPlans(benchModule, parsed, runtimeProfi
|
|
|
5485
5727
|
plans.push({
|
|
5486
5728
|
runtime,
|
|
5487
5729
|
createAdapter,
|
|
5488
|
-
adapterMode: resolvePackageBenchAdapterMode(parsed.quick, runtime.profile)
|
|
5730
|
+
adapterMode: resolvePackageBenchAdapterMode(parsed, parsed.quick, runtime.profile)
|
|
5489
5731
|
});
|
|
5490
5732
|
}
|
|
5491
5733
|
return plans;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/cli",
|
|
3
|
-
"version": "9.6.
|
|
3
|
+
"version": "9.6.19",
|
|
4
4
|
"description": "CLI for Remnic memory — init, query, doctor, daemon management",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -26,23 +26,23 @@
|
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"yaml": "^2.4.2",
|
|
29
|
-
"@remnic/
|
|
30
|
-
"@remnic/plugin-pi": "^9.6.
|
|
31
|
-
"@remnic/
|
|
29
|
+
"@remnic/server": "^9.6.19",
|
|
30
|
+
"@remnic/plugin-pi": "^9.6.19",
|
|
31
|
+
"@remnic/core": "^9.6.19"
|
|
32
32
|
},
|
|
33
33
|
"peerDependencies": {
|
|
34
|
-
"@remnic/bench": "^9.6.
|
|
35
|
-
"@remnic/export-weclone": "^9.6.
|
|
36
|
-
"@remnic/import-weclone": "^9.6.
|
|
37
|
-
"@remnic/import-chatgpt": "^9.6.
|
|
38
|
-
"@remnic/import-claude": "^9.6.
|
|
39
|
-
"@remnic/import-gemini": "^9.6.
|
|
40
|
-
"@remnic/import-lossless-claw": "^9.6.
|
|
41
|
-
"@remnic/import-mem0": "^9.6.
|
|
42
|
-
"@remnic/import-supermemory": "^9.6.
|
|
43
|
-
"@remnic/connector-limitless": "^9.6.
|
|
44
|
-
"@remnic/connector-bee": "^9.6.
|
|
45
|
-
"@remnic/connector-omi": "^9.6.
|
|
34
|
+
"@remnic/bench": "^9.6.19",
|
|
35
|
+
"@remnic/export-weclone": "^9.6.19",
|
|
36
|
+
"@remnic/import-weclone": "^9.6.19",
|
|
37
|
+
"@remnic/import-chatgpt": "^9.6.19",
|
|
38
|
+
"@remnic/import-claude": "^9.6.19",
|
|
39
|
+
"@remnic/import-gemini": "^9.6.19",
|
|
40
|
+
"@remnic/import-lossless-claw": "^9.6.19",
|
|
41
|
+
"@remnic/import-mem0": "^9.6.19",
|
|
42
|
+
"@remnic/import-supermemory": "^9.6.19",
|
|
43
|
+
"@remnic/connector-limitless": "^9.6.19",
|
|
44
|
+
"@remnic/connector-bee": "^9.6.19",
|
|
45
|
+
"@remnic/connector-omi": "^9.6.19"
|
|
46
46
|
},
|
|
47
47
|
"peerDependenciesMeta": {
|
|
48
48
|
"@remnic/bench": {
|
|
@@ -85,18 +85,18 @@
|
|
|
85
85
|
"devDependencies": {
|
|
86
86
|
"tsup": "^8.5.1",
|
|
87
87
|
"typescript": "^5.9.3",
|
|
88
|
-
"@remnic/bench": "9.6.
|
|
89
|
-
"@remnic/export-weclone": "9.6.
|
|
90
|
-
"@remnic/import-weclone": "9.6.
|
|
91
|
-
"@remnic/import-
|
|
92
|
-
"@remnic/import-
|
|
93
|
-
"@remnic/import-
|
|
94
|
-
"@remnic/import-
|
|
95
|
-
"@remnic/
|
|
96
|
-
"@remnic/connector-
|
|
97
|
-
"@remnic/
|
|
98
|
-
"@remnic/
|
|
99
|
-
"@remnic/
|
|
88
|
+
"@remnic/bench": "9.6.19",
|
|
89
|
+
"@remnic/export-weclone": "9.6.19",
|
|
90
|
+
"@remnic/import-weclone": "9.6.19",
|
|
91
|
+
"@remnic/import-chatgpt": "9.6.19",
|
|
92
|
+
"@remnic/import-gemini": "9.6.19",
|
|
93
|
+
"@remnic/import-lossless-claw": "9.6.19",
|
|
94
|
+
"@remnic/import-claude": "9.6.19",
|
|
95
|
+
"@remnic/import-mem0": "9.6.19",
|
|
96
|
+
"@remnic/connector-omi": "9.6.19",
|
|
97
|
+
"@remnic/import-supermemory": "9.6.19",
|
|
98
|
+
"@remnic/connector-bee": "9.6.19",
|
|
99
|
+
"@remnic/connector-limitless": "9.6.19"
|
|
100
100
|
},
|
|
101
101
|
"license": "MIT",
|
|
102
102
|
"repository": {
|