@remnic/cli 9.69.60 → 9.69.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +350 -20
- package/package.json +32 -32
package/dist/index.js
CHANGED
|
@@ -10231,25 +10231,54 @@ var BENCH_SECURITY_USAGE = `Usage: remnic bench security injection-suite --seeds
|
|
|
10231
10231
|
H5 injection-suite runner. Resume, host-fault pause, multi-host claim
|
|
10232
10232
|
leases, and --limit follow the H6 contract (issue #1963 / PR #2312).
|
|
10233
10233
|
|
|
10234
|
+
Additional commands:
|
|
10235
|
+
remnic bench security injection-suite-analyze --run DIR
|
|
10236
|
+
remnic bench security injection-suite-adaptive-online [injection-suite options]
|
|
10237
|
+
remnic bench security injection-suite-adaptive-online-analyze --run DIR
|
|
10238
|
+
remnic bench security injection-suite-publication-utility --observations FILE --out FILE
|
|
10239
|
+
remnic bench security injection-suite-replay --run DIR
|
|
10240
|
+
remnic bench security injection-suite-utility [injection-suite options] [--dataset-dir DIR]
|
|
10241
|
+
remnic bench security injection-suite-decide --base DIR --base DIR --utility FILE --utility FILE [--adaptive DIR --adaptive DIR] --out DIR
|
|
10242
|
+
|
|
10234
10243
|
Options:
|
|
10235
10244
|
--seeds N Positive seed count (required)
|
|
10245
|
+
--seed-base N First deterministic corpus seed (default: 71)
|
|
10236
10246
|
--variants-per-family N Variants per attack family (default: 25)
|
|
10237
|
-
--
|
|
10247
|
+
--family FAMILY Target one dev/pilot attack family; forbidden for main
|
|
10248
|
+
--stage base|adaptive-r1|adaptive-r2|adaptive-r3|benign|benign-use
|
|
10249
|
+
Frozen corpus stage (default: base)
|
|
10250
|
+
--attacker-executor openai-compat|ollama
|
|
10251
|
+
adaptive-online only: attacker transport
|
|
10252
|
+
--attacker-base-url URL Attacker endpoint (defaults mirror the defended executor)
|
|
10253
|
+
--attacker-model NAME Attacker model id (default: qwen3.8-27b-64k:latest)
|
|
10254
|
+
--attacker-model-digest SHA256
|
|
10255
|
+
Immutable served attacker-model digest
|
|
10256
|
+
--attacker-iterations N K attacker rewrites per base variant (default: 3)
|
|
10257
|
+
--attacker-prompt FILE Frozen attacker system prompt (default: bench fixture)
|
|
10258
|
+
--arm ID Repeat to select frozen defense arms
|
|
10259
|
+
--run-kind dev|pilot|main Run gate to enforce (default: dev)
|
|
10238
10260
|
--executor local|ollama|openai-compat
|
|
10239
10261
|
local = deterministic screen/fence (default)
|
|
10240
10262
|
ollama = native /api/chat
|
|
10241
10263
|
openai-compat = /v1/chat/completions
|
|
10242
|
-
(api.openai.com / integrate.api.nvidia.com
|
|
10243
|
-
https + provider key;
|
|
10244
|
-
|
|
10245
|
-
|
|
10264
|
+
(api.openai.com / integrate.api.nvidia.com /
|
|
10265
|
+
router.huggingface.co require https + provider key;
|
|
10266
|
+
other hosts require REMNIC_OPENAI_COMPAT_API_KEY
|
|
10267
|
+
and https, except loopback HTTP)
|
|
10246
10268
|
--base-url URL Endpoint (default: http://127.0.0.1:11434)
|
|
10247
10269
|
--model NAME Model id (default: qwen3.8-27b-64k:latest)
|
|
10270
|
+
--model-digest SHA256 Immutable served-model digest (required for main)
|
|
10271
|
+
--model-context-tokens N Native context length (required for main)
|
|
10248
10272
|
--request-timeout-ms N Per-call timeout (default: 300000)
|
|
10249
10273
|
--out DIR New run directory (default: ~/.remnic/bench/results/h5-injection-suite)
|
|
10250
10274
|
--run DIR Existing run directory; implies --resume
|
|
10275
|
+
--capture-responses Write raw responses.jsonl beside the checkpoints
|
|
10251
10276
|
--resume Continue an existing run (required if DIR already has run.json)
|
|
10277
|
+
--retry-ambiguous Retry a persisted in-flight paid request after operator review
|
|
10252
10278
|
--limit N Execute at most N planned rows (dry-run / smoke)
|
|
10279
|
+
--utility-benchmark ID Repeat: locomo, longmemeval, or drift-gen
|
|
10280
|
+
--longmemeval-dataset-dir DIR
|
|
10281
|
+
Frozen LongMemEval dataset for utility runs
|
|
10253
10282
|
`;
|
|
10254
10283
|
function parsePositiveInteger(raw, flag) {
|
|
10255
10284
|
const value = Number(raw);
|
|
@@ -10259,33 +10288,85 @@ function parsePositiveInteger(raw, flag) {
|
|
|
10259
10288
|
return value;
|
|
10260
10289
|
}
|
|
10261
10290
|
function parseBenchSecurityArgs(args) {
|
|
10262
|
-
if (args.length === 0 || args[0] === "--help" || args[0] === "-h")
|
|
10291
|
+
if (args.length === 0 || args[0] === "--help" || args[0] === "-h")
|
|
10292
|
+
return { help: true };
|
|
10263
10293
|
if (args[0] !== "injection-suite") {
|
|
10264
10294
|
throw new Error(`unknown bench security subcommand ${args[0]}`);
|
|
10265
10295
|
}
|
|
10266
10296
|
let seeds;
|
|
10297
|
+
let seedBase;
|
|
10267
10298
|
let variantsPerFamily = 25;
|
|
10299
|
+
let family;
|
|
10268
10300
|
let modelProfileId = "local-dry";
|
|
10301
|
+
let stage = "base";
|
|
10302
|
+
const arms = [];
|
|
10303
|
+
let runKind = "dev";
|
|
10269
10304
|
let outputDir = DEFAULT_OUTPUT_DIR;
|
|
10270
10305
|
let resume = false;
|
|
10306
|
+
let retryAmbiguous = false;
|
|
10271
10307
|
let limit;
|
|
10272
10308
|
let executor = "local";
|
|
10309
|
+
let datasetDir;
|
|
10310
|
+
let longmemevalDatasetDir;
|
|
10311
|
+
const utilityBenchmarks = [];
|
|
10273
10312
|
let baseUrl;
|
|
10274
10313
|
let model;
|
|
10314
|
+
let modelDigest;
|
|
10315
|
+
let modelContextTokens;
|
|
10275
10316
|
let requestTimeoutMs;
|
|
10317
|
+
let captureResponses = false;
|
|
10318
|
+
let attackerExecutor;
|
|
10319
|
+
let attackerBaseUrl;
|
|
10320
|
+
let attackerModel;
|
|
10321
|
+
let attackerModelDigest;
|
|
10322
|
+
let attackerIterations;
|
|
10323
|
+
let attackerPromptPath;
|
|
10276
10324
|
for (let index = 1; index < args.length; index += 1) {
|
|
10277
10325
|
const flag = args[index] ?? "";
|
|
10278
10326
|
const next = args[index + 1];
|
|
10279
10327
|
if (flag === "--seeds") {
|
|
10280
10328
|
seeds = parsePositiveInteger(next, "--seeds");
|
|
10281
10329
|
index += 1;
|
|
10330
|
+
} else if (flag === "--seed-base") {
|
|
10331
|
+
seedBase = parsePositiveInteger(next, "--seed-base");
|
|
10332
|
+
index += 1;
|
|
10282
10333
|
} else if (flag === "--variants-per-family") {
|
|
10283
10334
|
variantsPerFamily = parsePositiveInteger(next, "--variants-per-family");
|
|
10284
10335
|
index += 1;
|
|
10336
|
+
} else if (flag === "--family") {
|
|
10337
|
+
if (next !== "minja" && next !== "sleeper" && next !== "cross-session" && next !== "tool-hijack") {
|
|
10338
|
+
throw new Error(
|
|
10339
|
+
"--family must be minja, sleeper, cross-session, or tool-hijack"
|
|
10340
|
+
);
|
|
10341
|
+
}
|
|
10342
|
+
family = next;
|
|
10343
|
+
index += 1;
|
|
10285
10344
|
} else if (flag === "--model-profile") {
|
|
10286
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10345
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10346
|
+
throw new Error("missing value for --model-profile");
|
|
10287
10347
|
modelProfileId = next;
|
|
10288
10348
|
index += 1;
|
|
10349
|
+
} else if (flag === "--stage") {
|
|
10350
|
+
if (next !== "base" && next !== "adaptive-r1" && next !== "adaptive-r2" && next !== "adaptive-r3" && next !== "benign" && next !== "benign-use" && next !== "adaptive-online-r1") {
|
|
10351
|
+
throw new Error(
|
|
10352
|
+
"--stage must be base, adaptive-r1, adaptive-r2, adaptive-r3, benign, benign-use, or adaptive-online-r1"
|
|
10353
|
+
);
|
|
10354
|
+
}
|
|
10355
|
+
stage = next;
|
|
10356
|
+
index += 1;
|
|
10357
|
+
} else if (flag === "--arm") {
|
|
10358
|
+
if (next !== "none" && next !== "fencing" && next !== "quarantine" && next !== "both" && next !== "structured-boundary" && next !== "spotlighting-marking" && next !== "source-authenticated-fencing" && next !== "control-data-isolation" && next !== "layered-fence-quarantine") {
|
|
10359
|
+
throw new Error("unknown --arm defense baseline");
|
|
10360
|
+
}
|
|
10361
|
+
if (arms.includes(next)) throw new Error(`duplicate --arm ${next}`);
|
|
10362
|
+
arms.push(next);
|
|
10363
|
+
index += 1;
|
|
10364
|
+
} else if (flag === "--run-kind") {
|
|
10365
|
+
if (next !== "dev" && next !== "pilot" && next !== "main") {
|
|
10366
|
+
throw new Error("--run-kind must be dev, pilot, or main");
|
|
10367
|
+
}
|
|
10368
|
+
runKind = next;
|
|
10369
|
+
index += 1;
|
|
10289
10370
|
} else if (flag === "--executor") {
|
|
10290
10371
|
if (next !== "local" && next !== "ollama" && next !== "openai-compat") {
|
|
10291
10372
|
throw new Error("--executor must be local, ollama, or openai-compat");
|
|
@@ -10293,72 +10374,321 @@ function parseBenchSecurityArgs(args) {
|
|
|
10293
10374
|
executor = next;
|
|
10294
10375
|
index += 1;
|
|
10295
10376
|
} else if (flag === "--base-url") {
|
|
10296
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10377
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10378
|
+
throw new Error("missing value for --base-url");
|
|
10297
10379
|
baseUrl = next;
|
|
10298
10380
|
index += 1;
|
|
10299
10381
|
} else if (flag === "--model") {
|
|
10300
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10382
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10383
|
+
throw new Error("missing value for --model");
|
|
10301
10384
|
model = next;
|
|
10302
10385
|
index += 1;
|
|
10386
|
+
} else if (flag === "--model-digest") {
|
|
10387
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10388
|
+
throw new Error("missing value for --model-digest");
|
|
10389
|
+
modelDigest = next;
|
|
10390
|
+
index += 1;
|
|
10391
|
+
} else if (flag === "--dataset-dir") {
|
|
10392
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10393
|
+
throw new Error("missing value for --dataset-dir");
|
|
10394
|
+
datasetDir = expandTilde(next);
|
|
10395
|
+
index += 1;
|
|
10396
|
+
} else if (flag === "--longmemeval-dataset-dir") {
|
|
10397
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10398
|
+
throw new Error("missing value for --longmemeval-dataset-dir");
|
|
10399
|
+
longmemevalDatasetDir = expandTilde(next);
|
|
10400
|
+
index += 1;
|
|
10401
|
+
} else if (flag === "--utility-benchmark") {
|
|
10402
|
+
if (next !== "locomo" && next !== "longmemeval" && next !== "drift-gen") {
|
|
10403
|
+
throw new Error(
|
|
10404
|
+
"--utility-benchmark must be locomo, longmemeval, or drift-gen"
|
|
10405
|
+
);
|
|
10406
|
+
}
|
|
10407
|
+
utilityBenchmarks.push(next);
|
|
10408
|
+
index += 1;
|
|
10409
|
+
} else if (flag === "--model-context-tokens") {
|
|
10410
|
+
modelContextTokens = parsePositiveInteger(next, "--model-context-tokens");
|
|
10411
|
+
index += 1;
|
|
10303
10412
|
} else if (flag === "--request-timeout-ms") {
|
|
10304
10413
|
requestTimeoutMs = parsePositiveInteger(next, "--request-timeout-ms");
|
|
10305
10414
|
index += 1;
|
|
10306
10415
|
} else if (flag === "--out") {
|
|
10307
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10416
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10417
|
+
throw new Error("missing value for --out");
|
|
10308
10418
|
outputDir = expandTilde(next);
|
|
10309
10419
|
index += 1;
|
|
10310
10420
|
} else if (flag === "--run") {
|
|
10311
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10421
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10422
|
+
throw new Error("missing value for --run");
|
|
10312
10423
|
outputDir = expandTilde(next);
|
|
10313
10424
|
resume = true;
|
|
10314
10425
|
index += 1;
|
|
10315
10426
|
} else if (flag === "--resume") {
|
|
10316
10427
|
resume = true;
|
|
10428
|
+
} else if (flag === "--retry-ambiguous") {
|
|
10429
|
+
retryAmbiguous = true;
|
|
10317
10430
|
} else if (flag === "--limit") {
|
|
10318
10431
|
limit = parsePositiveInteger(next, "--limit");
|
|
10319
10432
|
index += 1;
|
|
10433
|
+
} else if (flag === "--capture-responses") {
|
|
10434
|
+
captureResponses = true;
|
|
10435
|
+
} else if (flag === "--attacker-executor") {
|
|
10436
|
+
if (next !== "openai-compat" && next !== "ollama") {
|
|
10437
|
+
throw new Error("--attacker-executor must be openai-compat or ollama");
|
|
10438
|
+
}
|
|
10439
|
+
attackerExecutor = next;
|
|
10440
|
+
index += 1;
|
|
10441
|
+
} else if (flag === "--attacker-base-url") {
|
|
10442
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10443
|
+
throw new Error("missing value for --attacker-base-url");
|
|
10444
|
+
attackerBaseUrl = next;
|
|
10445
|
+
index += 1;
|
|
10446
|
+
} else if (flag === "--attacker-model") {
|
|
10447
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10448
|
+
throw new Error("missing value for --attacker-model");
|
|
10449
|
+
attackerModel = next;
|
|
10450
|
+
index += 1;
|
|
10451
|
+
} else if (flag === "--attacker-model-digest") {
|
|
10452
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10453
|
+
throw new Error("missing value for --attacker-model-digest");
|
|
10454
|
+
attackerModelDigest = next;
|
|
10455
|
+
index += 1;
|
|
10456
|
+
} else if (flag === "--attacker-iterations") {
|
|
10457
|
+
attackerIterations = parsePositiveInteger(next, "--attacker-iterations");
|
|
10458
|
+
index += 1;
|
|
10459
|
+
} else if (flag === "--attacker-prompt") {
|
|
10460
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10461
|
+
throw new Error("missing value for --attacker-prompt");
|
|
10462
|
+
attackerPromptPath = expandTilde(next);
|
|
10463
|
+
index += 1;
|
|
10320
10464
|
} else {
|
|
10321
10465
|
throw new Error(`unknown option ${flag}`);
|
|
10322
10466
|
}
|
|
10323
10467
|
}
|
|
10324
|
-
if (seeds === void 0)
|
|
10468
|
+
if (seeds === void 0)
|
|
10469
|
+
throw new Error("injection-suite requires --seeds N");
|
|
10470
|
+
if (retryAmbiguous && !resume)
|
|
10471
|
+
throw new Error("--retry-ambiguous requires --run or --resume");
|
|
10325
10472
|
return {
|
|
10326
10473
|
seeds,
|
|
10474
|
+
...seedBase === void 0 ? {} : { seedBase },
|
|
10327
10475
|
variantsPerFamily,
|
|
10476
|
+
...family === void 0 ? {} : { family },
|
|
10328
10477
|
modelProfileId,
|
|
10478
|
+
stage,
|
|
10479
|
+
...arms.length === 0 ? {} : { arms },
|
|
10480
|
+
runKind,
|
|
10329
10481
|
outputDir,
|
|
10330
10482
|
resume,
|
|
10331
10483
|
executor,
|
|
10484
|
+
retryAmbiguous,
|
|
10332
10485
|
...limit === void 0 ? {} : { limit },
|
|
10333
10486
|
...baseUrl === void 0 ? {} : { baseUrl },
|
|
10334
10487
|
...model === void 0 ? {} : { model },
|
|
10335
|
-
...
|
|
10488
|
+
...modelDigest === void 0 ? {} : { modelDigest },
|
|
10489
|
+
...modelContextTokens === void 0 ? {} : { modelContextTokens },
|
|
10490
|
+
...datasetDir === void 0 ? {} : { datasetDir },
|
|
10491
|
+
...longmemevalDatasetDir === void 0 ? {} : { longmemevalDatasetDir },
|
|
10492
|
+
...utilityBenchmarks.length === 0 ? {} : { utilityBenchmarks },
|
|
10493
|
+
...requestTimeoutMs === void 0 ? {} : { requestTimeoutMs },
|
|
10494
|
+
captureResponses,
|
|
10495
|
+
...attackerExecutor === void 0 ? {} : { attackerExecutor },
|
|
10496
|
+
...attackerBaseUrl === void 0 ? {} : { attackerBaseUrl },
|
|
10497
|
+
...attackerModel === void 0 ? {} : { attackerModel },
|
|
10498
|
+
...attackerModelDigest === void 0 ? {} : { attackerModelDigest },
|
|
10499
|
+
...attackerIterations === void 0 ? {} : { attackerIterations },
|
|
10500
|
+
...attackerPromptPath === void 0 ? {} : { attackerPromptPath }
|
|
10336
10501
|
};
|
|
10337
10502
|
}
|
|
10338
10503
|
async function cmdBenchSecurity(args) {
|
|
10339
10504
|
try {
|
|
10340
|
-
|
|
10505
|
+
if (args[0] === "injection-suite-publication-utility") {
|
|
10506
|
+
if (args.length !== 5 || args[1] !== "--observations" || !args[2] || args[3] !== "--out" || !args[4]) {
|
|
10507
|
+
throw new Error(
|
|
10508
|
+
"injection-suite-publication-utility requires --observations FILE --out FILE"
|
|
10509
|
+
);
|
|
10510
|
+
}
|
|
10511
|
+
const bench2 = await loadBenchModule();
|
|
10512
|
+
const analyze = bench2.analyzeInjectionSuitePublicationUtilityFile;
|
|
10513
|
+
if (typeof analyze !== "function")
|
|
10514
|
+
throw new Error(
|
|
10515
|
+
"Installed @remnic/bench lacks H5 publication utility analysis"
|
|
10516
|
+
);
|
|
10517
|
+
const analysis = await analyze(
|
|
10518
|
+
expandTilde(args[2]),
|
|
10519
|
+
expandTilde(args[4])
|
|
10520
|
+
);
|
|
10521
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10522
|
+
return;
|
|
10523
|
+
}
|
|
10524
|
+
if (args[0] === "injection-suite-decide") {
|
|
10525
|
+
const baseRunDirs = [];
|
|
10526
|
+
const utilityStatisticsPaths = [];
|
|
10527
|
+
const adaptiveRunDirs = [];
|
|
10528
|
+
let outputDir;
|
|
10529
|
+
for (let index = 1; index < args.length; index += 2) {
|
|
10530
|
+
const flag = args[index];
|
|
10531
|
+
const value = args[index + 1];
|
|
10532
|
+
if (!value)
|
|
10533
|
+
throw new Error(`missing value for ${flag ?? "campaign flag"}`);
|
|
10534
|
+
if (flag === "--base") baseRunDirs.push(expandTilde(value));
|
|
10535
|
+
else if (flag === "--utility")
|
|
10536
|
+
utilityStatisticsPaths.push(expandTilde(value));
|
|
10537
|
+
else if (flag === "--adaptive")
|
|
10538
|
+
adaptiveRunDirs.push(expandTilde(value));
|
|
10539
|
+
else if (flag === "--out") outputDir = expandTilde(value);
|
|
10540
|
+
else throw new Error(`unknown campaign option ${flag}`);
|
|
10541
|
+
}
|
|
10542
|
+
if (!outputDir)
|
|
10543
|
+
throw new Error("injection-suite-decide requires --out DIR");
|
|
10544
|
+
const campaignBench = await loadBenchModule();
|
|
10545
|
+
const decide = campaignBench.decideInjectionSuiteCampaign;
|
|
10546
|
+
if (typeof decide !== "function")
|
|
10547
|
+
throw new Error("Installed @remnic/bench lacks H5 campaign decision");
|
|
10548
|
+
const decision = await decide({
|
|
10549
|
+
baseRunDirs,
|
|
10550
|
+
utilityStatisticsPaths,
|
|
10551
|
+
...adaptiveRunDirs.length > 0 ? { adaptiveRunDirs } : {},
|
|
10552
|
+
outputDir
|
|
10553
|
+
});
|
|
10554
|
+
console.log(JSON.stringify(decision, null, 2));
|
|
10555
|
+
return;
|
|
10556
|
+
}
|
|
10557
|
+
if (args[0] === "injection-suite-analyze" || args[0] === "injection-suite-adaptive-online-analyze" || args[0] === "injection-suite-publication-analyze" || args[0] === "injection-suite-replay") {
|
|
10558
|
+
if (args.length !== 3 || args[1] !== "--run" || !args[2]) {
|
|
10559
|
+
throw new Error(`${args[0]} requires --run DIR`);
|
|
10560
|
+
}
|
|
10561
|
+
const analysisBench = await loadBenchModule();
|
|
10562
|
+
const runDir = expandTilde(args[2]);
|
|
10563
|
+
if (args[0] === "injection-suite-adaptive-online-analyze") {
|
|
10564
|
+
const analyze = analysisBench.analyzeInjectionSuiteOnlineAdaptiveRun;
|
|
10565
|
+
if (typeof analyze !== "function")
|
|
10566
|
+
throw new Error(
|
|
10567
|
+
"Installed @remnic/bench lacks the adaptive-online analysis"
|
|
10568
|
+
);
|
|
10569
|
+
const analysis = await analyze(runDir);
|
|
10570
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10571
|
+
} else if (args[0] === "injection-suite-analyze") {
|
|
10572
|
+
const analyze = analysisBench.analyzeInjectionSuiteRun;
|
|
10573
|
+
if (typeof analyze !== "function")
|
|
10574
|
+
throw new Error("Installed @remnic/bench lacks H5 analysis");
|
|
10575
|
+
const analysis = await analyze(runDir);
|
|
10576
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10577
|
+
} else if (args[0] === "injection-suite-publication-analyze") {
|
|
10578
|
+
const analyze = analysisBench.analyzeInjectionSuitePublicationRun;
|
|
10579
|
+
if (typeof analyze !== "function")
|
|
10580
|
+
throw new Error(
|
|
10581
|
+
"Installed @remnic/bench lacks H5 publication analysis"
|
|
10582
|
+
);
|
|
10583
|
+
const analysis = await analyze(runDir);
|
|
10584
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10585
|
+
} else {
|
|
10586
|
+
const replay = analysisBench.replayInjectionSuiteStatistics;
|
|
10587
|
+
if (typeof replay !== "function")
|
|
10588
|
+
throw new Error("Installed @remnic/bench lacks H5 replay");
|
|
10589
|
+
await replay(runDir);
|
|
10590
|
+
console.log(JSON.stringify({ replay: "ok", runDir }));
|
|
10591
|
+
}
|
|
10592
|
+
return;
|
|
10593
|
+
}
|
|
10594
|
+
const utilityMode = args[0] === "injection-suite-utility";
|
|
10595
|
+
const onlineMode = args[0] === "injection-suite-adaptive-online";
|
|
10596
|
+
const parsed = parseBenchSecurityArgs(
|
|
10597
|
+
utilityMode || onlineMode ? ["injection-suite", ...args.slice(1)] : args
|
|
10598
|
+
);
|
|
10341
10599
|
if ("help" in parsed) {
|
|
10342
10600
|
console.log(BENCH_SECURITY_USAGE);
|
|
10343
10601
|
return;
|
|
10344
10602
|
}
|
|
10345
10603
|
const bench = await loadBenchModule();
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10604
|
+
if (utilityMode) {
|
|
10605
|
+
const utility = bench.runInjectionSuiteUtility;
|
|
10606
|
+
if (typeof utility !== "function")
|
|
10607
|
+
throw new Error("Installed @remnic/bench lacks H5 utility runner");
|
|
10608
|
+
const analysis = await utility({
|
|
10609
|
+
seeds: parsed.seeds,
|
|
10610
|
+
variantsPerFamily: parsed.variantsPerFamily,
|
|
10611
|
+
modelProfileId: parsed.modelProfileId,
|
|
10612
|
+
outputDir: parsed.outputDir,
|
|
10613
|
+
executor: parsed.executor,
|
|
10614
|
+
stage: parsed.stage,
|
|
10615
|
+
runKind: parsed.runKind,
|
|
10616
|
+
...parsed.resume ? { resume: true } : {},
|
|
10617
|
+
...parsed.retryAmbiguous ? { retryAmbiguous: true } : {},
|
|
10618
|
+
...parsed.limit === void 0 ? {} : { limit: parsed.limit },
|
|
10619
|
+
...parsed.baseUrl === void 0 ? {} : { baseUrl: parsed.baseUrl },
|
|
10620
|
+
...parsed.model === void 0 ? {} : { model: parsed.model },
|
|
10621
|
+
...parsed.modelDigest === void 0 ? {} : { modelDigest: parsed.modelDigest },
|
|
10622
|
+
...parsed.modelContextTokens === void 0 ? {} : { modelContextTokens: parsed.modelContextTokens },
|
|
10623
|
+
...parsed.requestTimeoutMs === void 0 ? {} : { requestTimeoutMs: parsed.requestTimeoutMs },
|
|
10624
|
+
...parsed.datasetDir === void 0 ? {} : { locomoDatasetDir: parsed.datasetDir },
|
|
10625
|
+
...parsed.longmemevalDatasetDir === void 0 ? {} : { longmemevalDatasetDir: parsed.longmemevalDatasetDir },
|
|
10626
|
+
...parsed.utilityBenchmarks === void 0 ? {} : { utilityBenchmarks: parsed.utilityBenchmarks },
|
|
10627
|
+
...parsed.arms === void 0 ? {} : { arms: parsed.arms }
|
|
10628
|
+
});
|
|
10629
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10630
|
+
return;
|
|
10349
10631
|
}
|
|
10350
|
-
const
|
|
10632
|
+
const suiteInput = {
|
|
10351
10633
|
seeds: parsed.seeds,
|
|
10634
|
+
...parsed.seedBase === void 0 ? {} : { seedBase: parsed.seedBase },
|
|
10352
10635
|
variantsPerFamily: parsed.variantsPerFamily,
|
|
10353
10636
|
modelProfileId: parsed.modelProfileId,
|
|
10354
10637
|
outputDir: parsed.outputDir,
|
|
10355
10638
|
executor: parsed.executor,
|
|
10639
|
+
stage: parsed.stage,
|
|
10640
|
+
runKind: parsed.runKind,
|
|
10641
|
+
...parsed.arms === void 0 ? {} : { arms: parsed.arms },
|
|
10642
|
+
...parsed.family === void 0 ? {} : { family: parsed.family },
|
|
10356
10643
|
...parsed.resume ? { resume: true } : {},
|
|
10644
|
+
...parsed.retryAmbiguous ? { retryAmbiguous: true } : {},
|
|
10357
10645
|
...parsed.limit === void 0 ? {} : { limit: parsed.limit },
|
|
10358
10646
|
...parsed.baseUrl === void 0 ? {} : { baseUrl: parsed.baseUrl },
|
|
10359
10647
|
...parsed.model === void 0 ? {} : { model: parsed.model },
|
|
10360
|
-
...parsed.
|
|
10361
|
-
|
|
10648
|
+
...parsed.modelDigest === void 0 ? {} : { modelDigest: parsed.modelDigest },
|
|
10649
|
+
...parsed.modelContextTokens === void 0 ? {} : { modelContextTokens: parsed.modelContextTokens },
|
|
10650
|
+
...parsed.requestTimeoutMs === void 0 ? {} : { requestTimeoutMs: parsed.requestTimeoutMs },
|
|
10651
|
+
...parsed.captureResponses ? { captureResponses: true } : {}
|
|
10652
|
+
};
|
|
10653
|
+
if (onlineMode) {
|
|
10654
|
+
const runOnline = bench.runInjectionSuiteOnlineAdaptive;
|
|
10655
|
+
if (typeof runOnline !== "function") {
|
|
10656
|
+
throw new Error(
|
|
10657
|
+
"Installed @remnic/bench lacks runInjectionSuiteOnlineAdaptive"
|
|
10658
|
+
);
|
|
10659
|
+
}
|
|
10660
|
+
if (parsed.executor === "local") {
|
|
10661
|
+
throw new Error(
|
|
10662
|
+
"adaptive-online-r1 requires --executor ollama or openai-compat (local executor cannot run product rows)"
|
|
10663
|
+
);
|
|
10664
|
+
}
|
|
10665
|
+
const attackerPromptPath = parsed.attackerPromptPath ?? bench.DEFAULT_ATTACKER_PROMPT_PATH_GETTER();
|
|
10666
|
+
const result2 = await runOnline({
|
|
10667
|
+
...suiteInput,
|
|
10668
|
+
attackerExecutor: parsed.attackerExecutor ?? "openai-compat",
|
|
10669
|
+
attackerIterations: parsed.attackerIterations ?? 3,
|
|
10670
|
+
attackerPromptPath,
|
|
10671
|
+
...parsed.attackerBaseUrl === void 0 ? {} : { attackerBaseUrl: parsed.attackerBaseUrl },
|
|
10672
|
+
...parsed.attackerModel === void 0 ? {} : { attackerModel: parsed.attackerModel },
|
|
10673
|
+
...parsed.attackerModelDigest === void 0 ? {} : { attackerModelDigest: parsed.attackerModelDigest }
|
|
10674
|
+
});
|
|
10675
|
+
if (result2.exitCode === 0) console.log(result2.output);
|
|
10676
|
+
else console.error(result2.output);
|
|
10677
|
+
if (result2.exitCode !== 0) process.exitCode = result2.exitCode;
|
|
10678
|
+
return;
|
|
10679
|
+
}
|
|
10680
|
+
if (parsed.stage === "adaptive-online-r1") {
|
|
10681
|
+
throw new Error(
|
|
10682
|
+
"stage adaptive-online-r1 requires the injection-suite-adaptive-online command"
|
|
10683
|
+
);
|
|
10684
|
+
}
|
|
10685
|
+
const run = bench.runInjectionSuiteCliCommand;
|
|
10686
|
+
if (typeof run !== "function") {
|
|
10687
|
+
throw new Error(
|
|
10688
|
+
"Installed @remnic/bench is missing runInjectionSuiteCliCommand"
|
|
10689
|
+
);
|
|
10690
|
+
}
|
|
10691
|
+
const result = await run(suiteInput);
|
|
10362
10692
|
if (result.exitCode === 0) console.log(result.output);
|
|
10363
10693
|
else console.error(result.output);
|
|
10364
10694
|
if (result.exitCode !== 0) process.exitCode = result.exitCode;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/cli",
|
|
3
|
-
"version": "9.69.
|
|
3
|
+
"version": "9.69.61",
|
|
4
4
|
"description": "CLI for Remnic memory — init, query, doctor, daemon management",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -26,26 +26,26 @@
|
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"yaml": "^2.4.2",
|
|
29
|
-
"@remnic/plugin-pi": "^9.69.
|
|
30
|
-
"@remnic/server": "^9.69.
|
|
31
|
-
"@remnic/core": "^9.69.
|
|
29
|
+
"@remnic/plugin-pi": "^9.69.61",
|
|
30
|
+
"@remnic/server": "^9.69.61",
|
|
31
|
+
"@remnic/core": "^9.69.61"
|
|
32
32
|
},
|
|
33
33
|
"peerDependencies": {
|
|
34
|
-
"@remnic/bench": "^9.69.
|
|
35
|
-
"@remnic/plugin-openclaw": "^9.69.
|
|
36
|
-
"@remnic/export-weclone": "^9.69.
|
|
37
|
-
"@remnic/import-weclone": "^9.69.
|
|
38
|
-
"@remnic/import-chatgpt": "^9.69.
|
|
39
|
-
"@remnic/import-claude": "^9.69.
|
|
40
|
-
"@remnic/import-gemini": "^9.69.
|
|
41
|
-
"@remnic/import-lossless-claw": "^9.69.
|
|
42
|
-
"@remnic/import-mem0": "^9.69.
|
|
43
|
-
"@remnic/import-supermemory": "^9.69.
|
|
44
|
-
"@remnic/import-okf": "^9.69.
|
|
45
|
-
"@remnic/connector-limitless": "^9.69.
|
|
46
|
-
"@remnic/connector-bee": "^9.69.
|
|
47
|
-
"@remnic/connector-omi": "^9.69.
|
|
48
|
-
"@remnic/capture-audio": "^9.69.
|
|
34
|
+
"@remnic/bench": "^9.69.61",
|
|
35
|
+
"@remnic/plugin-openclaw": "^9.69.61",
|
|
36
|
+
"@remnic/export-weclone": "^9.69.61",
|
|
37
|
+
"@remnic/import-weclone": "^9.69.61",
|
|
38
|
+
"@remnic/import-chatgpt": "^9.69.61",
|
|
39
|
+
"@remnic/import-claude": "^9.69.61",
|
|
40
|
+
"@remnic/import-gemini": "^9.69.61",
|
|
41
|
+
"@remnic/import-lossless-claw": "^9.69.61",
|
|
42
|
+
"@remnic/import-mem0": "^9.69.61",
|
|
43
|
+
"@remnic/import-supermemory": "^9.69.61",
|
|
44
|
+
"@remnic/import-okf": "^9.69.61",
|
|
45
|
+
"@remnic/connector-limitless": "^9.69.61",
|
|
46
|
+
"@remnic/connector-bee": "^9.69.61",
|
|
47
|
+
"@remnic/connector-omi": "^9.69.61",
|
|
48
|
+
"@remnic/capture-audio": "^9.69.61"
|
|
49
49
|
},
|
|
50
50
|
"peerDependenciesMeta": {
|
|
51
51
|
"@remnic/bench": {
|
|
@@ -97,19 +97,19 @@
|
|
|
97
97
|
"devDependencies": {
|
|
98
98
|
"tsup": "^8.5.1",
|
|
99
99
|
"typescript": "^5.9.3",
|
|
100
|
-
"@remnic/bench": "9.69.
|
|
101
|
-
"@remnic/plugin-openclaw": "9.69.
|
|
102
|
-
"@remnic/export-weclone": "9.69.
|
|
103
|
-
"@remnic/import-chatgpt": "9.69.
|
|
104
|
-
"@remnic/import-weclone": "9.69.
|
|
105
|
-
"@remnic/import-claude": "9.69.
|
|
106
|
-
"@remnic/import-
|
|
107
|
-
"@remnic/import-
|
|
108
|
-
"@remnic/import-mem0": "9.69.
|
|
109
|
-
"@remnic/import-supermemory": "9.69.
|
|
110
|
-
"@remnic/connector-
|
|
111
|
-
"@remnic/connector-
|
|
112
|
-
"@remnic/connector-omi": "9.69.
|
|
100
|
+
"@remnic/bench": "9.69.61",
|
|
101
|
+
"@remnic/plugin-openclaw": "9.69.61",
|
|
102
|
+
"@remnic/export-weclone": "9.69.61",
|
|
103
|
+
"@remnic/import-chatgpt": "9.69.61",
|
|
104
|
+
"@remnic/import-weclone": "9.69.61",
|
|
105
|
+
"@remnic/import-claude": "9.69.61",
|
|
106
|
+
"@remnic/import-lossless-claw": "9.69.61",
|
|
107
|
+
"@remnic/import-gemini": "9.69.61",
|
|
108
|
+
"@remnic/import-mem0": "9.69.61",
|
|
109
|
+
"@remnic/import-supermemory": "9.69.61",
|
|
110
|
+
"@remnic/connector-bee": "9.69.61",
|
|
111
|
+
"@remnic/connector-limitless": "9.69.61",
|
|
112
|
+
"@remnic/connector-omi": "9.69.61"
|
|
113
113
|
},
|
|
114
114
|
"license": "MIT",
|
|
115
115
|
"repository": {
|