@remnic/cli 9.69.60 → 9.69.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +356 -22
- package/package.json +32 -32
package/dist/index.js
CHANGED
|
@@ -5687,7 +5687,8 @@ import {
|
|
|
5687
5687
|
initLogger as initLogger4,
|
|
5688
5688
|
resolveRemnicConfigRecord as resolveRemnicConfigRecord15,
|
|
5689
5689
|
runAuditMemoryCliCommand,
|
|
5690
|
-
formatAuditMemoryReport
|
|
5690
|
+
formatAuditMemoryReport,
|
|
5691
|
+
auditScreenProfile
|
|
5691
5692
|
} from "@remnic/core";
|
|
5692
5693
|
async function cmdSecurity(rest) {
|
|
5693
5694
|
const action = rest[0] ?? "help";
|
|
@@ -5716,7 +5717,10 @@ async function cmdSecurity(rest) {
|
|
|
5716
5717
|
memoryDir: config.memoryDir,
|
|
5717
5718
|
storage: orchestrator.storage,
|
|
5718
5719
|
since: sinceFlag >= 0 ? rest[sinceFlag + 1] : void 0,
|
|
5719
|
-
quarantine: rest.includes("--quarantine")
|
|
5720
|
+
quarantine: rest.includes("--quarantine"),
|
|
5721
|
+
// Same profile the live write path screens with, and the previous
|
|
5722
|
+
// `default` weighting when that path screens nothing (#3078).
|
|
5723
|
+
profile: auditScreenProfile(config)
|
|
5720
5724
|
});
|
|
5721
5725
|
console.log(json ? JSON.stringify(report, null, 2) : formatAuditMemoryReport(report));
|
|
5722
5726
|
} finally {
|
|
@@ -10231,25 +10235,54 @@ var BENCH_SECURITY_USAGE = `Usage: remnic bench security injection-suite --seeds
|
|
|
10231
10235
|
H5 injection-suite runner. Resume, host-fault pause, multi-host claim
|
|
10232
10236
|
leases, and --limit follow the H6 contract (issue #1963 / PR #2312).
|
|
10233
10237
|
|
|
10238
|
+
Additional commands:
|
|
10239
|
+
remnic bench security injection-suite-analyze --run DIR
|
|
10240
|
+
remnic bench security injection-suite-adaptive-online [injection-suite options]
|
|
10241
|
+
remnic bench security injection-suite-adaptive-online-analyze --run DIR
|
|
10242
|
+
remnic bench security injection-suite-publication-utility --observations FILE --out FILE
|
|
10243
|
+
remnic bench security injection-suite-replay --run DIR
|
|
10244
|
+
remnic bench security injection-suite-utility [injection-suite options] [--dataset-dir DIR]
|
|
10245
|
+
remnic bench security injection-suite-decide --base DIR --base DIR --utility FILE --utility FILE [--adaptive DIR --adaptive DIR] --out DIR
|
|
10246
|
+
|
|
10234
10247
|
Options:
|
|
10235
10248
|
--seeds N Positive seed count (required)
|
|
10249
|
+
--seed-base N First deterministic corpus seed (default: 71)
|
|
10236
10250
|
--variants-per-family N Variants per attack family (default: 25)
|
|
10237
|
-
--
|
|
10251
|
+
--family FAMILY Target one dev/pilot attack family; forbidden for main
|
|
10252
|
+
--stage base|adaptive-r1|adaptive-r2|adaptive-r3|benign|benign-use
|
|
10253
|
+
Frozen corpus stage (default: base)
|
|
10254
|
+
--attacker-executor openai-compat|ollama
|
|
10255
|
+
adaptive-online only: attacker transport
|
|
10256
|
+
--attacker-base-url URL Attacker endpoint (defaults mirror the defended executor)
|
|
10257
|
+
--attacker-model NAME Attacker model id (default: qwen3.8-27b-64k:latest)
|
|
10258
|
+
--attacker-model-digest SHA256
|
|
10259
|
+
Immutable served attacker-model digest
|
|
10260
|
+
--attacker-iterations N K attacker rewrites per base variant (default: 3)
|
|
10261
|
+
--attacker-prompt FILE Frozen attacker system prompt (default: bench fixture)
|
|
10262
|
+
--arm ID Repeat to select frozen defense arms
|
|
10263
|
+
--run-kind dev|pilot|main Run gate to enforce (default: dev)
|
|
10238
10264
|
--executor local|ollama|openai-compat
|
|
10239
10265
|
local = deterministic screen/fence (default)
|
|
10240
10266
|
ollama = native /api/chat
|
|
10241
10267
|
openai-compat = /v1/chat/completions
|
|
10242
|
-
(api.openai.com / integrate.api.nvidia.com
|
|
10243
|
-
https + provider key;
|
|
10244
|
-
|
|
10245
|
-
|
|
10268
|
+
(api.openai.com / integrate.api.nvidia.com /
|
|
10269
|
+
router.huggingface.co require https + provider key;
|
|
10270
|
+
other hosts require REMNIC_OPENAI_COMPAT_API_KEY
|
|
10271
|
+
and https, except loopback HTTP)
|
|
10246
10272
|
--base-url URL Endpoint (default: http://127.0.0.1:11434)
|
|
10247
10273
|
--model NAME Model id (default: qwen3.8-27b-64k:latest)
|
|
10274
|
+
--model-digest SHA256 Immutable served-model digest (required for main)
|
|
10275
|
+
--model-context-tokens N Native context length (required for main)
|
|
10248
10276
|
--request-timeout-ms N Per-call timeout (default: 300000)
|
|
10249
10277
|
--out DIR New run directory (default: ~/.remnic/bench/results/h5-injection-suite)
|
|
10250
10278
|
--run DIR Existing run directory; implies --resume
|
|
10279
|
+
--capture-responses Write raw responses.jsonl beside the checkpoints
|
|
10251
10280
|
--resume Continue an existing run (required if DIR already has run.json)
|
|
10281
|
+
--retry-ambiguous Retry a persisted in-flight paid request after operator review
|
|
10252
10282
|
--limit N Execute at most N planned rows (dry-run / smoke)
|
|
10283
|
+
--utility-benchmark ID Repeat: locomo, longmemeval, or drift-gen
|
|
10284
|
+
--longmemeval-dataset-dir DIR
|
|
10285
|
+
Frozen LongMemEval dataset for utility runs
|
|
10253
10286
|
`;
|
|
10254
10287
|
function parsePositiveInteger(raw, flag) {
|
|
10255
10288
|
const value = Number(raw);
|
|
@@ -10259,33 +10292,85 @@ function parsePositiveInteger(raw, flag) {
|
|
|
10259
10292
|
return value;
|
|
10260
10293
|
}
|
|
10261
10294
|
function parseBenchSecurityArgs(args) {
|
|
10262
|
-
if (args.length === 0 || args[0] === "--help" || args[0] === "-h")
|
|
10295
|
+
if (args.length === 0 || args[0] === "--help" || args[0] === "-h")
|
|
10296
|
+
return { help: true };
|
|
10263
10297
|
if (args[0] !== "injection-suite") {
|
|
10264
10298
|
throw new Error(`unknown bench security subcommand ${args[0]}`);
|
|
10265
10299
|
}
|
|
10266
10300
|
let seeds;
|
|
10301
|
+
let seedBase;
|
|
10267
10302
|
let variantsPerFamily = 25;
|
|
10303
|
+
let family;
|
|
10268
10304
|
let modelProfileId = "local-dry";
|
|
10305
|
+
let stage = "base";
|
|
10306
|
+
const arms = [];
|
|
10307
|
+
let runKind = "dev";
|
|
10269
10308
|
let outputDir = DEFAULT_OUTPUT_DIR;
|
|
10270
10309
|
let resume = false;
|
|
10310
|
+
let retryAmbiguous = false;
|
|
10271
10311
|
let limit;
|
|
10272
10312
|
let executor = "local";
|
|
10313
|
+
let datasetDir;
|
|
10314
|
+
let longmemevalDatasetDir;
|
|
10315
|
+
const utilityBenchmarks = [];
|
|
10273
10316
|
let baseUrl;
|
|
10274
10317
|
let model;
|
|
10318
|
+
let modelDigest;
|
|
10319
|
+
let modelContextTokens;
|
|
10275
10320
|
let requestTimeoutMs;
|
|
10321
|
+
let captureResponses = false;
|
|
10322
|
+
let attackerExecutor;
|
|
10323
|
+
let attackerBaseUrl;
|
|
10324
|
+
let attackerModel;
|
|
10325
|
+
let attackerModelDigest;
|
|
10326
|
+
let attackerIterations;
|
|
10327
|
+
let attackerPromptPath;
|
|
10276
10328
|
for (let index = 1; index < args.length; index += 1) {
|
|
10277
10329
|
const flag = args[index] ?? "";
|
|
10278
10330
|
const next = args[index + 1];
|
|
10279
10331
|
if (flag === "--seeds") {
|
|
10280
10332
|
seeds = parsePositiveInteger(next, "--seeds");
|
|
10281
10333
|
index += 1;
|
|
10334
|
+
} else if (flag === "--seed-base") {
|
|
10335
|
+
seedBase = parsePositiveInteger(next, "--seed-base");
|
|
10336
|
+
index += 1;
|
|
10282
10337
|
} else if (flag === "--variants-per-family") {
|
|
10283
10338
|
variantsPerFamily = parsePositiveInteger(next, "--variants-per-family");
|
|
10284
10339
|
index += 1;
|
|
10340
|
+
} else if (flag === "--family") {
|
|
10341
|
+
if (next !== "minja" && next !== "sleeper" && next !== "cross-session" && next !== "tool-hijack") {
|
|
10342
|
+
throw new Error(
|
|
10343
|
+
"--family must be minja, sleeper, cross-session, or tool-hijack"
|
|
10344
|
+
);
|
|
10345
|
+
}
|
|
10346
|
+
family = next;
|
|
10347
|
+
index += 1;
|
|
10285
10348
|
} else if (flag === "--model-profile") {
|
|
10286
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10349
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10350
|
+
throw new Error("missing value for --model-profile");
|
|
10287
10351
|
modelProfileId = next;
|
|
10288
10352
|
index += 1;
|
|
10353
|
+
} else if (flag === "--stage") {
|
|
10354
|
+
if (next !== "base" && next !== "adaptive-r1" && next !== "adaptive-r2" && next !== "adaptive-r3" && next !== "benign" && next !== "benign-use" && next !== "adaptive-online-r1") {
|
|
10355
|
+
throw new Error(
|
|
10356
|
+
"--stage must be base, adaptive-r1, adaptive-r2, adaptive-r3, benign, benign-use, or adaptive-online-r1"
|
|
10357
|
+
);
|
|
10358
|
+
}
|
|
10359
|
+
stage = next;
|
|
10360
|
+
index += 1;
|
|
10361
|
+
} else if (flag === "--arm") {
|
|
10362
|
+
if (next !== "none" && next !== "fencing" && next !== "quarantine" && next !== "both" && next !== "structured-boundary" && next !== "spotlighting-marking" && next !== "source-authenticated-fencing" && next !== "control-data-isolation" && next !== "layered-fence-quarantine") {
|
|
10363
|
+
throw new Error("unknown --arm defense baseline");
|
|
10364
|
+
}
|
|
10365
|
+
if (arms.includes(next)) throw new Error(`duplicate --arm ${next}`);
|
|
10366
|
+
arms.push(next);
|
|
10367
|
+
index += 1;
|
|
10368
|
+
} else if (flag === "--run-kind") {
|
|
10369
|
+
if (next !== "dev" && next !== "pilot" && next !== "main") {
|
|
10370
|
+
throw new Error("--run-kind must be dev, pilot, or main");
|
|
10371
|
+
}
|
|
10372
|
+
runKind = next;
|
|
10373
|
+
index += 1;
|
|
10289
10374
|
} else if (flag === "--executor") {
|
|
10290
10375
|
if (next !== "local" && next !== "ollama" && next !== "openai-compat") {
|
|
10291
10376
|
throw new Error("--executor must be local, ollama, or openai-compat");
|
|
@@ -10293,72 +10378,321 @@ function parseBenchSecurityArgs(args) {
|
|
|
10293
10378
|
executor = next;
|
|
10294
10379
|
index += 1;
|
|
10295
10380
|
} else if (flag === "--base-url") {
|
|
10296
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10381
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10382
|
+
throw new Error("missing value for --base-url");
|
|
10297
10383
|
baseUrl = next;
|
|
10298
10384
|
index += 1;
|
|
10299
10385
|
} else if (flag === "--model") {
|
|
10300
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10386
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10387
|
+
throw new Error("missing value for --model");
|
|
10301
10388
|
model = next;
|
|
10302
10389
|
index += 1;
|
|
10390
|
+
} else if (flag === "--model-digest") {
|
|
10391
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10392
|
+
throw new Error("missing value for --model-digest");
|
|
10393
|
+
modelDigest = next;
|
|
10394
|
+
index += 1;
|
|
10395
|
+
} else if (flag === "--dataset-dir") {
|
|
10396
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10397
|
+
throw new Error("missing value for --dataset-dir");
|
|
10398
|
+
datasetDir = expandTilde(next);
|
|
10399
|
+
index += 1;
|
|
10400
|
+
} else if (flag === "--longmemeval-dataset-dir") {
|
|
10401
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10402
|
+
throw new Error("missing value for --longmemeval-dataset-dir");
|
|
10403
|
+
longmemevalDatasetDir = expandTilde(next);
|
|
10404
|
+
index += 1;
|
|
10405
|
+
} else if (flag === "--utility-benchmark") {
|
|
10406
|
+
if (next !== "locomo" && next !== "longmemeval" && next !== "drift-gen") {
|
|
10407
|
+
throw new Error(
|
|
10408
|
+
"--utility-benchmark must be locomo, longmemeval, or drift-gen"
|
|
10409
|
+
);
|
|
10410
|
+
}
|
|
10411
|
+
utilityBenchmarks.push(next);
|
|
10412
|
+
index += 1;
|
|
10413
|
+
} else if (flag === "--model-context-tokens") {
|
|
10414
|
+
modelContextTokens = parsePositiveInteger(next, "--model-context-tokens");
|
|
10415
|
+
index += 1;
|
|
10303
10416
|
} else if (flag === "--request-timeout-ms") {
|
|
10304
10417
|
requestTimeoutMs = parsePositiveInteger(next, "--request-timeout-ms");
|
|
10305
10418
|
index += 1;
|
|
10306
10419
|
} else if (flag === "--out") {
|
|
10307
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10420
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10421
|
+
throw new Error("missing value for --out");
|
|
10308
10422
|
outputDir = expandTilde(next);
|
|
10309
10423
|
index += 1;
|
|
10310
10424
|
} else if (flag === "--run") {
|
|
10311
|
-
if (next === void 0 || next.startsWith("-"))
|
|
10425
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10426
|
+
throw new Error("missing value for --run");
|
|
10312
10427
|
outputDir = expandTilde(next);
|
|
10313
10428
|
resume = true;
|
|
10314
10429
|
index += 1;
|
|
10315
10430
|
} else if (flag === "--resume") {
|
|
10316
10431
|
resume = true;
|
|
10432
|
+
} else if (flag === "--retry-ambiguous") {
|
|
10433
|
+
retryAmbiguous = true;
|
|
10317
10434
|
} else if (flag === "--limit") {
|
|
10318
10435
|
limit = parsePositiveInteger(next, "--limit");
|
|
10319
10436
|
index += 1;
|
|
10437
|
+
} else if (flag === "--capture-responses") {
|
|
10438
|
+
captureResponses = true;
|
|
10439
|
+
} else if (flag === "--attacker-executor") {
|
|
10440
|
+
if (next !== "openai-compat" && next !== "ollama") {
|
|
10441
|
+
throw new Error("--attacker-executor must be openai-compat or ollama");
|
|
10442
|
+
}
|
|
10443
|
+
attackerExecutor = next;
|
|
10444
|
+
index += 1;
|
|
10445
|
+
} else if (flag === "--attacker-base-url") {
|
|
10446
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10447
|
+
throw new Error("missing value for --attacker-base-url");
|
|
10448
|
+
attackerBaseUrl = next;
|
|
10449
|
+
index += 1;
|
|
10450
|
+
} else if (flag === "--attacker-model") {
|
|
10451
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10452
|
+
throw new Error("missing value for --attacker-model");
|
|
10453
|
+
attackerModel = next;
|
|
10454
|
+
index += 1;
|
|
10455
|
+
} else if (flag === "--attacker-model-digest") {
|
|
10456
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10457
|
+
throw new Error("missing value for --attacker-model-digest");
|
|
10458
|
+
attackerModelDigest = next;
|
|
10459
|
+
index += 1;
|
|
10460
|
+
} else if (flag === "--attacker-iterations") {
|
|
10461
|
+
attackerIterations = parsePositiveInteger(next, "--attacker-iterations");
|
|
10462
|
+
index += 1;
|
|
10463
|
+
} else if (flag === "--attacker-prompt") {
|
|
10464
|
+
if (next === void 0 || next.startsWith("-"))
|
|
10465
|
+
throw new Error("missing value for --attacker-prompt");
|
|
10466
|
+
attackerPromptPath = expandTilde(next);
|
|
10467
|
+
index += 1;
|
|
10320
10468
|
} else {
|
|
10321
10469
|
throw new Error(`unknown option ${flag}`);
|
|
10322
10470
|
}
|
|
10323
10471
|
}
|
|
10324
|
-
if (seeds === void 0)
|
|
10472
|
+
if (seeds === void 0)
|
|
10473
|
+
throw new Error("injection-suite requires --seeds N");
|
|
10474
|
+
if (retryAmbiguous && !resume)
|
|
10475
|
+
throw new Error("--retry-ambiguous requires --run or --resume");
|
|
10325
10476
|
return {
|
|
10326
10477
|
seeds,
|
|
10478
|
+
...seedBase === void 0 ? {} : { seedBase },
|
|
10327
10479
|
variantsPerFamily,
|
|
10480
|
+
...family === void 0 ? {} : { family },
|
|
10328
10481
|
modelProfileId,
|
|
10482
|
+
stage,
|
|
10483
|
+
...arms.length === 0 ? {} : { arms },
|
|
10484
|
+
runKind,
|
|
10329
10485
|
outputDir,
|
|
10330
10486
|
resume,
|
|
10331
10487
|
executor,
|
|
10488
|
+
retryAmbiguous,
|
|
10332
10489
|
...limit === void 0 ? {} : { limit },
|
|
10333
10490
|
...baseUrl === void 0 ? {} : { baseUrl },
|
|
10334
10491
|
...model === void 0 ? {} : { model },
|
|
10335
|
-
...
|
|
10492
|
+
...modelDigest === void 0 ? {} : { modelDigest },
|
|
10493
|
+
...modelContextTokens === void 0 ? {} : { modelContextTokens },
|
|
10494
|
+
...datasetDir === void 0 ? {} : { datasetDir },
|
|
10495
|
+
...longmemevalDatasetDir === void 0 ? {} : { longmemevalDatasetDir },
|
|
10496
|
+
...utilityBenchmarks.length === 0 ? {} : { utilityBenchmarks },
|
|
10497
|
+
...requestTimeoutMs === void 0 ? {} : { requestTimeoutMs },
|
|
10498
|
+
captureResponses,
|
|
10499
|
+
...attackerExecutor === void 0 ? {} : { attackerExecutor },
|
|
10500
|
+
...attackerBaseUrl === void 0 ? {} : { attackerBaseUrl },
|
|
10501
|
+
...attackerModel === void 0 ? {} : { attackerModel },
|
|
10502
|
+
...attackerModelDigest === void 0 ? {} : { attackerModelDigest },
|
|
10503
|
+
...attackerIterations === void 0 ? {} : { attackerIterations },
|
|
10504
|
+
...attackerPromptPath === void 0 ? {} : { attackerPromptPath }
|
|
10336
10505
|
};
|
|
10337
10506
|
}
|
|
10338
10507
|
async function cmdBenchSecurity(args) {
|
|
10339
10508
|
try {
|
|
10340
|
-
|
|
10509
|
+
if (args[0] === "injection-suite-publication-utility") {
|
|
10510
|
+
if (args.length !== 5 || args[1] !== "--observations" || !args[2] || args[3] !== "--out" || !args[4]) {
|
|
10511
|
+
throw new Error(
|
|
10512
|
+
"injection-suite-publication-utility requires --observations FILE --out FILE"
|
|
10513
|
+
);
|
|
10514
|
+
}
|
|
10515
|
+
const bench2 = await loadBenchModule();
|
|
10516
|
+
const analyze = bench2.analyzeInjectionSuitePublicationUtilityFile;
|
|
10517
|
+
if (typeof analyze !== "function")
|
|
10518
|
+
throw new Error(
|
|
10519
|
+
"Installed @remnic/bench lacks H5 publication utility analysis"
|
|
10520
|
+
);
|
|
10521
|
+
const analysis = await analyze(
|
|
10522
|
+
expandTilde(args[2]),
|
|
10523
|
+
expandTilde(args[4])
|
|
10524
|
+
);
|
|
10525
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10526
|
+
return;
|
|
10527
|
+
}
|
|
10528
|
+
if (args[0] === "injection-suite-decide") {
|
|
10529
|
+
const baseRunDirs = [];
|
|
10530
|
+
const utilityStatisticsPaths = [];
|
|
10531
|
+
const adaptiveRunDirs = [];
|
|
10532
|
+
let outputDir;
|
|
10533
|
+
for (let index = 1; index < args.length; index += 2) {
|
|
10534
|
+
const flag = args[index];
|
|
10535
|
+
const value = args[index + 1];
|
|
10536
|
+
if (!value)
|
|
10537
|
+
throw new Error(`missing value for ${flag ?? "campaign flag"}`);
|
|
10538
|
+
if (flag === "--base") baseRunDirs.push(expandTilde(value));
|
|
10539
|
+
else if (flag === "--utility")
|
|
10540
|
+
utilityStatisticsPaths.push(expandTilde(value));
|
|
10541
|
+
else if (flag === "--adaptive")
|
|
10542
|
+
adaptiveRunDirs.push(expandTilde(value));
|
|
10543
|
+
else if (flag === "--out") outputDir = expandTilde(value);
|
|
10544
|
+
else throw new Error(`unknown campaign option ${flag}`);
|
|
10545
|
+
}
|
|
10546
|
+
if (!outputDir)
|
|
10547
|
+
throw new Error("injection-suite-decide requires --out DIR");
|
|
10548
|
+
const campaignBench = await loadBenchModule();
|
|
10549
|
+
const decide = campaignBench.decideInjectionSuiteCampaign;
|
|
10550
|
+
if (typeof decide !== "function")
|
|
10551
|
+
throw new Error("Installed @remnic/bench lacks H5 campaign decision");
|
|
10552
|
+
const decision = await decide({
|
|
10553
|
+
baseRunDirs,
|
|
10554
|
+
utilityStatisticsPaths,
|
|
10555
|
+
...adaptiveRunDirs.length > 0 ? { adaptiveRunDirs } : {},
|
|
10556
|
+
outputDir
|
|
10557
|
+
});
|
|
10558
|
+
console.log(JSON.stringify(decision, null, 2));
|
|
10559
|
+
return;
|
|
10560
|
+
}
|
|
10561
|
+
if (args[0] === "injection-suite-analyze" || args[0] === "injection-suite-adaptive-online-analyze" || args[0] === "injection-suite-publication-analyze" || args[0] === "injection-suite-replay") {
|
|
10562
|
+
if (args.length !== 3 || args[1] !== "--run" || !args[2]) {
|
|
10563
|
+
throw new Error(`${args[0]} requires --run DIR`);
|
|
10564
|
+
}
|
|
10565
|
+
const analysisBench = await loadBenchModule();
|
|
10566
|
+
const runDir = expandTilde(args[2]);
|
|
10567
|
+
if (args[0] === "injection-suite-adaptive-online-analyze") {
|
|
10568
|
+
const analyze = analysisBench.analyzeInjectionSuiteOnlineAdaptiveRun;
|
|
10569
|
+
if (typeof analyze !== "function")
|
|
10570
|
+
throw new Error(
|
|
10571
|
+
"Installed @remnic/bench lacks the adaptive-online analysis"
|
|
10572
|
+
);
|
|
10573
|
+
const analysis = await analyze(runDir);
|
|
10574
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10575
|
+
} else if (args[0] === "injection-suite-analyze") {
|
|
10576
|
+
const analyze = analysisBench.analyzeInjectionSuiteRun;
|
|
10577
|
+
if (typeof analyze !== "function")
|
|
10578
|
+
throw new Error("Installed @remnic/bench lacks H5 analysis");
|
|
10579
|
+
const analysis = await analyze(runDir);
|
|
10580
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10581
|
+
} else if (args[0] === "injection-suite-publication-analyze") {
|
|
10582
|
+
const analyze = analysisBench.analyzeInjectionSuitePublicationRun;
|
|
10583
|
+
if (typeof analyze !== "function")
|
|
10584
|
+
throw new Error(
|
|
10585
|
+
"Installed @remnic/bench lacks H5 publication analysis"
|
|
10586
|
+
);
|
|
10587
|
+
const analysis = await analyze(runDir);
|
|
10588
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10589
|
+
} else {
|
|
10590
|
+
const replay = analysisBench.replayInjectionSuiteStatistics;
|
|
10591
|
+
if (typeof replay !== "function")
|
|
10592
|
+
throw new Error("Installed @remnic/bench lacks H5 replay");
|
|
10593
|
+
await replay(runDir);
|
|
10594
|
+
console.log(JSON.stringify({ replay: "ok", runDir }));
|
|
10595
|
+
}
|
|
10596
|
+
return;
|
|
10597
|
+
}
|
|
10598
|
+
const utilityMode = args[0] === "injection-suite-utility";
|
|
10599
|
+
const onlineMode = args[0] === "injection-suite-adaptive-online";
|
|
10600
|
+
const parsed = parseBenchSecurityArgs(
|
|
10601
|
+
utilityMode || onlineMode ? ["injection-suite", ...args.slice(1)] : args
|
|
10602
|
+
);
|
|
10341
10603
|
if ("help" in parsed) {
|
|
10342
10604
|
console.log(BENCH_SECURITY_USAGE);
|
|
10343
10605
|
return;
|
|
10344
10606
|
}
|
|
10345
10607
|
const bench = await loadBenchModule();
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10608
|
+
if (utilityMode) {
|
|
10609
|
+
const utility = bench.runInjectionSuiteUtility;
|
|
10610
|
+
if (typeof utility !== "function")
|
|
10611
|
+
throw new Error("Installed @remnic/bench lacks H5 utility runner");
|
|
10612
|
+
const analysis = await utility({
|
|
10613
|
+
seeds: parsed.seeds,
|
|
10614
|
+
variantsPerFamily: parsed.variantsPerFamily,
|
|
10615
|
+
modelProfileId: parsed.modelProfileId,
|
|
10616
|
+
outputDir: parsed.outputDir,
|
|
10617
|
+
executor: parsed.executor,
|
|
10618
|
+
stage: parsed.stage,
|
|
10619
|
+
runKind: parsed.runKind,
|
|
10620
|
+
...parsed.resume ? { resume: true } : {},
|
|
10621
|
+
...parsed.retryAmbiguous ? { retryAmbiguous: true } : {},
|
|
10622
|
+
...parsed.limit === void 0 ? {} : { limit: parsed.limit },
|
|
10623
|
+
...parsed.baseUrl === void 0 ? {} : { baseUrl: parsed.baseUrl },
|
|
10624
|
+
...parsed.model === void 0 ? {} : { model: parsed.model },
|
|
10625
|
+
...parsed.modelDigest === void 0 ? {} : { modelDigest: parsed.modelDigest },
|
|
10626
|
+
...parsed.modelContextTokens === void 0 ? {} : { modelContextTokens: parsed.modelContextTokens },
|
|
10627
|
+
...parsed.requestTimeoutMs === void 0 ? {} : { requestTimeoutMs: parsed.requestTimeoutMs },
|
|
10628
|
+
...parsed.datasetDir === void 0 ? {} : { locomoDatasetDir: parsed.datasetDir },
|
|
10629
|
+
...parsed.longmemevalDatasetDir === void 0 ? {} : { longmemevalDatasetDir: parsed.longmemevalDatasetDir },
|
|
10630
|
+
...parsed.utilityBenchmarks === void 0 ? {} : { utilityBenchmarks: parsed.utilityBenchmarks },
|
|
10631
|
+
...parsed.arms === void 0 ? {} : { arms: parsed.arms }
|
|
10632
|
+
});
|
|
10633
|
+
console.log(JSON.stringify(analysis, null, 2));
|
|
10634
|
+
return;
|
|
10349
10635
|
}
|
|
10350
|
-
const
|
|
10636
|
+
const suiteInput = {
|
|
10351
10637
|
seeds: parsed.seeds,
|
|
10638
|
+
...parsed.seedBase === void 0 ? {} : { seedBase: parsed.seedBase },
|
|
10352
10639
|
variantsPerFamily: parsed.variantsPerFamily,
|
|
10353
10640
|
modelProfileId: parsed.modelProfileId,
|
|
10354
10641
|
outputDir: parsed.outputDir,
|
|
10355
10642
|
executor: parsed.executor,
|
|
10643
|
+
stage: parsed.stage,
|
|
10644
|
+
runKind: parsed.runKind,
|
|
10645
|
+
...parsed.arms === void 0 ? {} : { arms: parsed.arms },
|
|
10646
|
+
...parsed.family === void 0 ? {} : { family: parsed.family },
|
|
10356
10647
|
...parsed.resume ? { resume: true } : {},
|
|
10648
|
+
...parsed.retryAmbiguous ? { retryAmbiguous: true } : {},
|
|
10357
10649
|
...parsed.limit === void 0 ? {} : { limit: parsed.limit },
|
|
10358
10650
|
...parsed.baseUrl === void 0 ? {} : { baseUrl: parsed.baseUrl },
|
|
10359
10651
|
...parsed.model === void 0 ? {} : { model: parsed.model },
|
|
10360
|
-
...parsed.
|
|
10361
|
-
|
|
10652
|
+
...parsed.modelDigest === void 0 ? {} : { modelDigest: parsed.modelDigest },
|
|
10653
|
+
...parsed.modelContextTokens === void 0 ? {} : { modelContextTokens: parsed.modelContextTokens },
|
|
10654
|
+
...parsed.requestTimeoutMs === void 0 ? {} : { requestTimeoutMs: parsed.requestTimeoutMs },
|
|
10655
|
+
...parsed.captureResponses ? { captureResponses: true } : {}
|
|
10656
|
+
};
|
|
10657
|
+
if (onlineMode) {
|
|
10658
|
+
const runOnline = bench.runInjectionSuiteOnlineAdaptive;
|
|
10659
|
+
if (typeof runOnline !== "function") {
|
|
10660
|
+
throw new Error(
|
|
10661
|
+
"Installed @remnic/bench lacks runInjectionSuiteOnlineAdaptive"
|
|
10662
|
+
);
|
|
10663
|
+
}
|
|
10664
|
+
if (parsed.executor === "local") {
|
|
10665
|
+
throw new Error(
|
|
10666
|
+
"adaptive-online-r1 requires --executor ollama or openai-compat (local executor cannot run product rows)"
|
|
10667
|
+
);
|
|
10668
|
+
}
|
|
10669
|
+
const attackerPromptPath = parsed.attackerPromptPath ?? bench.DEFAULT_ATTACKER_PROMPT_PATH_GETTER();
|
|
10670
|
+
const result2 = await runOnline({
|
|
10671
|
+
...suiteInput,
|
|
10672
|
+
attackerExecutor: parsed.attackerExecutor ?? "openai-compat",
|
|
10673
|
+
attackerIterations: parsed.attackerIterations ?? 3,
|
|
10674
|
+
attackerPromptPath,
|
|
10675
|
+
...parsed.attackerBaseUrl === void 0 ? {} : { attackerBaseUrl: parsed.attackerBaseUrl },
|
|
10676
|
+
...parsed.attackerModel === void 0 ? {} : { attackerModel: parsed.attackerModel },
|
|
10677
|
+
...parsed.attackerModelDigest === void 0 ? {} : { attackerModelDigest: parsed.attackerModelDigest }
|
|
10678
|
+
});
|
|
10679
|
+
if (result2.exitCode === 0) console.log(result2.output);
|
|
10680
|
+
else console.error(result2.output);
|
|
10681
|
+
if (result2.exitCode !== 0) process.exitCode = result2.exitCode;
|
|
10682
|
+
return;
|
|
10683
|
+
}
|
|
10684
|
+
if (parsed.stage === "adaptive-online-r1") {
|
|
10685
|
+
throw new Error(
|
|
10686
|
+
"stage adaptive-online-r1 requires the injection-suite-adaptive-online command"
|
|
10687
|
+
);
|
|
10688
|
+
}
|
|
10689
|
+
const run = bench.runInjectionSuiteCliCommand;
|
|
10690
|
+
if (typeof run !== "function") {
|
|
10691
|
+
throw new Error(
|
|
10692
|
+
"Installed @remnic/bench is missing runInjectionSuiteCliCommand"
|
|
10693
|
+
);
|
|
10694
|
+
}
|
|
10695
|
+
const result = await run(suiteInput);
|
|
10362
10696
|
if (result.exitCode === 0) console.log(result.output);
|
|
10363
10697
|
else console.error(result.output);
|
|
10364
10698
|
if (result.exitCode !== 0) process.exitCode = result.exitCode;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/cli",
|
|
3
|
-
"version": "9.69.
|
|
3
|
+
"version": "9.69.63",
|
|
4
4
|
"description": "CLI for Remnic memory — init, query, doctor, daemon management",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -26,26 +26,26 @@
|
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"yaml": "^2.4.2",
|
|
29
|
-
"@remnic/plugin-pi": "^9.69.
|
|
30
|
-
"@remnic/server": "^9.69.
|
|
31
|
-
"@remnic/core": "^9.69.
|
|
29
|
+
"@remnic/plugin-pi": "^9.69.63",
|
|
30
|
+
"@remnic/server": "^9.69.63",
|
|
31
|
+
"@remnic/core": "^9.69.63"
|
|
32
32
|
},
|
|
33
33
|
"peerDependencies": {
|
|
34
|
-
"@remnic/bench": "^9.69.
|
|
35
|
-
"@remnic/plugin-openclaw": "^9.69.
|
|
36
|
-
"@remnic/export-weclone": "^9.69.
|
|
37
|
-
"@remnic/import-weclone": "^9.69.
|
|
38
|
-
"@remnic/import-chatgpt": "^9.69.
|
|
39
|
-
"@remnic/import-claude": "^9.69.
|
|
40
|
-
"@remnic/import-gemini": "^9.69.
|
|
41
|
-
"@remnic/import-lossless-claw": "^9.69.
|
|
42
|
-
"@remnic/import-mem0": "^9.69.
|
|
43
|
-
"@remnic/import-supermemory": "^9.69.
|
|
44
|
-
"@remnic/import-okf": "^9.69.
|
|
45
|
-
"@remnic/connector-limitless": "^9.69.
|
|
46
|
-
"@remnic/connector-bee": "^9.69.
|
|
47
|
-
"@remnic/connector-omi": "^9.69.
|
|
48
|
-
"@remnic/capture-audio": "^9.69.
|
|
34
|
+
"@remnic/bench": "^9.69.63",
|
|
35
|
+
"@remnic/plugin-openclaw": "^9.69.63",
|
|
36
|
+
"@remnic/export-weclone": "^9.69.63",
|
|
37
|
+
"@remnic/import-weclone": "^9.69.63",
|
|
38
|
+
"@remnic/import-chatgpt": "^9.69.63",
|
|
39
|
+
"@remnic/import-claude": "^9.69.63",
|
|
40
|
+
"@remnic/import-gemini": "^9.69.63",
|
|
41
|
+
"@remnic/import-lossless-claw": "^9.69.63",
|
|
42
|
+
"@remnic/import-mem0": "^9.69.63",
|
|
43
|
+
"@remnic/import-supermemory": "^9.69.63",
|
|
44
|
+
"@remnic/import-okf": "^9.69.63",
|
|
45
|
+
"@remnic/connector-limitless": "^9.69.63",
|
|
46
|
+
"@remnic/connector-bee": "^9.69.63",
|
|
47
|
+
"@remnic/connector-omi": "^9.69.63",
|
|
48
|
+
"@remnic/capture-audio": "^9.69.63"
|
|
49
49
|
},
|
|
50
50
|
"peerDependenciesMeta": {
|
|
51
51
|
"@remnic/bench": {
|
|
@@ -97,19 +97,19 @@
|
|
|
97
97
|
"devDependencies": {
|
|
98
98
|
"tsup": "^8.5.1",
|
|
99
99
|
"typescript": "^5.9.3",
|
|
100
|
-
"@remnic/bench": "9.69.
|
|
101
|
-
"@remnic/plugin-openclaw": "9.69.
|
|
102
|
-
"@remnic/export-weclone": "9.69.
|
|
103
|
-
"@remnic/import-chatgpt": "9.69.
|
|
104
|
-
"@remnic/import-weclone": "9.69.
|
|
105
|
-
"@remnic/import-
|
|
106
|
-
"@remnic/import-
|
|
107
|
-
"@remnic/import-
|
|
108
|
-
"@remnic/import-
|
|
109
|
-
"@remnic/import-supermemory": "9.69.
|
|
110
|
-
"@remnic/connector-limitless": "9.69.
|
|
111
|
-
"@remnic/connector-
|
|
112
|
-
"@remnic/connector-
|
|
100
|
+
"@remnic/bench": "9.69.63",
|
|
101
|
+
"@remnic/plugin-openclaw": "9.69.63",
|
|
102
|
+
"@remnic/export-weclone": "9.69.63",
|
|
103
|
+
"@remnic/import-chatgpt": "9.69.63",
|
|
104
|
+
"@remnic/import-weclone": "9.69.63",
|
|
105
|
+
"@remnic/import-gemini": "9.69.63",
|
|
106
|
+
"@remnic/import-lossless-claw": "9.69.63",
|
|
107
|
+
"@remnic/import-mem0": "9.69.63",
|
|
108
|
+
"@remnic/import-claude": "9.69.63",
|
|
109
|
+
"@remnic/import-supermemory": "9.69.63",
|
|
110
|
+
"@remnic/connector-limitless": "9.69.63",
|
|
111
|
+
"@remnic/connector-omi": "9.69.63",
|
|
112
|
+
"@remnic/connector-bee": "9.69.63"
|
|
113
113
|
},
|
|
114
114
|
"license": "MIT",
|
|
115
115
|
"repository": {
|