@remnic/bench 9.69.89 → 9.69.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +102 -1
- package/dist/index.js +1257 -229
- package/fixtures/h1-outcome/README.md +65 -0
- package/fixtures/h1-outcome/arms/h1-w0.json +19 -0
- package/fixtures/h1-outcome/arms/h1-w015.json +19 -0
- package/fixtures/h1-outcome/arms/h1-w030.json +19 -0
- package/fixtures/h1-outcome/arms/h1-w050.json +19 -0
- package/fixtures/h1-outcome/arms/memory-worth-base.json +15 -0
- package/fixtures/h1-outcome/corpus-ci/21/gold/facts.jsonl +64 -0
- package/fixtures/h1-outcome/corpus-ci/21/gold/probes.jsonl +62 -0
- package/fixtures/h1-outcome/corpus-ci/21/users/u1/sessions.jsonl +4 -0
- package/fixtures/h1-outcome/corpus-ci/21/users/u2/sessions.jsonl +4 -0
- package/fixtures/h1-outcome/corpus-ci/dataset.manifest.json +32 -0
- package/fixtures/h1-outcome/corpus-ci/tasks.jsonl +12 -0
- package/fixtures/h1-outcome/corpus-ci/warm-store.json +1 -0
- package/fixtures/h1-outcome/decision-rule.json +27 -0
- package/fixtures/h1-outcome/regenerate.ts +34 -0
- package/fixtures/h1-outcome/tasks.jsonl +120 -0
- package/package.json +6 -4
package/dist/index.js
CHANGED
|
@@ -31607,16 +31607,16 @@ async function runIngestionEntityRecallBenchmark(options) {
|
|
|
31607
31607
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31608
31608
|
}
|
|
31609
31609
|
const fixture = emailFixture.generate();
|
|
31610
|
-
const
|
|
31610
|
+
const fixtureDir2 = await mkdtemp7(path21.join(tmpdir2(), "bench-email-"));
|
|
31611
31611
|
try {
|
|
31612
31612
|
await options.ingestionAdapter.reset();
|
|
31613
31613
|
for (const file of fixture.files) {
|
|
31614
|
-
const filePath = path21.join(
|
|
31614
|
+
const filePath = path21.join(fixtureDir2, file.relativePath);
|
|
31615
31615
|
await mkdir6(path21.dirname(filePath), { recursive: true });
|
|
31616
31616
|
await writeFile6(filePath, file.content, "utf8");
|
|
31617
31617
|
}
|
|
31618
31618
|
const { result: ingestionLog, durationMs } = await timed(
|
|
31619
|
-
async () => options.ingestionAdapter.ingest(await realpath5(
|
|
31619
|
+
async () => options.ingestionAdapter.ingest(await realpath5(fixtureDir2))
|
|
31620
31620
|
);
|
|
31621
31621
|
if (ingestionLog.errors.length > 0) {
|
|
31622
31622
|
const message = ingestionLog.errors.join(" | ");
|
|
@@ -31694,7 +31694,7 @@ async function runIngestionEntityRecallBenchmark(options) {
|
|
|
31694
31694
|
];
|
|
31695
31695
|
return buildResult(options, tasks, durationMs);
|
|
31696
31696
|
} finally {
|
|
31697
|
-
await rm8(
|
|
31697
|
+
await rm8(fixtureDir2, { recursive: true, force: true });
|
|
31698
31698
|
}
|
|
31699
31699
|
}
|
|
31700
31700
|
async function buildResult(options, tasks, totalLatencyMs) {
|
|
@@ -31761,16 +31761,16 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
31761
31761
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31762
31762
|
}
|
|
31763
31763
|
const fixture = emailFixture.generate();
|
|
31764
|
-
const
|
|
31764
|
+
const fixtureDir2 = await mkdtemp8(path22.join(tmpdir3(), "bench-email-"));
|
|
31765
31765
|
try {
|
|
31766
31766
|
await options.ingestionAdapter.reset();
|
|
31767
31767
|
for (const file of fixture.files) {
|
|
31768
|
-
const filePath = path22.join(
|
|
31768
|
+
const filePath = path22.join(fixtureDir2, file.relativePath);
|
|
31769
31769
|
await mkdir7(path22.dirname(filePath), { recursive: true });
|
|
31770
31770
|
await writeFile7(filePath, file.content, "utf8");
|
|
31771
31771
|
}
|
|
31772
31772
|
const { result: ingestionLog, durationMs } = await timed(
|
|
31773
|
-
async () => options.ingestionAdapter.ingest(await realpath6(
|
|
31773
|
+
async () => options.ingestionAdapter.ingest(await realpath6(fixtureDir2))
|
|
31774
31774
|
);
|
|
31775
31775
|
if (ingestionLog.errors.length > 0) {
|
|
31776
31776
|
const message = ingestionLog.errors.join(" | ");
|
|
@@ -31907,7 +31907,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
31907
31907
|
}
|
|
31908
31908
|
};
|
|
31909
31909
|
} finally {
|
|
31910
|
-
await rm9(
|
|
31910
|
+
await rm9(fixtureDir2, { recursive: true, force: true });
|
|
31911
31911
|
}
|
|
31912
31912
|
}
|
|
31913
31913
|
|
|
@@ -31934,16 +31934,16 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
31934
31934
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31935
31935
|
}
|
|
31936
31936
|
const fixture = emailFixture.generate();
|
|
31937
|
-
const
|
|
31937
|
+
const fixtureDir2 = await mkdtemp9(path23.join(tmpdir4(), "bench-email-"));
|
|
31938
31938
|
try {
|
|
31939
31939
|
await options.ingestionAdapter.reset();
|
|
31940
31940
|
for (const file of fixture.files) {
|
|
31941
|
-
const filePath = path23.join(
|
|
31941
|
+
const filePath = path23.join(fixtureDir2, file.relativePath);
|
|
31942
31942
|
await mkdir8(path23.dirname(filePath), { recursive: true });
|
|
31943
31943
|
await writeFile8(filePath, file.content, "utf8");
|
|
31944
31944
|
}
|
|
31945
31945
|
const { result: ingestionLog, durationMs } = await timed(
|
|
31946
|
-
async () => options.ingestionAdapter.ingest(await realpath7(
|
|
31946
|
+
async () => options.ingestionAdapter.ingest(await realpath7(fixtureDir2))
|
|
31947
31947
|
);
|
|
31948
31948
|
const graph = await options.ingestionAdapter.getMemoryGraph();
|
|
31949
31949
|
const { precision, recall, f1 } = backlinkF1(graph.links, fixture.goldGraph.links);
|
|
@@ -32008,7 +32008,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
32008
32008
|
}
|
|
32009
32009
|
};
|
|
32010
32010
|
} finally {
|
|
32011
|
-
await rm10(
|
|
32011
|
+
await rm10(fixtureDir2, { recursive: true, force: true });
|
|
32012
32012
|
}
|
|
32013
32013
|
}
|
|
32014
32014
|
|
|
@@ -32036,16 +32036,16 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
32036
32036
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
32037
32037
|
}
|
|
32038
32038
|
const fixture = emailFixture.generate();
|
|
32039
|
-
const
|
|
32039
|
+
const fixtureDir2 = await mkdtemp10(path24.join(tmpdir5(), "bench-friction-"));
|
|
32040
32040
|
try {
|
|
32041
32041
|
await options.ingestionAdapter.reset();
|
|
32042
32042
|
for (const file of fixture.files) {
|
|
32043
|
-
const filePath = path24.join(
|
|
32043
|
+
const filePath = path24.join(fixtureDir2, file.relativePath);
|
|
32044
32044
|
await mkdir9(path24.dirname(filePath), { recursive: true });
|
|
32045
32045
|
await writeFile9(filePath, file.content, "utf8");
|
|
32046
32046
|
}
|
|
32047
32047
|
const { result: ingestionLog, durationMs } = await timed(
|
|
32048
|
-
async () => options.ingestionAdapter.ingest(await realpath8(
|
|
32048
|
+
async () => options.ingestionAdapter.ingest(await realpath8(fixtureDir2))
|
|
32049
32049
|
);
|
|
32050
32050
|
const commandsCount = ingestionLog.commandsIssued.length;
|
|
32051
32051
|
const promptsCount = ingestionLog.promptsShown.length;
|
|
@@ -32114,7 +32114,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
32114
32114
|
}
|
|
32115
32115
|
};
|
|
32116
32116
|
} finally {
|
|
32117
|
-
await rm11(
|
|
32117
|
+
await rm11(fixtureDir2, { recursive: true, force: true });
|
|
32118
32118
|
}
|
|
32119
32119
|
}
|
|
32120
32120
|
|
|
@@ -32213,17 +32213,17 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
32213
32213
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
32214
32214
|
}
|
|
32215
32215
|
const fixture = emailFixture.generate();
|
|
32216
|
-
const
|
|
32216
|
+
const fixtureDir2 = await mkdtemp11(path25.join(tmpdir6(), "bench-citation-"));
|
|
32217
32217
|
try {
|
|
32218
32218
|
await options.ingestionAdapter.reset();
|
|
32219
32219
|
for (const file of fixture.files) {
|
|
32220
|
-
const filePath = path25.join(
|
|
32220
|
+
const filePath = path25.join(fixtureDir2, file.relativePath);
|
|
32221
32221
|
await mkdir10(path25.dirname(filePath), { recursive: true });
|
|
32222
32222
|
await writeFile10(filePath, file.content, "utf8");
|
|
32223
32223
|
}
|
|
32224
32224
|
const benchmarkStart = performance.now();
|
|
32225
32225
|
const { result: ingestionLog, durationMs: ingestionDurationMs } = await timed(
|
|
32226
|
-
async () => options.ingestionAdapter.ingest(await realpath9(
|
|
32226
|
+
async () => options.ingestionAdapter.ingest(await realpath9(fixtureDir2))
|
|
32227
32227
|
);
|
|
32228
32228
|
if (ingestionLog.errors.length > 0) {
|
|
32229
32229
|
const durationMs = Math.round(performance.now() - benchmarkStart);
|
|
@@ -32416,7 +32416,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
32416
32416
|
}
|
|
32417
32417
|
};
|
|
32418
32418
|
} finally {
|
|
32419
|
-
await rm12(
|
|
32419
|
+
await rm12(fixtureDir2, { recursive: true, force: true });
|
|
32420
32420
|
}
|
|
32421
32421
|
}
|
|
32422
32422
|
function citationSupportScore(claim, citedSources) {
|
|
@@ -34087,10 +34087,10 @@ async function runSingleCase2(caseDef, options) {
|
|
|
34087
34087
|
for (let i = 1; i < flushTurnIndices.length; i += 1) {
|
|
34088
34088
|
turnsBetween.push(flushTurnIndices[i] - flushTurnIndices[i - 1]);
|
|
34089
34089
|
}
|
|
34090
|
-
const
|
|
34090
|
+
const mean6 = turnsBetween.length > 0 ? turnsBetween.reduce((acc, v) => acc + v, 0) / turnsBetween.length : 0;
|
|
34091
34091
|
return {
|
|
34092
34092
|
flushTurnIndices,
|
|
34093
|
-
turnsBetweenFlushesMean:
|
|
34093
|
+
turnsBetweenFlushesMean: mean6,
|
|
34094
34094
|
replayLatencyMs
|
|
34095
34095
|
};
|
|
34096
34096
|
}
|
|
@@ -35614,8 +35614,8 @@ function nonResurrection(log, corrections) {
|
|
|
35614
35614
|
}
|
|
35615
35615
|
function collateralDelta(before, after) {
|
|
35616
35616
|
if (before.length === 0) return 0;
|
|
35617
|
-
const
|
|
35618
|
-
return
|
|
35617
|
+
const mean6 = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
35618
|
+
return mean6(after) - mean6(before);
|
|
35619
35619
|
}
|
|
35620
35620
|
function scopePrecision(log, corrections) {
|
|
35621
35621
|
let scopedCount = 0;
|
|
@@ -39639,14 +39639,14 @@ function toCaseJsonl(cases) {
|
|
|
39639
39639
|
return `${cases.map((row) => JSON.stringify(row)).join("\n")}
|
|
39640
39640
|
`;
|
|
39641
39641
|
}
|
|
39642
|
-
async function validateStagedMemoryFixture(
|
|
39642
|
+
async function validateStagedMemoryFixture(fixtureDir2) {
|
|
39643
39643
|
const errors = [];
|
|
39644
39644
|
const warnings = [];
|
|
39645
|
-
const dirStat = await lstat5(
|
|
39645
|
+
const dirStat = await lstat5(fixtureDir2).catch(() => void 0);
|
|
39646
39646
|
if (!dirStat?.isDirectory()) {
|
|
39647
39647
|
return {
|
|
39648
39648
|
ok: false,
|
|
39649
|
-
errors: [`fixture directory not found: ${path32.basename(
|
|
39649
|
+
errors: [`fixture directory not found: ${path32.basename(fixtureDir2)}`],
|
|
39650
39650
|
warnings,
|
|
39651
39651
|
stats: { users: 0, cases: 0, distractorsPerCase: 0, transitions: 0 }
|
|
39652
39652
|
};
|
|
@@ -39654,7 +39654,7 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39654
39654
|
if (dirStat.isSymbolicLink()) {
|
|
39655
39655
|
errors.push("fixture directory must not be a symlink");
|
|
39656
39656
|
}
|
|
39657
|
-
const manifestPath = path32.join(
|
|
39657
|
+
const manifestPath = path32.join(fixtureDir2, MANIFEST_FILE);
|
|
39658
39658
|
const rawManifest = await readFile19(manifestPath, "utf8").catch(() => void 0);
|
|
39659
39659
|
if (rawManifest === void 0) {
|
|
39660
39660
|
return {
|
|
@@ -39707,7 +39707,7 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39707
39707
|
errors.push(`manifest.files must hash ${CASES_FILE}`);
|
|
39708
39708
|
}
|
|
39709
39709
|
for (const [relPath, expectedHash] of Object.entries(manifest.files)) {
|
|
39710
|
-
const filePath = path32.join(
|
|
39710
|
+
const filePath = path32.join(fixtureDir2, relPath);
|
|
39711
39711
|
const stat6 = await lstat5(filePath).catch(() => void 0);
|
|
39712
39712
|
if (!stat6) {
|
|
39713
39713
|
errors.push(`fixture file listed in manifest is missing: ${relPath}`);
|
|
@@ -39721,7 +39721,7 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39721
39721
|
errors.push(`fixture file fails its manifest hash: ${relPath}`);
|
|
39722
39722
|
}
|
|
39723
39723
|
}
|
|
39724
|
-
const rawCases = await readFile19(path32.join(
|
|
39724
|
+
const rawCases = await readFile19(path32.join(fixtureDir2, CASES_FILE), "utf8").catch(() => void 0);
|
|
39725
39725
|
if (rawCases === void 0) {
|
|
39726
39726
|
return {
|
|
39727
39727
|
ok: false,
|
|
@@ -39861,16 +39861,16 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39861
39861
|
}
|
|
39862
39862
|
};
|
|
39863
39863
|
}
|
|
39864
|
-
async function loadStagedMemoryFixture(
|
|
39865
|
-
const report = await validateStagedMemoryFixture(
|
|
39864
|
+
async function loadStagedMemoryFixture(fixtureDir2) {
|
|
39865
|
+
const report = await validateStagedMemoryFixture(fixtureDir2);
|
|
39866
39866
|
if (!report.ok) {
|
|
39867
39867
|
throw new Error(`staged-memory fixture failed preflight:
|
|
39868
39868
|
${report.errors.map((e) => ` - ${e}`).join("\n")}`);
|
|
39869
39869
|
}
|
|
39870
39870
|
const rawManifest = JSON.parse(
|
|
39871
|
-
await readFile19(path32.join(
|
|
39871
|
+
await readFile19(path32.join(fixtureDir2, MANIFEST_FILE), "utf8")
|
|
39872
39872
|
);
|
|
39873
|
-
const rawCases = await readFile19(path32.join(
|
|
39873
|
+
const rawCases = await readFile19(path32.join(fixtureDir2, CASES_FILE), "utf8");
|
|
39874
39874
|
const cases = rawCases.split("\n").filter((line) => line.trim()).map((line) => JSON.parse(line));
|
|
39875
39875
|
return {
|
|
39876
39876
|
manifest: rawManifest,
|
|
@@ -40525,7 +40525,7 @@ async function runStagedMemoryBenchmark(options) {
|
|
|
40525
40525
|
const persistCost = tokensPerSuccess(summaries["persist-only"]);
|
|
40526
40526
|
const costGate = stagedCost === "NA" || persistCost === "NA" ? true : stagedCost <= persistCost * (1 + STAGED_COST_TOLERANCE);
|
|
40527
40527
|
gateResults.staged_input_tokens_per_success_within_10pct_of_persist_only = costGate;
|
|
40528
|
-
const
|
|
40528
|
+
const failedGates2 = Object.entries(gateResults).filter(([, pass]) => !pass).map(([metric]) => metric);
|
|
40529
40529
|
const stagedOptions = {
|
|
40530
40530
|
schemaVersion: 1,
|
|
40531
40531
|
benchmark: STAGED_MEMORY_BENCHMARK_ID,
|
|
@@ -40610,8 +40610,8 @@ async function runStagedMemoryBenchmark(options) {
|
|
|
40610
40610
|
hardware: process.arch
|
|
40611
40611
|
}
|
|
40612
40612
|
};
|
|
40613
|
-
if (
|
|
40614
|
-
throw new Error(`staged-memory-synthetic-v1 gates failed: ${
|
|
40613
|
+
if (failedGates2.length > 0) {
|
|
40614
|
+
throw new Error(`staged-memory-synthetic-v1 gates failed: ${failedGates2.join(", ")}`, {
|
|
40615
40615
|
cause: result
|
|
40616
40616
|
});
|
|
40617
40617
|
}
|
|
@@ -41745,8 +41745,8 @@ var LOCOMO_CATEGORY_ORDER2 = ["single_hop", "multi_hop", "temporal", "open_domai
|
|
|
41745
41745
|
var LOCOMO_TASK_CATEGORY_PATTERN2 = /-(single_hop|multi_hop|temporal|open_domain|adversarial)$/;
|
|
41746
41746
|
var SOURCE_TURN_PATTERN = /^\[([^,\]\s]+),\s*turn\s+(\d+),\s*([^,\]]+?)(?:,\s*score\s+[^\]]+)?\]/i;
|
|
41747
41747
|
var SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
|
41748
|
-
function sanitizeLoComoResultReference(
|
|
41749
|
-
const reference = basename2(
|
|
41748
|
+
function sanitizeLoComoResultReference(path56) {
|
|
41749
|
+
const reference = basename2(path56).replace(/[\u0000-\u001f\u007f`]/g, "_");
|
|
41750
41750
|
if (!reference) throw new Error("Result path must identify a file.");
|
|
41751
41751
|
return reference;
|
|
41752
41752
|
}
|
|
@@ -43304,50 +43304,50 @@ function assertMemoryIdRef(value) {
|
|
|
43304
43304
|
throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
|
|
43305
43305
|
}
|
|
43306
43306
|
}
|
|
43307
|
-
function assertJsonConfig(value,
|
|
43307
|
+
function assertJsonConfig(value, path56 = "retrievalConfig") {
|
|
43308
43308
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
43309
43309
|
if (typeof value === "number") {
|
|
43310
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
43310
|
+
if (!Number.isFinite(value)) throw new Error(`${path56} must contain only finite JSON numbers.`);
|
|
43311
43311
|
return value;
|
|
43312
43312
|
}
|
|
43313
43313
|
if (Array.isArray(value)) {
|
|
43314
|
-
return value.map((entry, index) => assertJsonConfig(entry, `${
|
|
43314
|
+
return value.map((entry, index) => assertJsonConfig(entry, `${path56}[${index}]`));
|
|
43315
43315
|
}
|
|
43316
43316
|
if (!value || typeof value !== "object") {
|
|
43317
|
-
throw new Error(`${
|
|
43317
|
+
throw new Error(`${path56} must be JSON-serializable and provider-free.`);
|
|
43318
43318
|
}
|
|
43319
43319
|
const output = {};
|
|
43320
43320
|
for (const key of Object.keys(value).sort()) {
|
|
43321
43321
|
const child = value[key];
|
|
43322
43322
|
if (key === "openaiApiKey") {
|
|
43323
43323
|
if (child !== false) {
|
|
43324
|
-
throw new Error(`${
|
|
43324
|
+
throw new Error(`${path56}.${key} must be exactly false for provider-free capture.`);
|
|
43325
43325
|
}
|
|
43326
43326
|
output[key] = false;
|
|
43327
43327
|
continue;
|
|
43328
43328
|
}
|
|
43329
43329
|
if (isSecretKey(key)) {
|
|
43330
|
-
throw new Error(`${
|
|
43330
|
+
throw new Error(`${path56}.${key} contains secret-bearing configuration.`);
|
|
43331
43331
|
}
|
|
43332
43332
|
if (child === void 0) continue;
|
|
43333
43333
|
if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
|
|
43334
|
-
throw new Error(`${
|
|
43334
|
+
throw new Error(`${path56}.${key} is provider-capable configuration.`);
|
|
43335
43335
|
}
|
|
43336
|
-
output[key] = assertJsonConfig(child, `${
|
|
43336
|
+
output[key] = assertJsonConfig(child, `${path56}.${key}`);
|
|
43337
43337
|
}
|
|
43338
43338
|
return output;
|
|
43339
43339
|
}
|
|
43340
|
-
function sanitizeProviderFreeRetrievalConfig(value,
|
|
43340
|
+
function sanitizeProviderFreeRetrievalConfig(value, path56 = "retrievalConfig") {
|
|
43341
43341
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
43342
43342
|
if (typeof value === "number") {
|
|
43343
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
43343
|
+
if (!Number.isFinite(value)) throw new Error(`${path56} must contain only finite JSON numbers.`);
|
|
43344
43344
|
return value;
|
|
43345
43345
|
}
|
|
43346
43346
|
if (Array.isArray(value)) {
|
|
43347
|
-
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${
|
|
43347
|
+
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path56}[${index}]`));
|
|
43348
43348
|
}
|
|
43349
43349
|
if (!value || typeof value !== "object") {
|
|
43350
|
-
throw new Error(`${
|
|
43350
|
+
throw new Error(`${path56} must be JSON-serializable.`);
|
|
43351
43351
|
}
|
|
43352
43352
|
const output = {};
|
|
43353
43353
|
for (const key of Object.keys(value).sort()) {
|
|
@@ -43356,7 +43356,7 @@ function sanitizeProviderFreeRetrievalConfig(value, path53 = "retrievalConfig")
|
|
|
43356
43356
|
if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
|
|
43357
43357
|
continue;
|
|
43358
43358
|
}
|
|
43359
|
-
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${
|
|
43359
|
+
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path56}.${key}`);
|
|
43360
43360
|
}
|
|
43361
43361
|
return output;
|
|
43362
43362
|
}
|
|
@@ -45742,6 +45742,1028 @@ function getAblationCell(id) {
|
|
|
45742
45742
|
return cell;
|
|
45743
45743
|
}
|
|
45744
45744
|
|
|
45745
|
+
// src/ablations/outcome-prior.ts
|
|
45746
|
+
import { existsSync as existsSync3, readdirSync, readFileSync as readFileSync3 } from "fs";
|
|
45747
|
+
import path40 from "path";
|
|
45748
|
+
|
|
45749
|
+
// src/ablations/outcome-prior-gates.ts
|
|
45750
|
+
import { createHash as createHash21 } from "crypto";
|
|
45751
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
45752
|
+
import path39 from "path";
|
|
45753
|
+
import { rerankWithOutcomePrior } from "@remnic/core/rerank-outcome.js";
|
|
45754
|
+
|
|
45755
|
+
// src/ablations/outcome-prior-corpus.ts
|
|
45756
|
+
import { createHash as createHash20 } from "crypto";
|
|
45757
|
+
import { readFileSync } from "fs";
|
|
45758
|
+
import path38 from "path";
|
|
45759
|
+
var H1_FULL_RECIPE = Object.freeze({ users: 5, epochs: 12, seed: 21 });
|
|
45760
|
+
var H1_CI_RECIPE = Object.freeze({ users: 2, epochs: 4, seed: 21 });
|
|
45761
|
+
var H1_CI_TASK_COUNT = 12;
|
|
45762
|
+
var H1_MIN_OBSERVED_FRACTION = 0.6;
|
|
45763
|
+
var H1_WARM_SUCCESS_RATE_MIN = 0.2;
|
|
45764
|
+
var H1_WARM_SUCCESS_RATE_MAX = 0.8;
|
|
45765
|
+
var H1_WARM_SPLIT_TIMESTAMP = "1970-01-01T00:00:00.000Z";
|
|
45766
|
+
var TASK_KEYS = Object.freeze([
|
|
45767
|
+
"schemaVersion",
|
|
45768
|
+
"taskId",
|
|
45769
|
+
"userId",
|
|
45770
|
+
"instruction",
|
|
45771
|
+
"successCheck",
|
|
45772
|
+
"goldFactIds"
|
|
45773
|
+
]);
|
|
45774
|
+
var SUCCESS_CHECK_KEYS = Object.freeze(["type", "value"]);
|
|
45775
|
+
var WARM_STORE_KEYS = Object.freeze([
|
|
45776
|
+
"schemaVersion",
|
|
45777
|
+
"provenance",
|
|
45778
|
+
"seed",
|
|
45779
|
+
"users",
|
|
45780
|
+
"epochs",
|
|
45781
|
+
"splitTimestamp",
|
|
45782
|
+
"counters",
|
|
45783
|
+
"trajectories"
|
|
45784
|
+
]);
|
|
45785
|
+
var PROVENANCE_KEYS = Object.freeze(["generator", "generatorVersion", "manifestName"]);
|
|
45786
|
+
var COUNTER_KEYS = Object.freeze(["success", "fail"]);
|
|
45787
|
+
var TRAJECTORY_KEYS = Object.freeze(["taskId", "outcome", "factIds"]);
|
|
45788
|
+
var TASK_ID_PATTERN = /^h1t(?:-ci)?-[0-9]{2,}$/;
|
|
45789
|
+
var USER_ID_PATTERN = /^u[1-9][0-9]*$/;
|
|
45790
|
+
var FACT_ID_PATTERN = /^gf-u[1-9][0-9]*-[1-9][0-9]*-[1-9][0-9]*$/;
|
|
45791
|
+
function h1OutcomeFixtureRoot() {
|
|
45792
|
+
const candidates = [
|
|
45793
|
+
path38.resolve(import.meta.dirname, "../../fixtures/h1-outcome"),
|
|
45794
|
+
path38.resolve(import.meta.dirname, "../fixtures/h1-outcome")
|
|
45795
|
+
];
|
|
45796
|
+
for (const candidate of candidates) {
|
|
45797
|
+
if (existsSyncDecision(candidate)) return candidate;
|
|
45798
|
+
}
|
|
45799
|
+
throw new Error("h1-outcome fixtures are not installed next to @remnic/bench");
|
|
45800
|
+
}
|
|
45801
|
+
function existsSyncDecision(candidate) {
|
|
45802
|
+
try {
|
|
45803
|
+
readFileSync(path38.join(candidate, "decision-rule.json"));
|
|
45804
|
+
return true;
|
|
45805
|
+
} catch {
|
|
45806
|
+
return false;
|
|
45807
|
+
}
|
|
45808
|
+
}
|
|
45809
|
+
function isPlainObject5(value) {
|
|
45810
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
45811
|
+
const proto = Object.getPrototypeOf(value);
|
|
45812
|
+
return proto === Object.prototype || proto === null;
|
|
45813
|
+
}
|
|
45814
|
+
function own(raw, key) {
|
|
45815
|
+
return Object.hasOwn(raw, key) ? raw[key] : void 0;
|
|
45816
|
+
}
|
|
45817
|
+
function assertExactKeys2(raw, allowed, label) {
|
|
45818
|
+
const names = Object.getOwnPropertyNames(raw);
|
|
45819
|
+
for (const name of names) {
|
|
45820
|
+
if (!allowed.includes(name)) {
|
|
45821
|
+
throw new Error(`${label} has unexpected key ${JSON.stringify(name)}`);
|
|
45822
|
+
}
|
|
45823
|
+
}
|
|
45824
|
+
for (const name of allowed) {
|
|
45825
|
+
if (!Object.hasOwn(raw, name)) throw new Error(`${label} is missing key ${JSON.stringify(name)}`);
|
|
45826
|
+
}
|
|
45827
|
+
}
|
|
45828
|
+
function assertExactText(value, label) {
|
|
45829
|
+
if (typeof value !== "string" || value.length === 0 || value.trim() !== value || value.trim().length === 0) {
|
|
45830
|
+
throw new Error(`${label} must be a non-empty string with no surrounding whitespace`);
|
|
45831
|
+
}
|
|
45832
|
+
return value;
|
|
45833
|
+
}
|
|
45834
|
+
function assertPositiveInt(value, label) {
|
|
45835
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
45836
|
+
throw new Error(`${label} must be a positive integer`);
|
|
45837
|
+
}
|
|
45838
|
+
return value;
|
|
45839
|
+
}
|
|
45840
|
+
function assertNonNegativeInt(value, label) {
|
|
45841
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
45842
|
+
throw new Error(`${label} must be a non-negative integer`);
|
|
45843
|
+
}
|
|
45844
|
+
return value;
|
|
45845
|
+
}
|
|
45846
|
+
function compareText(left, right) {
|
|
45847
|
+
if (left === right) return 0;
|
|
45848
|
+
return left < right ? -1 : 1;
|
|
45849
|
+
}
|
|
45850
|
+
function escapeRegexLiteral(value) {
|
|
45851
|
+
let out = "";
|
|
45852
|
+
for (const ch of value) {
|
|
45853
|
+
if ("\\^$.*+?()[]{}|".includes(ch)) out += "\\";
|
|
45854
|
+
out += ch;
|
|
45855
|
+
}
|
|
45856
|
+
return out;
|
|
45857
|
+
}
|
|
45858
|
+
function isDeliberateWarmFailure(index) {
|
|
45859
|
+
if (!Number.isInteger(index) || index < 0) {
|
|
45860
|
+
throw new Error("warm task index must be a non-negative integer");
|
|
45861
|
+
}
|
|
45862
|
+
return index % 4 === 3;
|
|
45863
|
+
}
|
|
45864
|
+
function canonicalJson(value) {
|
|
45865
|
+
return `${stringifyCanonical(value)}
|
|
45866
|
+
`;
|
|
45867
|
+
}
|
|
45868
|
+
function stringifyCanonical(value) {
|
|
45869
|
+
if (value === null) return "null";
|
|
45870
|
+
if (typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
|
|
45871
|
+
if (typeof value === "number") {
|
|
45872
|
+
if (!Number.isFinite(value)) throw new Error("canonical JSON rejects non-finite numbers");
|
|
45873
|
+
return JSON.stringify(value);
|
|
45874
|
+
}
|
|
45875
|
+
if (Array.isArray(value)) return `[${value.map((item) => stringifyCanonical(item)).join(",")}]`;
|
|
45876
|
+
if (!isPlainObject5(value)) throw new Error("canonical JSON rejects unsupported values");
|
|
45877
|
+
const keys = Object.getOwnPropertyNames(value).sort(compareText);
|
|
45878
|
+
return `{${keys.map((key) => `${JSON.stringify(key)}:${stringifyCanonical(own(value, key))}`).join(",")}}`;
|
|
45879
|
+
}
|
|
45880
|
+
function hashCanonical(value) {
|
|
45881
|
+
return createHash20("sha256").update(canonicalJson(value)).digest("hex");
|
|
45882
|
+
}
|
|
45883
|
+
function resolveGoldAnswer(task, facts) {
|
|
45884
|
+
const values = [];
|
|
45885
|
+
for (const id of task.goldFactIds) {
|
|
45886
|
+
const fact3 = facts.get(id);
|
|
45887
|
+
if (!fact3 || fact3.userId !== task.userId) {
|
|
45888
|
+
return { ok: false, reason: `task ${task.taskId} gold fact ${id} does not resolve for ${task.userId}` };
|
|
45889
|
+
}
|
|
45890
|
+
values.push(fact3.value);
|
|
45891
|
+
}
|
|
45892
|
+
return { ok: true, expected: values.join(" | ") };
|
|
45893
|
+
}
|
|
45894
|
+
function replayTrajectories(tasks, facts) {
|
|
45895
|
+
const reasons = [];
|
|
45896
|
+
const replay = [];
|
|
45897
|
+
for (let index = 0; index < tasks.length; index += 1) {
|
|
45898
|
+
const task = tasks[index];
|
|
45899
|
+
const resolved = resolveGoldAnswer(task, facts);
|
|
45900
|
+
if (!resolved.ok) {
|
|
45901
|
+
reasons.push(resolved.reason);
|
|
45902
|
+
continue;
|
|
45903
|
+
}
|
|
45904
|
+
if (task.successCheck.value !== escapeRegexLiteral(resolved.expected)) {
|
|
45905
|
+
reasons.push(`task ${task.taskId} successCheck does not match its gold values`);
|
|
45906
|
+
continue;
|
|
45907
|
+
}
|
|
45908
|
+
const answer = isDeliberateWarmFailure(index) ? "unresolved" : resolved.expected;
|
|
45909
|
+
replay.push({
|
|
45910
|
+
taskId: task.taskId,
|
|
45911
|
+
outcome: answer === resolved.expected ? "success" : "fail",
|
|
45912
|
+
factIds: [...task.goldFactIds]
|
|
45913
|
+
});
|
|
45914
|
+
}
|
|
45915
|
+
if (reasons.length > 0) return { reasons };
|
|
45916
|
+
return { replay };
|
|
45917
|
+
}
|
|
45918
|
+
function countersFromTrajectories(trajectories) {
|
|
45919
|
+
const expected = /* @__PURE__ */ new Map();
|
|
45920
|
+
for (const trajectory of trajectories) {
|
|
45921
|
+
for (const id of trajectory.factIds) {
|
|
45922
|
+
const current = expected.get(id) ?? { success: 0, fail: 0 };
|
|
45923
|
+
if (trajectory.outcome === "success") current.success += 1;
|
|
45924
|
+
else current.fail += 1;
|
|
45925
|
+
expected.set(id, current);
|
|
45926
|
+
}
|
|
45927
|
+
}
|
|
45928
|
+
return expected;
|
|
45929
|
+
}
|
|
45930
|
+
function sameTrajectory(left, right) {
|
|
45931
|
+
if (left.taskId !== right.taskId || left.outcome !== right.outcome) return false;
|
|
45932
|
+
if (left.factIds.length !== right.factIds.length) return false;
|
|
45933
|
+
for (let index = 0; index < left.factIds.length; index += 1) {
|
|
45934
|
+
if (left.factIds[index] !== right.factIds[index]) return false;
|
|
45935
|
+
}
|
|
45936
|
+
return true;
|
|
45937
|
+
}
|
|
45938
|
+
function verifyOutcomeWarm(tasks, facts, store) {
|
|
45939
|
+
const reasons = [];
|
|
45940
|
+
const factIds = [...facts.keys()];
|
|
45941
|
+
let observed = 0;
|
|
45942
|
+
let successTotal = 0;
|
|
45943
|
+
let failTotal = 0;
|
|
45944
|
+
for (const id of factIds) {
|
|
45945
|
+
const counter = store.counters[id];
|
|
45946
|
+
if (!counter) continue;
|
|
45947
|
+
if (!Number.isInteger(counter.success) || !Number.isInteger(counter.fail) || counter.success < 0 || counter.fail < 0) {
|
|
45948
|
+
reasons.push(`counter for ${id} is not a pair of non-negative integers`);
|
|
45949
|
+
continue;
|
|
45950
|
+
}
|
|
45951
|
+
successTotal += counter.success;
|
|
45952
|
+
failTotal += counter.fail;
|
|
45953
|
+
if (counter.success + counter.fail >= 1) observed += 1;
|
|
45954
|
+
}
|
|
45955
|
+
for (const id of Object.getOwnPropertyNames(store.counters)) {
|
|
45956
|
+
if (!facts.has(id)) reasons.push(`counter ${id} is not a gold fact`);
|
|
45957
|
+
}
|
|
45958
|
+
const fraction = factIds.length === 0 ? 0 : observed / factIds.length;
|
|
45959
|
+
if (!(fraction >= H1_MIN_OBSERVED_FRACTION)) {
|
|
45960
|
+
reasons.push(`observed gold-fact fraction ${fraction} is below ${H1_MIN_OBSERVED_FRACTION}`);
|
|
45961
|
+
}
|
|
45962
|
+
if (!(successTotal > 0)) reasons.push("warm store has no success counters");
|
|
45963
|
+
if (!(failTotal > 0)) reasons.push("warm store has no failure counters");
|
|
45964
|
+
const replayed = replayTrajectories(tasks, facts);
|
|
45965
|
+
if ("reasons" in replayed) {
|
|
45966
|
+
reasons.push(...replayed.reasons);
|
|
45967
|
+
} else {
|
|
45968
|
+
const replay = replayed.replay;
|
|
45969
|
+
if (replay.length !== store.trajectories.length) {
|
|
45970
|
+
reasons.push("trajectory count does not match the task list");
|
|
45971
|
+
} else {
|
|
45972
|
+
for (let index = 0; index < replay.length; index += 1) {
|
|
45973
|
+
if (!sameTrajectory(replay[index], store.trajectories[index])) {
|
|
45974
|
+
reasons.push("trajectories do not match the warm replay");
|
|
45975
|
+
break;
|
|
45976
|
+
}
|
|
45977
|
+
}
|
|
45978
|
+
}
|
|
45979
|
+
const expected = countersFromTrajectories(replay);
|
|
45980
|
+
const matched = /* @__PURE__ */ new Set();
|
|
45981
|
+
for (const id of expected.keys()) {
|
|
45982
|
+
matched.add(id);
|
|
45983
|
+
const want = expected.get(id);
|
|
45984
|
+
const got = Object.hasOwn(store.counters, id) ? store.counters[id] : void 0;
|
|
45985
|
+
if (!want || !got || got.success !== want.success || got.fail !== want.fail) {
|
|
45986
|
+
reasons.push(`counter for ${id} does not match replayed trajectories`);
|
|
45987
|
+
}
|
|
45988
|
+
}
|
|
45989
|
+
for (const id of Object.getOwnPropertyNames(store.counters)) {
|
|
45990
|
+
if (matched.has(id)) continue;
|
|
45991
|
+
const got = store.counters[id];
|
|
45992
|
+
if (got && (got.success !== 0 || got.fail !== 0)) {
|
|
45993
|
+
reasons.push(`counter for ${id} does not match replayed trajectories`);
|
|
45994
|
+
}
|
|
45995
|
+
}
|
|
45996
|
+
}
|
|
45997
|
+
let trajectoryFacts = 0;
|
|
45998
|
+
let successes = 0;
|
|
45999
|
+
for (const trajectory of store.trajectories) {
|
|
46000
|
+
trajectoryFacts += trajectory.factIds.length;
|
|
46001
|
+
if (trajectory.outcome === "success") successes += 1;
|
|
46002
|
+
}
|
|
46003
|
+
if (successTotal + failTotal !== trajectoryFacts) {
|
|
46004
|
+
reasons.push("counter totals do not equal trajectory-recorded outcomes");
|
|
46005
|
+
}
|
|
46006
|
+
const rate = tasks.length === 0 ? 0 : successes / tasks.length;
|
|
46007
|
+
if (!(rate >= H1_WARM_SUCCESS_RATE_MIN && rate <= H1_WARM_SUCCESS_RATE_MAX)) {
|
|
46008
|
+
reasons.push(`task success rate ${rate} is outside ${H1_WARM_SUCCESS_RATE_MIN}..${H1_WARM_SUCCESS_RATE_MAX}`);
|
|
46009
|
+
}
|
|
46010
|
+
for (const task of tasks) {
|
|
46011
|
+
const epochs = /* @__PURE__ */ new Set();
|
|
46012
|
+
for (const id of task.goldFactIds) {
|
|
46013
|
+
const fact3 = facts.get(id);
|
|
46014
|
+
if (!fact3) reasons.push(`task ${task.taskId} gold fact ${id} does not resolve`);
|
|
46015
|
+
else epochs.add(fact3.introducedEpoch);
|
|
46016
|
+
}
|
|
46017
|
+
if (epochs.size < 2) reasons.push(`task ${task.taskId} does not span two epochs`);
|
|
46018
|
+
}
|
|
46019
|
+
if (store.provenance.generator !== "drift-gen") reasons.push("provenance generator must be drift-gen");
|
|
46020
|
+
return {
|
|
46021
|
+
ok: reasons.length === 0,
|
|
46022
|
+
reasons,
|
|
46023
|
+
goldFactCount: factIds.length,
|
|
46024
|
+
observedGoldFactCount: observed,
|
|
46025
|
+
observedFraction: fraction,
|
|
46026
|
+
taskSuccessRate: rate,
|
|
46027
|
+
successCounterTotal: successTotal,
|
|
46028
|
+
failCounterTotal: failTotal
|
|
46029
|
+
};
|
|
46030
|
+
}
|
|
46031
|
+
function parseOutcomeTask(raw) {
|
|
46032
|
+
if (!isPlainObject5(raw)) throw new Error("outcome task must be a plain object");
|
|
46033
|
+
assertExactKeys2(raw, TASK_KEYS, "outcome task");
|
|
46034
|
+
if (own(raw, "schemaVersion") !== 1) throw new Error("outcome task schemaVersion must be 1");
|
|
46035
|
+
const taskId = assertExactText(own(raw, "taskId"), "taskId");
|
|
46036
|
+
if (!TASK_ID_PATTERN.test(taskId)) throw new Error(`taskId ${JSON.stringify(taskId)} is not an h1 task id`);
|
|
46037
|
+
const userId = assertExactText(own(raw, "userId"), "userId");
|
|
46038
|
+
if (!USER_ID_PATTERN.test(userId)) throw new Error(`userId ${JSON.stringify(userId)} is not a drift-gen user id`);
|
|
46039
|
+
const instruction = assertExactText(own(raw, "instruction"), "instruction");
|
|
46040
|
+
const checkRaw = own(raw, "successCheck");
|
|
46041
|
+
if (!isPlainObject5(checkRaw)) throw new Error("successCheck must be a plain object");
|
|
46042
|
+
assertExactKeys2(checkRaw, SUCCESS_CHECK_KEYS, "successCheck");
|
|
46043
|
+
if (own(checkRaw, "type") !== "regex") throw new Error("successCheck.type must be regex");
|
|
46044
|
+
const value = assertExactText(own(checkRaw, "value"), "successCheck.value");
|
|
46045
|
+
const idsRaw = own(raw, "goldFactIds");
|
|
46046
|
+
if (!Array.isArray(idsRaw) || idsRaw.length < 2 || idsRaw.length > 4) {
|
|
46047
|
+
throw new Error("goldFactIds must contain 2 to 4 ids");
|
|
46048
|
+
}
|
|
46049
|
+
const goldFactIds = [];
|
|
46050
|
+
const seen = /* @__PURE__ */ new Set();
|
|
46051
|
+
for (const id of idsRaw) {
|
|
46052
|
+
const text = assertExactText(id, "goldFactIds entry");
|
|
46053
|
+
if (!FACT_ID_PATTERN.test(text)) throw new Error(`gold fact id ${JSON.stringify(text)} is not a drift-gen id`);
|
|
46054
|
+
if (seen.has(text)) throw new Error(`duplicate gold fact id ${text}`);
|
|
46055
|
+
seen.add(text);
|
|
46056
|
+
goldFactIds.push(text);
|
|
46057
|
+
}
|
|
46058
|
+
return {
|
|
46059
|
+
schemaVersion: 1,
|
|
46060
|
+
taskId,
|
|
46061
|
+
userId,
|
|
46062
|
+
instruction,
|
|
46063
|
+
successCheck: { type: "regex", value },
|
|
46064
|
+
goldFactIds
|
|
46065
|
+
};
|
|
46066
|
+
}
|
|
46067
|
+
function parseOutcomeTasksJsonl(text) {
|
|
46068
|
+
if (!text.endsWith("\n")) throw new Error("tasks JSONL must end with a newline");
|
|
46069
|
+
const lines = text.slice(0, -1).split("\n");
|
|
46070
|
+
if (lines.length === 0 || lines.some((line) => line.length === 0)) {
|
|
46071
|
+
throw new Error("tasks JSONL must be non-empty lines");
|
|
46072
|
+
}
|
|
46073
|
+
const tasks = lines.map((line) => parseOutcomeTask(JSON.parse(line)));
|
|
46074
|
+
const ids = /* @__PURE__ */ new Set();
|
|
46075
|
+
for (const task of tasks) {
|
|
46076
|
+
if (ids.has(task.taskId)) throw new Error(`duplicate task id ${task.taskId}`);
|
|
46077
|
+
ids.add(task.taskId);
|
|
46078
|
+
}
|
|
46079
|
+
return tasks;
|
|
46080
|
+
}
|
|
46081
|
+
function parseTrajectoryOutcome(value, index) {
|
|
46082
|
+
if (value === "success" || value === "fail") return value;
|
|
46083
|
+
throw new Error(`trajectory ${index} outcome must be success or fail`);
|
|
46084
|
+
}
|
|
46085
|
+
function parseCounter(raw, label) {
|
|
46086
|
+
if (!isPlainObject5(raw)) throw new Error(`${label} must be a plain object`);
|
|
46087
|
+
assertExactKeys2(raw, COUNTER_KEYS, label);
|
|
46088
|
+
return {
|
|
46089
|
+
success: assertNonNegativeInt(own(raw, "success"), `${label}.success`),
|
|
46090
|
+
fail: assertNonNegativeInt(own(raw, "fail"), `${label}.fail`)
|
|
46091
|
+
};
|
|
46092
|
+
}
|
|
46093
|
+
function parseOutcomeWarmStore(raw) {
|
|
46094
|
+
if (!isPlainObject5(raw)) throw new Error("warm store must be a plain object");
|
|
46095
|
+
assertExactKeys2(raw, WARM_STORE_KEYS, "warm store");
|
|
46096
|
+
if (own(raw, "schemaVersion") !== 1) throw new Error("warm store schemaVersion must be 1");
|
|
46097
|
+
const provenanceRaw = own(raw, "provenance");
|
|
46098
|
+
if (!isPlainObject5(provenanceRaw)) throw new Error("warm store provenance must be a plain object");
|
|
46099
|
+
assertExactKeys2(provenanceRaw, PROVENANCE_KEYS, "warm store provenance");
|
|
46100
|
+
if (own(provenanceRaw, "generator") !== "drift-gen") throw new Error("warm store provenance generator must be drift-gen");
|
|
46101
|
+
if (own(provenanceRaw, "generatorVersion") !== DRIFT_GEN_VERSION) {
|
|
46102
|
+
throw new Error("warm store provenance generatorVersion must match drift-gen");
|
|
46103
|
+
}
|
|
46104
|
+
if (own(provenanceRaw, "manifestName") !== "drift-gen-core") {
|
|
46105
|
+
throw new Error("warm store provenance manifestName must be drift-gen-core");
|
|
46106
|
+
}
|
|
46107
|
+
if (own(raw, "splitTimestamp") !== H1_WARM_SPLIT_TIMESTAMP) {
|
|
46108
|
+
throw new Error("warm store splitTimestamp must be the fixed sentinel");
|
|
46109
|
+
}
|
|
46110
|
+
const countersRaw = own(raw, "counters");
|
|
46111
|
+
if (!isPlainObject5(countersRaw)) throw new Error("warm store counters must be a plain object");
|
|
46112
|
+
const counters = {};
|
|
46113
|
+
for (const id of Object.getOwnPropertyNames(countersRaw)) {
|
|
46114
|
+
if (!FACT_ID_PATTERN.test(id)) throw new Error(`warm counter id ${JSON.stringify(id)} is not a drift-gen id`);
|
|
46115
|
+
counters[id] = parseCounter(own(countersRaw, id), `counter ${id}`);
|
|
46116
|
+
}
|
|
46117
|
+
const trajectoriesRaw = own(raw, "trajectories");
|
|
46118
|
+
if (!Array.isArray(trajectoriesRaw)) throw new Error("warm store trajectories must be an array");
|
|
46119
|
+
const trajectories = trajectoriesRaw.map((entry, index) => {
|
|
46120
|
+
if (!isPlainObject5(entry)) throw new Error(`trajectory ${index} must be a plain object`);
|
|
46121
|
+
assertExactKeys2(entry, TRAJECTORY_KEYS, `trajectory ${index}`);
|
|
46122
|
+
const taskId = assertExactText(own(entry, "taskId"), `trajectory ${index} taskId`);
|
|
46123
|
+
const outcome = parseTrajectoryOutcome(own(entry, "outcome"), index);
|
|
46124
|
+
const factIdsRaw = own(entry, "factIds");
|
|
46125
|
+
if (!Array.isArray(factIdsRaw) || factIdsRaw.length < 2) {
|
|
46126
|
+
throw new Error(`trajectory ${index} factIds must contain at least two ids`);
|
|
46127
|
+
}
|
|
46128
|
+
const factIds = factIdsRaw.map((id) => {
|
|
46129
|
+
const text = assertExactText(id, `trajectory ${index} fact id`);
|
|
46130
|
+
if (!FACT_ID_PATTERN.test(text)) throw new Error(`trajectory fact id ${JSON.stringify(text)} is not a drift-gen id`);
|
|
46131
|
+
return text;
|
|
46132
|
+
});
|
|
46133
|
+
return { taskId, outcome, factIds };
|
|
46134
|
+
});
|
|
46135
|
+
return {
|
|
46136
|
+
schemaVersion: 1,
|
|
46137
|
+
provenance: {
|
|
46138
|
+
generator: "drift-gen",
|
|
46139
|
+
generatorVersion: DRIFT_GEN_VERSION,
|
|
46140
|
+
manifestName: "drift-gen-core"
|
|
46141
|
+
},
|
|
46142
|
+
seed: assertPositiveInt(own(raw, "seed"), "warm store seed"),
|
|
46143
|
+
users: assertPositiveInt(own(raw, "users"), "warm store users"),
|
|
46144
|
+
epochs: assertPositiveInt(own(raw, "epochs"), "warm store epochs"),
|
|
46145
|
+
splitTimestamp: H1_WARM_SPLIT_TIMESTAMP,
|
|
46146
|
+
counters,
|
|
46147
|
+
trajectories
|
|
46148
|
+
};
|
|
46149
|
+
}
|
|
46150
|
+
var FACT_REF_REQUIRED = Object.freeze(["id", "userId", "introducedEpoch", "value"]);
|
|
46151
|
+
function parseOutcomeFactRef(raw) {
|
|
46152
|
+
if (!isPlainObject5(raw)) throw new Error("gold fact must be a plain object");
|
|
46153
|
+
for (const key of FACT_REF_REQUIRED) {
|
|
46154
|
+
if (!Object.hasOwn(raw, key)) throw new Error(`gold fact is missing ${key}`);
|
|
46155
|
+
}
|
|
46156
|
+
const id = assertExactText(own(raw, "id"), "gold fact id");
|
|
46157
|
+
if (!FACT_ID_PATTERN.test(id)) throw new Error(`gold fact id ${JSON.stringify(id)} is not a drift-gen id`);
|
|
46158
|
+
const userId = assertExactText(own(raw, "userId"), "gold fact userId");
|
|
46159
|
+
if (!USER_ID_PATTERN.test(userId)) throw new Error(`gold fact userId ${JSON.stringify(userId)} is not a drift-gen user id`);
|
|
46160
|
+
const introducedEpoch = assertPositiveInt(own(raw, "introducedEpoch"), "gold fact introducedEpoch");
|
|
46161
|
+
const value = assertExactText(own(raw, "value"), "gold fact value");
|
|
46162
|
+
return { id, userId, introducedEpoch, value };
|
|
46163
|
+
}
|
|
46164
|
+
function loadFactRefsJsonl(text) {
|
|
46165
|
+
if (!text.endsWith("\n")) throw new Error("facts JSONL must end with a newline");
|
|
46166
|
+
const refs = /* @__PURE__ */ new Map();
|
|
46167
|
+
for (const line of text.slice(0, -1).split("\n")) {
|
|
46168
|
+
if (line.length === 0) throw new Error("facts JSONL has an empty line");
|
|
46169
|
+
const fact3 = parseOutcomeFactRef(JSON.parse(line));
|
|
46170
|
+
if (refs.has(fact3.id)) throw new Error(`duplicate gold fact id ${fact3.id}`);
|
|
46171
|
+
refs.set(fact3.id, fact3);
|
|
46172
|
+
}
|
|
46173
|
+
return refs;
|
|
46174
|
+
}
|
|
46175
|
+
|
|
46176
|
+
// src/ablations/outcome-prior-gates.ts
|
|
46177
|
+
function textScore(taskId, factId) {
|
|
46178
|
+
const digest = createHash21("sha256").update(`${taskId}\0${factId}`).digest();
|
|
46179
|
+
return digest.readUInt32BE(0) / 4294967296;
|
|
46180
|
+
}
|
|
46181
|
+
function compareText2(left, right) {
|
|
46182
|
+
if (left === right) return 0;
|
|
46183
|
+
return left < right ? -1 : 1;
|
|
46184
|
+
}
|
|
46185
|
+
function rankSmokeByArm(tasks, facts, store, arms, rank = rerankWithOutcomePrior) {
|
|
46186
|
+
const hashes = [hashCanonical(store)];
|
|
46187
|
+
const byArm = {};
|
|
46188
|
+
const k = arms[0]?.retrieval.k ?? 0;
|
|
46189
|
+
for (const arm of arms) {
|
|
46190
|
+
if (arm.retrieval.k !== k) throw new Error("smoke arms must share one k");
|
|
46191
|
+
const rows = [];
|
|
46192
|
+
for (const task of tasks) {
|
|
46193
|
+
const candidates = [...facts.values()].filter((fact3) => fact3.userId === task.userId).sort((left, right) => compareText2(left.id, right.id)).map((fact3) => {
|
|
46194
|
+
const counter = Object.hasOwn(store.counters, fact3.id) ? store.counters[fact3.id] : void 0;
|
|
46195
|
+
return {
|
|
46196
|
+
item: fact3.id,
|
|
46197
|
+
textScore: textScore(task.taskId, fact3.id),
|
|
46198
|
+
success: counter?.success,
|
|
46199
|
+
fail: counter?.fail
|
|
46200
|
+
};
|
|
46201
|
+
});
|
|
46202
|
+
const ranked = rank(candidates, arm.outcomeBoostWeight);
|
|
46203
|
+
const top = ranked.slice(0, arm.retrieval.k).map((row) => ({
|
|
46204
|
+
id: row.item,
|
|
46205
|
+
score: row.score,
|
|
46206
|
+
state: row.state
|
|
46207
|
+
}));
|
|
46208
|
+
rows.push({ taskId: task.taskId, top });
|
|
46209
|
+
}
|
|
46210
|
+
byArm[arm.id] = rows;
|
|
46211
|
+
hashes.push(hashCanonical(store));
|
|
46212
|
+
}
|
|
46213
|
+
const warmStoreHash = hashes[0] ?? "";
|
|
46214
|
+
if (hashes.some((hash) => hash !== warmStoreHash)) {
|
|
46215
|
+
throw new Error("warm store changed while ranking an arm");
|
|
46216
|
+
}
|
|
46217
|
+
return {
|
|
46218
|
+
hash: createHash21("sha256").update(hashCanonical(byArm)).digest("hex"),
|
|
46219
|
+
warmStoreHash,
|
|
46220
|
+
byArm
|
|
46221
|
+
};
|
|
46222
|
+
}
|
|
46223
|
+
function loadCiSnapshot(root = h1OutcomeFixtureRoot()) {
|
|
46224
|
+
const dir = path39.join(root, "corpus-ci");
|
|
46225
|
+
const tasks = parseOutcomeTasksJsonl(readFileSync2(path39.join(dir, "tasks.jsonl"), "utf8"));
|
|
46226
|
+
const facts = loadFactRefsJsonl(readFileSync2(path39.join(dir, String(H1_CI_RECIPE.seed), "gold", "facts.jsonl"), "utf8"));
|
|
46227
|
+
const store = parseOutcomeWarmStore(JSON.parse(readFileSync2(path39.join(dir, "warm-store.json"), "utf8")));
|
|
46228
|
+
return { tasks, facts, store };
|
|
46229
|
+
}
|
|
46230
|
+
function sameRows(left, right) {
|
|
46231
|
+
return hashCanonical(left) === hashCanonical(right);
|
|
46232
|
+
}
|
|
46233
|
+
function smokeIsArmOrderInvariant(forward, reversed) {
|
|
46234
|
+
const leftIds = Object.getOwnPropertyNames(forward.byArm);
|
|
46235
|
+
const rightIds = Object.getOwnPropertyNames(reversed.byArm);
|
|
46236
|
+
if (leftIds.length !== rightIds.length) return false;
|
|
46237
|
+
for (const id of leftIds) {
|
|
46238
|
+
const left = forward.byArm[id];
|
|
46239
|
+
const right = reversed.byArm[id];
|
|
46240
|
+
if (left === void 0 || right === void 0 || !sameRows(left, right)) return false;
|
|
46241
|
+
}
|
|
46242
|
+
return true;
|
|
46243
|
+
}
|
|
46244
|
+
function gateError(error) {
|
|
46245
|
+
return error instanceof Error && error.message.length > 0 ? error.message : "pre-main gate failed";
|
|
46246
|
+
}
|
|
46247
|
+
function failedGates(reasons) {
|
|
46248
|
+
return {
|
|
46249
|
+
ok: false,
|
|
46250
|
+
reasons,
|
|
46251
|
+
smokeHash: "",
|
|
46252
|
+
warmStoreHash: "",
|
|
46253
|
+
repeated: false,
|
|
46254
|
+
armOrderInvariant: false,
|
|
46255
|
+
warmStoreImmutable: false
|
|
46256
|
+
};
|
|
46257
|
+
}
|
|
46258
|
+
async function runOutcomePriorPreMainGates(arms, root = h1OutcomeFixtureRoot()) {
|
|
46259
|
+
const reasons = [];
|
|
46260
|
+
const corpusDir = path39.join(root, "corpus-ci");
|
|
46261
|
+
let report;
|
|
46262
|
+
try {
|
|
46263
|
+
report = await validateDriftCorpus(corpusDir);
|
|
46264
|
+
} catch (error) {
|
|
46265
|
+
return failedGates([gateError(error)]);
|
|
46266
|
+
}
|
|
46267
|
+
if (!report.ok) reasons.push(...report.errors.map((error) => `drift-gen: ${error}`));
|
|
46268
|
+
if (report.stats.users !== H1_CI_RECIPE.users || report.stats.epochs !== H1_CI_RECIPE.epochs) {
|
|
46269
|
+
reasons.push("CI snapshot is not 2 users and 4 epochs");
|
|
46270
|
+
}
|
|
46271
|
+
let tasks;
|
|
46272
|
+
let facts;
|
|
46273
|
+
let store;
|
|
46274
|
+
try {
|
|
46275
|
+
({ tasks, facts, store } = loadCiSnapshot(root));
|
|
46276
|
+
} catch (error) {
|
|
46277
|
+
reasons.push(gateError(error));
|
|
46278
|
+
return failedGates(reasons);
|
|
46279
|
+
}
|
|
46280
|
+
if (tasks.length !== H1_CI_TASK_COUNT) reasons.push(`CI snapshot has ${tasks.length} tasks, need ${H1_CI_TASK_COUNT}`);
|
|
46281
|
+
if (store.seed !== H1_CI_RECIPE.seed || store.users !== H1_CI_RECIPE.users || store.epochs !== H1_CI_RECIPE.epochs) {
|
|
46282
|
+
reasons.push("warm store recipe does not match the CI snapshot");
|
|
46283
|
+
}
|
|
46284
|
+
let verification;
|
|
46285
|
+
try {
|
|
46286
|
+
verification = verifyOutcomeWarm(tasks, facts, store);
|
|
46287
|
+
} catch (error) {
|
|
46288
|
+
reasons.push(gateError(error));
|
|
46289
|
+
return failedGates(reasons);
|
|
46290
|
+
}
|
|
46291
|
+
if (!verification.ok) reasons.push(...verification.reasons);
|
|
46292
|
+
let forward;
|
|
46293
|
+
let reversed;
|
|
46294
|
+
let repeated;
|
|
46295
|
+
try {
|
|
46296
|
+
forward = rankSmokeByArm(tasks, facts, store, arms);
|
|
46297
|
+
reversed = rankSmokeByArm(tasks, facts, store, [...arms].reverse());
|
|
46298
|
+
repeated = rankSmokeByArm(tasks, facts, store, arms);
|
|
46299
|
+
} catch (error) {
|
|
46300
|
+
reasons.push(gateError(error));
|
|
46301
|
+
return failedGates(reasons);
|
|
46302
|
+
}
|
|
46303
|
+
const armOrderInvariant = smokeIsArmOrderInvariant(forward, reversed);
|
|
46304
|
+
const hashesMatch = forward.hash === repeated.hash && forward.warmStoreHash === repeated.warmStoreHash;
|
|
46305
|
+
if (!armOrderInvariant) reasons.push("arm order changed the smoke hash");
|
|
46306
|
+
if (!hashesMatch) reasons.push("repeated fake-model smoke hashes differ");
|
|
46307
|
+
if (forward.warmStoreHash !== reversed.warmStoreHash) reasons.push("warm store hash changed across arm order");
|
|
46308
|
+
return {
|
|
46309
|
+
ok: reasons.length === 0,
|
|
46310
|
+
reasons,
|
|
46311
|
+
smokeHash: forward.hash,
|
|
46312
|
+
warmStoreHash: forward.warmStoreHash,
|
|
46313
|
+
repeated: hashesMatch,
|
|
46314
|
+
armOrderInvariant,
|
|
46315
|
+
warmStoreImmutable: forward.warmStoreHash === reversed.warmStoreHash && forward.warmStoreHash === repeated.warmStoreHash
|
|
46316
|
+
};
|
|
46317
|
+
}
|
|
46318
|
+
|
|
46319
|
+
// src/ablations/outcome-prior.ts
|
|
46320
|
+
var OUTCOME_PRIOR_CLI_COMMAND = "remnic bench ablate outcome-prior";
|
|
46321
|
+
var ARM_KEYS = Object.freeze([
|
|
46322
|
+
"schemaVersion",
|
|
46323
|
+
"id",
|
|
46324
|
+
"role",
|
|
46325
|
+
"outcomeBoostWeight",
|
|
46326
|
+
"retrieval",
|
|
46327
|
+
"configOverrides"
|
|
46328
|
+
]);
|
|
46329
|
+
var H1_OVERRIDE_KEYS = Object.freeze([
|
|
46330
|
+
"outcomeBoostEnabled",
|
|
46331
|
+
"outcomeBoostWeight",
|
|
46332
|
+
"recallMemoryWorthFilterEnabled",
|
|
46333
|
+
"trustScoreEnabled",
|
|
46334
|
+
"boostAccessCount",
|
|
46335
|
+
"reinforcementRecallBoostEnabled",
|
|
46336
|
+
"recencyWeight"
|
|
46337
|
+
]);
|
|
46338
|
+
var MEMORY_WORTH_OVERRIDE_KEYS = Object.freeze([
|
|
46339
|
+
"recallMemoryWorthFilterEnabled",
|
|
46340
|
+
"outcomeBoostEnabled",
|
|
46341
|
+
"outcomeBoostWeight"
|
|
46342
|
+
]);
|
|
46343
|
+
var PICK_WEIGHTS = Object.freeze([0.15, 0.3, 0.5]);
|
|
46344
|
+
var H1_WEIGHTS = Object.freeze([0, 0.15, 0.3, 0.5]);
|
|
46345
|
+
var REFUSED_PHASES = Object.freeze(["warm", "pilot", "main"]);
|
|
46346
|
+
function fixtureDir() {
|
|
46347
|
+
const candidates = [
|
|
46348
|
+
path40.resolve(import.meta.dirname, "../../fixtures/h1-outcome"),
|
|
46349
|
+
path40.resolve(import.meta.dirname, "../fixtures/h1-outcome")
|
|
46350
|
+
];
|
|
46351
|
+
for (const candidate of candidates) {
|
|
46352
|
+
if (existsSync3(path40.join(candidate, "decision-rule.json"))) return candidate;
|
|
46353
|
+
}
|
|
46354
|
+
throw new Error("h1-outcome fixtures are not installed next to @remnic/bench");
|
|
46355
|
+
}
|
|
46356
|
+
function isPlainObject6(value) {
|
|
46357
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
46358
|
+
const proto = Object.getPrototypeOf(value);
|
|
46359
|
+
return proto === Object.prototype || proto === null;
|
|
46360
|
+
}
|
|
46361
|
+
function own2(raw, key) {
|
|
46362
|
+
return Object.hasOwn(raw, key) ? raw[key] : void 0;
|
|
46363
|
+
}
|
|
46364
|
+
function assertExactKeys3(raw, allowed, label) {
|
|
46365
|
+
const names = Object.getOwnPropertyNames(raw);
|
|
46366
|
+
for (const name of names) {
|
|
46367
|
+
if (!allowed.includes(name)) {
|
|
46368
|
+
throw new Error(`${label} has unexpected key ${JSON.stringify(name)}`);
|
|
46369
|
+
}
|
|
46370
|
+
}
|
|
46371
|
+
for (const name of allowed) {
|
|
46372
|
+
if (!Object.hasOwn(raw, name)) {
|
|
46373
|
+
throw new Error(`${label} is missing key ${JSON.stringify(name)}`);
|
|
46374
|
+
}
|
|
46375
|
+
}
|
|
46376
|
+
}
|
|
46377
|
+
function assertUnitWeight(value, label) {
|
|
46378
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
|
|
46379
|
+
throw new Error(`${label} must be a finite number in [0, 1]`);
|
|
46380
|
+
}
|
|
46381
|
+
return value;
|
|
46382
|
+
}
|
|
46383
|
+
function assertPositiveInt2(value, label) {
|
|
46384
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
46385
|
+
throw new Error(`${label} must be a positive integer`);
|
|
46386
|
+
}
|
|
46387
|
+
return value;
|
|
46388
|
+
}
|
|
46389
|
+
function sameNumbers(left, right) {
|
|
46390
|
+
if (left.length !== right.length) return false;
|
|
46391
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
46392
|
+
if (left[index] !== right[index]) return false;
|
|
46393
|
+
}
|
|
46394
|
+
return true;
|
|
46395
|
+
}
|
|
46396
|
+
function parseOutcomePriorArm(raw) {
|
|
46397
|
+
if (!isPlainObject6(raw)) throw new Error("outcome-prior arm must be a plain object");
|
|
46398
|
+
assertExactKeys3(raw, ARM_KEYS, "outcome-prior arm");
|
|
46399
|
+
if (own2(raw, "schemaVersion") !== 1) throw new Error("outcome-prior arm schemaVersion must be 1");
|
|
46400
|
+
const id = own2(raw, "id");
|
|
46401
|
+
if (typeof id !== "string" || id.length === 0) throw new Error("outcome-prior arm id must be a non-empty string");
|
|
46402
|
+
const role = own2(raw, "role");
|
|
46403
|
+
if (role !== "h1-arm" && role !== "memory-worth-base") {
|
|
46404
|
+
throw new Error("outcome-prior arm role must be h1-arm or memory-worth-base");
|
|
46405
|
+
}
|
|
46406
|
+
const outcomeBoostWeight = assertUnitWeight(own2(raw, "outcomeBoostWeight"), "outcomeBoostWeight");
|
|
46407
|
+
const retrievalRaw = own2(raw, "retrieval");
|
|
46408
|
+
if (!isPlainObject6(retrievalRaw)) throw new Error("outcome-prior arm retrieval must be a plain object");
|
|
46409
|
+
assertExactKeys3(retrievalRaw, ["k", "contextCapTokens"], "outcome-prior arm retrieval");
|
|
46410
|
+
const retrieval = {
|
|
46411
|
+
k: assertPositiveInt2(own2(retrievalRaw, "k"), "retrieval.k"),
|
|
46412
|
+
contextCapTokens: assertPositiveInt2(own2(retrievalRaw, "contextCapTokens"), "retrieval.contextCapTokens")
|
|
46413
|
+
};
|
|
46414
|
+
const overridesRaw = own2(raw, "configOverrides");
|
|
46415
|
+
if (!isPlainObject6(overridesRaw)) throw new Error("outcome-prior arm configOverrides must be a plain object");
|
|
46416
|
+
const allowed = role === "h1-arm" ? H1_OVERRIDE_KEYS : MEMORY_WORTH_OVERRIDE_KEYS;
|
|
46417
|
+
assertExactKeys3(overridesRaw, allowed, "outcome-prior arm configOverrides");
|
|
46418
|
+
const configOverrides = {};
|
|
46419
|
+
for (const key of allowed) {
|
|
46420
|
+
const value = own2(overridesRaw, key);
|
|
46421
|
+
if (typeof value !== "boolean" && typeof value !== "number") {
|
|
46422
|
+
throw new Error(`outcome-prior arm configOverrides.${key} must be a boolean or number`);
|
|
46423
|
+
}
|
|
46424
|
+
if (typeof value === "number" && !Number.isFinite(value)) {
|
|
46425
|
+
throw new Error(`outcome-prior arm configOverrides.${key} must be finite`);
|
|
46426
|
+
}
|
|
46427
|
+
configOverrides[key] = value;
|
|
46428
|
+
}
|
|
46429
|
+
if (role === "h1-arm") {
|
|
46430
|
+
if (configOverrides.outcomeBoostEnabled !== true) {
|
|
46431
|
+
throw new Error("h1 arm must set outcomeBoostEnabled true");
|
|
46432
|
+
}
|
|
46433
|
+
if (configOverrides.outcomeBoostWeight !== outcomeBoostWeight) {
|
|
46434
|
+
throw new Error("h1 arm outcomeBoostWeight must match configOverrides.outcomeBoostWeight");
|
|
46435
|
+
}
|
|
46436
|
+
if (configOverrides.recallMemoryWorthFilterEnabled !== false) {
|
|
46437
|
+
throw new Error("h1 arm must force recallMemoryWorthFilterEnabled false");
|
|
46438
|
+
}
|
|
46439
|
+
if (configOverrides.trustScoreEnabled !== false) {
|
|
46440
|
+
throw new Error("h1 arm must force trustScoreEnabled false");
|
|
46441
|
+
}
|
|
46442
|
+
if (configOverrides.boostAccessCount !== false) {
|
|
46443
|
+
throw new Error("h1 arm must force boostAccessCount false");
|
|
46444
|
+
}
|
|
46445
|
+
if (configOverrides.reinforcementRecallBoostEnabled !== false) {
|
|
46446
|
+
throw new Error("h1 arm must force reinforcementRecallBoostEnabled false");
|
|
46447
|
+
}
|
|
46448
|
+
if (configOverrides.recencyWeight !== 0) {
|
|
46449
|
+
throw new Error("h1 arm must force recencyWeight 0");
|
|
46450
|
+
}
|
|
46451
|
+
} else {
|
|
46452
|
+
if (outcomeBoostWeight !== 0 || configOverrides.outcomeBoostWeight !== 0) {
|
|
46453
|
+
throw new Error("memory-worth base arm must keep outcomeBoostWeight at 0");
|
|
46454
|
+
}
|
|
46455
|
+
if (configOverrides.outcomeBoostEnabled !== false) {
|
|
46456
|
+
throw new Error("memory-worth base arm must set outcomeBoostEnabled false");
|
|
46457
|
+
}
|
|
46458
|
+
if (configOverrides.recallMemoryWorthFilterEnabled !== true) {
|
|
46459
|
+
throw new Error("memory-worth base arm must set recallMemoryWorthFilterEnabled true");
|
|
46460
|
+
}
|
|
46461
|
+
}
|
|
46462
|
+
return {
|
|
46463
|
+
schemaVersion: 1,
|
|
46464
|
+
id,
|
|
46465
|
+
role,
|
|
46466
|
+
outcomeBoostWeight,
|
|
46467
|
+
retrieval,
|
|
46468
|
+
configOverrides
|
|
46469
|
+
};
|
|
46470
|
+
}
|
|
46471
|
+
function assertEpochPair(value, start, end, label) {
|
|
46472
|
+
if (!Array.isArray(value) || value.length !== 2 || value[0] !== start || value[1] !== end) {
|
|
46473
|
+
throw new Error(`${label} must be [${start}, ${end}]`);
|
|
46474
|
+
}
|
|
46475
|
+
}
|
|
46476
|
+
function parseOutcomePriorDecisionRule(raw) {
|
|
46477
|
+
if (!isPlainObject6(raw)) throw new Error("outcome-prior decision rule must be a plain object");
|
|
46478
|
+
assertExactKeys3(
|
|
46479
|
+
raw,
|
|
46480
|
+
[
|
|
46481
|
+
"schemaVersion",
|
|
46482
|
+
"ruleId",
|
|
46483
|
+
"controllingIssue",
|
|
46484
|
+
"controllingComment",
|
|
46485
|
+
"clock",
|
|
46486
|
+
"primaryMetric",
|
|
46487
|
+
"h1b",
|
|
46488
|
+
"thresholds",
|
|
46489
|
+
"analysis"
|
|
46490
|
+
],
|
|
46491
|
+
"outcome-prior decision rule"
|
|
46492
|
+
);
|
|
46493
|
+
if (own2(raw, "schemaVersion") !== 1) throw new Error("decision rule schemaVersion must be 1");
|
|
46494
|
+
if (own2(raw, "ruleId") !== "h1-outcome-prior-decision-v1") {
|
|
46495
|
+
throw new Error("decision rule id must be h1-outcome-prior-decision-v1");
|
|
46496
|
+
}
|
|
46497
|
+
if (own2(raw, "controllingIssue") !== 1958) throw new Error("decision rule controllingIssue must be 1958");
|
|
46498
|
+
const controllingComment = own2(raw, "controllingComment");
|
|
46499
|
+
if (controllingComment !== "https://github.com/joshuaswarren/remnic/issues/1958#issuecomment-4998194342") {
|
|
46500
|
+
throw new Error("decision rule controllingComment must cite comment 4998194342");
|
|
46501
|
+
}
|
|
46502
|
+
if (own2(raw, "primaryMetric") !== "recall@k") throw new Error("decision rule primary metric must be recall@k");
|
|
46503
|
+
if (own2(raw, "h1b") !== "NOT RUN") throw new Error("decision rule h1b must be NOT RUN");
|
|
46504
|
+
const clockRaw = own2(raw, "clock");
|
|
46505
|
+
if (!isPlainObject6(clockRaw)) throw new Error("decision rule clock must be a plain object");
|
|
46506
|
+
assertExactKeys3(clockRaw, ["warmEpochs", "pickEpochs", "lockedEpochs", "pickWeights"], "decision rule clock");
|
|
46507
|
+
const pickWeights = own2(clockRaw, "pickWeights");
|
|
46508
|
+
if (!Array.isArray(pickWeights) || !sameNumbers(pickWeights, PICK_WEIGHTS)) {
|
|
46509
|
+
throw new Error("decision rule pickWeights must be 0.15, 0.30, 0.50");
|
|
46510
|
+
}
|
|
46511
|
+
const thresholdsRaw = own2(raw, "thresholds");
|
|
46512
|
+
if (!isPlainObject6(thresholdsRaw)) throw new Error("decision rule thresholds must be a plain object");
|
|
46513
|
+
assertExactKeys3(
|
|
46514
|
+
thresholdsRaw,
|
|
46515
|
+
["minRelativeGain", "confidenceLevel", "ciLowerStrictlyAbove", "alpha"],
|
|
46516
|
+
"decision rule thresholds"
|
|
46517
|
+
);
|
|
46518
|
+
if (own2(thresholdsRaw, "minRelativeGain") !== 0.05) throw new Error("minRelativeGain must be 0.05");
|
|
46519
|
+
if (own2(thresholdsRaw, "confidenceLevel") !== 0.95) throw new Error("confidenceLevel must be 0.95");
|
|
46520
|
+
if (own2(thresholdsRaw, "ciLowerStrictlyAbove") !== 0) throw new Error("ciLowerStrictlyAbove must be 0");
|
|
46521
|
+
if (own2(thresholdsRaw, "alpha") !== 0.05) throw new Error("alpha must be 0.05");
|
|
46522
|
+
const analysisRaw = own2(raw, "analysis");
|
|
46523
|
+
if (!isPlainObject6(analysisRaw)) throw new Error("decision rule analysis must be a plain object");
|
|
46524
|
+
assertExactKeys3(
|
|
46525
|
+
analysisRaw,
|
|
46526
|
+
["bootstrapDraws", "pairing", "interval", "significance", "pickStageMultiplicity"],
|
|
46527
|
+
"decision rule analysis"
|
|
46528
|
+
);
|
|
46529
|
+
if (own2(analysisRaw, "bootstrapDraws") !== 1e4) throw new Error("bootstrapDraws must be 10000");
|
|
46530
|
+
if (own2(analysisRaw, "pairing") !== "user-data-seed-run-seed") {
|
|
46531
|
+
throw new Error("pairing must be user-data-seed-run-seed");
|
|
46532
|
+
}
|
|
46533
|
+
if (own2(analysisRaw, "interval") !== "paired-bootstrap-grouped-by-user") {
|
|
46534
|
+
throw new Error("interval must be paired-bootstrap-grouped-by-user");
|
|
46535
|
+
}
|
|
46536
|
+
if (own2(analysisRaw, "significance") !== "paired-shuffle") throw new Error("significance must be paired-shuffle");
|
|
46537
|
+
if (own2(analysisRaw, "pickStageMultiplicity") !== "holm") throw new Error("pickStageMultiplicity must be holm");
|
|
46538
|
+
return {
|
|
46539
|
+
schemaVersion: 1,
|
|
46540
|
+
ruleId: "h1-outcome-prior-decision-v1",
|
|
46541
|
+
controllingIssue: 1958,
|
|
46542
|
+
controllingComment,
|
|
46543
|
+
clock: {
|
|
46544
|
+
warmEpochs: (assertEpochPair(own2(clockRaw, "warmEpochs"), 1, 8, "warmEpochs"), [1, 8]),
|
|
46545
|
+
pickEpochs: (assertEpochPair(own2(clockRaw, "pickEpochs"), 9, 10, "pickEpochs"), [9, 10]),
|
|
46546
|
+
lockedEpochs: (assertEpochPair(own2(clockRaw, "lockedEpochs"), 11, 12, "lockedEpochs"), [11, 12]),
|
|
46547
|
+
pickWeights: [0.15, 0.3, 0.5]
|
|
46548
|
+
},
|
|
46549
|
+
primaryMetric: "recall@k",
|
|
46550
|
+
h1b: "NOT RUN",
|
|
46551
|
+
thresholds: {
|
|
46552
|
+
minRelativeGain: 0.05,
|
|
46553
|
+
confidenceLevel: 0.95,
|
|
46554
|
+
ciLowerStrictlyAbove: 0,
|
|
46555
|
+
alpha: 0.05
|
|
46556
|
+
},
|
|
46557
|
+
analysis: {
|
|
46558
|
+
bootstrapDraws: 1e4,
|
|
46559
|
+
pairing: "user-data-seed-run-seed",
|
|
46560
|
+
interval: "paired-bootstrap-grouped-by-user",
|
|
46561
|
+
significance: "paired-shuffle",
|
|
46562
|
+
pickStageMultiplicity: "holm"
|
|
46563
|
+
}
|
|
46564
|
+
};
|
|
46565
|
+
}
|
|
46566
|
+
function loadOutcomePriorDecisionRule() {
|
|
46567
|
+
const filePath = path40.join(fixtureDir(), "decision-rule.json");
|
|
46568
|
+
return parseOutcomePriorDecisionRule(JSON.parse(readFileSync3(filePath, "utf8")));
|
|
46569
|
+
}
|
|
46570
|
+
function loadOutcomePriorArms() {
|
|
46571
|
+
const dir = path40.join(fixtureDir(), "arms");
|
|
46572
|
+
const names = readdirSync(dir).filter((name) => name.endsWith(".json")).sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
|
|
46573
|
+
const arms = names.map(
|
|
46574
|
+
(name) => parseOutcomePriorArm(JSON.parse(readFileSync3(path40.join(dir, name), "utf8")))
|
|
46575
|
+
);
|
|
46576
|
+
if (arms.length === 0) throw new Error("outcome-prior arm directory is empty");
|
|
46577
|
+
const ids = /* @__PURE__ */ new Set();
|
|
46578
|
+
const h1Weights = [];
|
|
46579
|
+
const first = arms[0];
|
|
46580
|
+
let memoryWorthCount = 0;
|
|
46581
|
+
for (const arm of arms) {
|
|
46582
|
+
if (ids.has(arm.id)) throw new Error(`duplicate outcome-prior arm id ${arm.id}`);
|
|
46583
|
+
ids.add(arm.id);
|
|
46584
|
+
if (arm.retrieval.k !== first.retrieval.k || arm.retrieval.contextCapTokens !== first.retrieval.contextCapTokens) {
|
|
46585
|
+
throw new Error("outcome-prior arms must share one retrieval budget");
|
|
46586
|
+
}
|
|
46587
|
+
if (arm.role === "h1-arm") h1Weights.push(arm.outcomeBoostWeight);
|
|
46588
|
+
else memoryWorthCount += 1;
|
|
46589
|
+
}
|
|
46590
|
+
h1Weights.sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
|
|
46591
|
+
if (!sameNumbers(h1Weights, H1_WEIGHTS)) {
|
|
46592
|
+
throw new Error("h1 arms must be exactly weights 0, 0.15, 0.30, and 0.50");
|
|
46593
|
+
}
|
|
46594
|
+
if (memoryWorthCount !== 1) throw new Error("outcome-prior fixtures must include one memory-worth base arm");
|
|
46595
|
+
return arms;
|
|
46596
|
+
}
|
|
46597
|
+
function mean5(values) {
|
|
46598
|
+
let sum = 0;
|
|
46599
|
+
for (const value of values) sum += value;
|
|
46600
|
+
return sum / values.length;
|
|
46601
|
+
}
|
|
46602
|
+
function evaluateH1Decision(input) {
|
|
46603
|
+
const reasons = [];
|
|
46604
|
+
const candidate = input.candidateRecall;
|
|
46605
|
+
const baseline = input.baselineRecall;
|
|
46606
|
+
const estimable = candidate.length > 0 && candidate.length === baseline.length && candidate.every((value) => Number.isFinite(value)) && baseline.every((value) => Number.isFinite(value)) && mean5(baseline) > 0;
|
|
46607
|
+
if (!estimable) {
|
|
46608
|
+
return {
|
|
46609
|
+
relativeGain: null,
|
|
46610
|
+
confidenceInterval: null,
|
|
46611
|
+
cohensD: null,
|
|
46612
|
+
decision: "not-estimable",
|
|
46613
|
+
reasons: ["paired recall arrays are empty, mismatched, non-finite, or the baseline mean is not positive"]
|
|
46614
|
+
};
|
|
46615
|
+
}
|
|
46616
|
+
const relativeGain = (mean5(candidate) - mean5(baseline)) / mean5(baseline);
|
|
46617
|
+
const confidenceInterval = pairedDeltaConfidenceInterval(
|
|
46618
|
+
[...candidate],
|
|
46619
|
+
[...baseline],
|
|
46620
|
+
input.bootstrap ?? { iterations: input.rule.analysis.bootstrapDraws, level: input.rule.thresholds.confidenceLevel }
|
|
46621
|
+
);
|
|
46622
|
+
const effect = cohensD([...candidate], [...baseline]);
|
|
46623
|
+
if (!(relativeGain >= input.rule.thresholds.minRelativeGain)) {
|
|
46624
|
+
reasons.push("relative gain is below 5%");
|
|
46625
|
+
}
|
|
46626
|
+
if (!(confidenceInterval.lower > input.rule.thresholds.ciLowerStrictlyAbove)) {
|
|
46627
|
+
reasons.push("95% interval does not stay strictly above 0");
|
|
46628
|
+
}
|
|
46629
|
+
const pValue = input.pValue;
|
|
46630
|
+
const pReady = typeof pValue === "number" && Number.isFinite(pValue) && pValue >= 0 && pValue <= 1;
|
|
46631
|
+
if (!pReady) reasons.push("paired shuffle p-value is missing");
|
|
46632
|
+
else if (!(pValue < input.rule.thresholds.alpha)) reasons.push("paired shuffle p-value is not below 0.05");
|
|
46633
|
+
if (!pReady && reasons.length === 1) {
|
|
46634
|
+
return { relativeGain, confidenceInterval, cohensD: effect, decision: "not-estimable", reasons };
|
|
46635
|
+
}
|
|
46636
|
+
if (reasons.length > 0) {
|
|
46637
|
+
return { relativeGain, confidenceInterval, cohensD: effect, decision: "rejected", reasons };
|
|
46638
|
+
}
|
|
46639
|
+
return { relativeGain, confidenceInterval, cohensD: effect, decision: "supported", reasons };
|
|
46640
|
+
}
|
|
46641
|
+
function loadFrozenFixtures() {
|
|
46642
|
+
return { arms: loadOutcomePriorArms(), rule: loadOutcomePriorDecisionRule() };
|
|
46643
|
+
}
|
|
46644
|
+
function fixtureLoadMessage(error) {
|
|
46645
|
+
return error instanceof Error && error.message.length > 0 ? error.message : "outcome-prior fixtures failed to load";
|
|
46646
|
+
}
|
|
46647
|
+
function fixtureLoadFailure(message) {
|
|
46648
|
+
return {
|
|
46649
|
+
ok: false,
|
|
46650
|
+
exitCode: 1,
|
|
46651
|
+
runsExecuted: 0,
|
|
46652
|
+
h1b: "NOT RUN",
|
|
46653
|
+
armIds: [],
|
|
46654
|
+
ruleId: "",
|
|
46655
|
+
message,
|
|
46656
|
+
gates: {
|
|
46657
|
+
smokeHash: "",
|
|
46658
|
+
warmStoreHash: "",
|
|
46659
|
+
repeated: false,
|
|
46660
|
+
armOrderInvariant: false,
|
|
46661
|
+
warmStoreImmutable: false
|
|
46662
|
+
}
|
|
46663
|
+
};
|
|
46664
|
+
}
|
|
46665
|
+
function usageRejection(message) {
|
|
46666
|
+
return {
|
|
46667
|
+
ok: false,
|
|
46668
|
+
exitCode: 2,
|
|
46669
|
+
runsExecuted: 0,
|
|
46670
|
+
h1b: "NOT RUN",
|
|
46671
|
+
armIds: [],
|
|
46672
|
+
ruleId: "",
|
|
46673
|
+
message
|
|
46674
|
+
};
|
|
46675
|
+
}
|
|
46676
|
+
function classifyScaffoldArgv(argv) {
|
|
46677
|
+
let gates = false;
|
|
46678
|
+
let phase;
|
|
46679
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
46680
|
+
const token = argv[index] ?? "";
|
|
46681
|
+
if (token === "--gates") {
|
|
46682
|
+
if (gates) return { kind: "invalid", message: "duplicate --gates" };
|
|
46683
|
+
gates = true;
|
|
46684
|
+
continue;
|
|
46685
|
+
}
|
|
46686
|
+
if (token === "--phase" || token.startsWith("--phase=")) {
|
|
46687
|
+
if (phase !== void 0) return { kind: "invalid", message: "duplicate --phase" };
|
|
46688
|
+
const attached = token.startsWith("--phase=");
|
|
46689
|
+
const value = attached ? token.slice("--phase=".length) : argv[index + 1];
|
|
46690
|
+
if (!attached) index += 1;
|
|
46691
|
+
if (value === void 0 || value.length === 0 || value.startsWith("--")) {
|
|
46692
|
+
return { kind: "invalid", message: "--phase requires warm, pilot, or main" };
|
|
46693
|
+
}
|
|
46694
|
+
if (!REFUSED_PHASES.includes(value)) {
|
|
46695
|
+
return { kind: "invalid", message: `invalid phase ${value}` };
|
|
46696
|
+
}
|
|
46697
|
+
phase = value;
|
|
46698
|
+
continue;
|
|
46699
|
+
}
|
|
46700
|
+
return { kind: "invalid", message: `unknown argument ${token}` };
|
|
46701
|
+
}
|
|
46702
|
+
if (phase !== void 0) return { kind: "refused", phase };
|
|
46703
|
+
if (gates) return { kind: "gates" };
|
|
46704
|
+
return { kind: "list" };
|
|
46705
|
+
}
|
|
46706
|
+
function scaffoldWithoutGates(parsed) {
|
|
46707
|
+
if (parsed.kind === "invalid") return usageRejection(parsed.message);
|
|
46708
|
+
if (parsed.kind === "refused") {
|
|
46709
|
+
return {
|
|
46710
|
+
ok: false,
|
|
46711
|
+
exitCode: 2,
|
|
46712
|
+
runsExecuted: 0,
|
|
46713
|
+
h1b: "NOT RUN",
|
|
46714
|
+
armIds: [],
|
|
46715
|
+
ruleId: "",
|
|
46716
|
+
message: `refusing phase ${parsed.phase}: scaffolding only \u2014 no experiment runs`
|
|
46717
|
+
};
|
|
46718
|
+
}
|
|
46719
|
+
const arms = loadOutcomePriorArms();
|
|
46720
|
+
const rule = loadOutcomePriorDecisionRule();
|
|
46721
|
+
return {
|
|
46722
|
+
ok: true,
|
|
46723
|
+
exitCode: 0,
|
|
46724
|
+
runsExecuted: 0,
|
|
46725
|
+
h1b: rule.h1b,
|
|
46726
|
+
armIds: arms.map((arm) => arm.id),
|
|
46727
|
+
ruleId: rule.ruleId,
|
|
46728
|
+
message: "scaffolding only \u2014 no experiment runs"
|
|
46729
|
+
};
|
|
46730
|
+
}
|
|
46731
|
+
function runOutcomePriorScaffoldCli(argv) {
|
|
46732
|
+
const parsed = classifyScaffoldArgv(argv);
|
|
46733
|
+
if (parsed.kind === "gates") {
|
|
46734
|
+
return usageRejection("--gates is asynchronous; call runOutcomePriorGatesCli");
|
|
46735
|
+
}
|
|
46736
|
+
return scaffoldWithoutGates(parsed);
|
|
46737
|
+
}
|
|
46738
|
+
async function runOutcomePriorGatesCli(argv, loadFixtures = loadFrozenFixtures) {
|
|
46739
|
+
const parsed = classifyScaffoldArgv(argv);
|
|
46740
|
+
if (parsed.kind !== "gates") return scaffoldWithoutGates(parsed);
|
|
46741
|
+
let arms;
|
|
46742
|
+
let rule;
|
|
46743
|
+
try {
|
|
46744
|
+
({ arms, rule } = loadFixtures());
|
|
46745
|
+
} catch (error) {
|
|
46746
|
+
return fixtureLoadFailure(fixtureLoadMessage(error));
|
|
46747
|
+
}
|
|
46748
|
+
const gates = await runOutcomePriorPreMainGates(arms);
|
|
46749
|
+
return {
|
|
46750
|
+
ok: gates.ok,
|
|
46751
|
+
exitCode: gates.ok ? 0 : 1,
|
|
46752
|
+
runsExecuted: 0,
|
|
46753
|
+
h1b: rule.h1b,
|
|
46754
|
+
armIds: arms.map((arm) => arm.id),
|
|
46755
|
+
ruleId: rule.ruleId,
|
|
46756
|
+
message: gates.ok ? "pre-main gates passed \u2014 no experiment runs" : `pre-main gates failed: ${gates.reasons.join("; ")}`,
|
|
46757
|
+
gates: {
|
|
46758
|
+
smokeHash: gates.smokeHash,
|
|
46759
|
+
warmStoreHash: gates.warmStoreHash,
|
|
46760
|
+
repeated: gates.repeated,
|
|
46761
|
+
armOrderInvariant: gates.armOrderInvariant,
|
|
46762
|
+
warmStoreImmutable: gates.warmStoreImmutable
|
|
46763
|
+
}
|
|
46764
|
+
};
|
|
46765
|
+
}
|
|
46766
|
+
|
|
45745
46767
|
// src/benchmarks/remnic/procedural-recall/real-scenarios.ts
|
|
45746
46768
|
var PROCEDURAL_REAL_SCENARIOS = [
|
|
45747
46769
|
// --- exact-re-run (5) ---------------------------------------------------
|
|
@@ -46804,27 +47826,27 @@ function createMitigatedTarget(config) {
|
|
|
46804
47826
|
|
|
46805
47827
|
// src/security/injection-suite/runner.ts
|
|
46806
47828
|
import { writeFileAtomically as writeFileAtomically3 } from "@remnic/core/maintenance/atomic-file";
|
|
46807
|
-
import { createHash as
|
|
47829
|
+
import { createHash as createHash29 } from "crypto";
|
|
46808
47830
|
import {
|
|
46809
47831
|
mkdir as mkdir19,
|
|
46810
47832
|
open as open3,
|
|
46811
47833
|
readFile as readFile27,
|
|
46812
47834
|
writeFile as writeFile17
|
|
46813
47835
|
} from "fs/promises";
|
|
46814
|
-
import
|
|
47836
|
+
import path45 from "path";
|
|
46815
47837
|
import { setTimeout as delay3 } from "timers/promises";
|
|
46816
47838
|
|
|
46817
47839
|
// src/security/injection-suite/claims.ts
|
|
46818
47840
|
import { hostname } from "os";
|
|
46819
47841
|
import { mkdir as mkdir17, readFile as readFile25, rename as rename6, rm as rm17, stat as stat4, utimes, writeFile as writeFile16 } from "fs/promises";
|
|
46820
|
-
import
|
|
47842
|
+
import path42 from "path";
|
|
46821
47843
|
import { randomUUID as randomUUID37 } from "crypto";
|
|
46822
47844
|
|
|
46823
47845
|
// src/security/injection-suite/store.ts
|
|
46824
47846
|
import { writeFileAtomically } from "@remnic/core/maintenance/atomic-file";
|
|
46825
|
-
import { createHash as
|
|
47847
|
+
import { createHash as createHash22 } from "crypto";
|
|
46826
47848
|
import { readFile as readFile24 } from "fs/promises";
|
|
46827
|
-
import
|
|
47849
|
+
import path41 from "path";
|
|
46828
47850
|
|
|
46829
47851
|
// src/security/injection-suite/types.ts
|
|
46830
47852
|
var INJECTION_SUITE_VERSION = "h5-injection-suite-v3";
|
|
@@ -46876,9 +47898,9 @@ function buildInjectionSuiteRowKey(identity) {
|
|
|
46876
47898
|
identity.variantId,
|
|
46877
47899
|
String(identity.seed)
|
|
46878
47900
|
].join("\0");
|
|
46879
|
-
return `h5-row-v2-${
|
|
47901
|
+
return `h5-row-v2-${createHash22("sha256").update(payload).digest("hex")}`;
|
|
46880
47902
|
}
|
|
46881
|
-
function
|
|
47903
|
+
function canonicalJson2(value) {
|
|
46882
47904
|
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
|
|
46883
47905
|
const record = value;
|
|
46884
47906
|
const sorted = {};
|
|
@@ -46893,11 +47915,11 @@ var InjectionSuiteRowStore = class {
|
|
|
46893
47915
|
outputDir;
|
|
46894
47916
|
checkpointsDir;
|
|
46895
47917
|
constructor(outputDir) {
|
|
46896
|
-
this.outputDir =
|
|
46897
|
-
this.checkpointsDir =
|
|
47918
|
+
this.outputDir = path41.resolve(outputDir);
|
|
47919
|
+
this.checkpointsDir = path41.join(this.outputDir, "checkpoints");
|
|
46898
47920
|
}
|
|
46899
47921
|
checkpointPath(identity) {
|
|
46900
|
-
return
|
|
47922
|
+
return path41.join(this.checkpointsDir, `${buildInjectionSuiteRowKey(identity)}.json`);
|
|
46901
47923
|
}
|
|
46902
47924
|
async load(identity) {
|
|
46903
47925
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
@@ -46911,7 +47933,7 @@ var InjectionSuiteRowStore = class {
|
|
|
46911
47933
|
try {
|
|
46912
47934
|
const parsed = JSON.parse(raw);
|
|
46913
47935
|
if (parsed.rowKey !== rowKey) throw new Error("checkpoint rowKey does not match identity");
|
|
46914
|
-
if (
|
|
47936
|
+
if (canonicalJson2(parsed.identity) !== canonicalJson2(identity)) {
|
|
46915
47937
|
throw new Error("checkpoint identity does not exactly match requested identity");
|
|
46916
47938
|
}
|
|
46917
47939
|
if (!Array.isArray(parsed.tries)) throw new Error("checkpoint tries must be an array");
|
|
@@ -46988,7 +48010,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
46988
48010
|
heartbeatMs;
|
|
46989
48011
|
heartbeats = /* @__PURE__ */ new Map();
|
|
46990
48012
|
lockPath(rowKey) {
|
|
46991
|
-
return
|
|
48013
|
+
return path42.join(this.checkpointsDir, `${rowKey}.lock`);
|
|
46992
48014
|
}
|
|
46993
48015
|
async tryClaim(identity) {
|
|
46994
48016
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
@@ -47014,7 +48036,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
47014
48036
|
claimedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
47015
48037
|
};
|
|
47016
48038
|
try {
|
|
47017
|
-
await writeFile16(
|
|
48039
|
+
await writeFile16(path42.join(lockPath, "owner.json"), `${JSON.stringify(owner)}
|
|
47018
48040
|
`, {
|
|
47019
48041
|
flag: "wx"
|
|
47020
48042
|
});
|
|
@@ -47028,7 +48050,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
47028
48050
|
async release(claim) {
|
|
47029
48051
|
this.stopHeartbeat(claim.lockPath);
|
|
47030
48052
|
try {
|
|
47031
|
-
const ownerPath =
|
|
48053
|
+
const ownerPath = path42.join(claim.lockPath, "owner.json");
|
|
47032
48054
|
const owner = JSON.parse(await readFile25(ownerPath, "utf8"));
|
|
47033
48055
|
if (owner.ownerToken !== claim.ownerToken) return;
|
|
47034
48056
|
await rename6(ownerPath, `${ownerPath}.released-${claim.ownerToken}`);
|
|
@@ -47044,7 +48066,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
47044
48066
|
await rm17(released, { recursive: true, force: true });
|
|
47045
48067
|
}
|
|
47046
48068
|
async assertOwner(claim) {
|
|
47047
|
-
const owner = JSON.parse(await readFile25(
|
|
48069
|
+
const owner = JSON.parse(await readFile25(path42.join(claim.lockPath, "owner.json"), "utf8"));
|
|
47048
48070
|
if (owner.ownerToken !== claim.ownerToken) {
|
|
47049
48071
|
throw new Error(`lost injection-suite claim ${claim.rowKey}`);
|
|
47050
48072
|
}
|
|
@@ -47052,7 +48074,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
47052
48074
|
startHeartbeat(lockPath) {
|
|
47053
48075
|
this.stopHeartbeat(lockPath);
|
|
47054
48076
|
const timer = setInterval(() => {
|
|
47055
|
-
void utimes(
|
|
48077
|
+
void utimes(path42.join(lockPath, "owner.json"), /* @__PURE__ */ new Date(), /* @__PURE__ */ new Date()).catch(() => void 0);
|
|
47056
48078
|
}, this.heartbeatMs);
|
|
47057
48079
|
timer.unref?.();
|
|
47058
48080
|
this.heartbeats.set(lockPath, timer);
|
|
@@ -47063,7 +48085,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
47063
48085
|
clearInterval(timer);
|
|
47064
48086
|
}
|
|
47065
48087
|
async reclaimIfExpired(lockPath) {
|
|
47066
|
-
const ownerPath =
|
|
48088
|
+
const ownerPath = path42.join(lockPath, "owner.json");
|
|
47067
48089
|
let leaseMs = this.leaseMs;
|
|
47068
48090
|
let stampMs;
|
|
47069
48091
|
try {
|
|
@@ -47090,7 +48112,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
47090
48112
|
};
|
|
47091
48113
|
|
|
47092
48114
|
// src/security/injection-suite/generator.ts
|
|
47093
|
-
import { createHash as
|
|
48115
|
+
import { createHash as createHash23 } from "crypto";
|
|
47094
48116
|
|
|
47095
48117
|
// src/security/injection-templates/cross-session.ts
|
|
47096
48118
|
function crossConversation(templateId, payload, triggerPrompt) {
|
|
@@ -47277,7 +48299,7 @@ var ADAPTIVE_WRAPPERS_BY_STAGE = {
|
|
|
47277
48299
|
"adaptive-r3": ADAPTIVE_R3_WRAPPERS
|
|
47278
48300
|
};
|
|
47279
48301
|
function digestHex(value, length) {
|
|
47280
|
-
return
|
|
48302
|
+
return createHash23("sha256").update(value).digest("hex").slice(0, length);
|
|
47281
48303
|
}
|
|
47282
48304
|
function rngFor(family, index, seed) {
|
|
47283
48305
|
const mixed = Number.parseInt(
|
|
@@ -47452,7 +48474,7 @@ function validateInjectionSuiteVariant(variant) {
|
|
|
47452
48474
|
return errors;
|
|
47453
48475
|
}
|
|
47454
48476
|
function injectionSuiteVariantHash(variant) {
|
|
47455
|
-
return
|
|
48477
|
+
return createHash23("sha256").update(JSON.stringify(variant)).digest("hex");
|
|
47456
48478
|
}
|
|
47457
48479
|
function onlineVariantId(family, index, iteration) {
|
|
47458
48480
|
return `adaptive-online-r1-${family}-${index}-k${iteration}`;
|
|
@@ -47473,12 +48495,12 @@ function parseOnlineVariantId(variantId) {
|
|
|
47473
48495
|
}
|
|
47474
48496
|
|
|
47475
48497
|
// src/security/injection-suite/freeze.ts
|
|
47476
|
-
import { createHash as
|
|
48498
|
+
import { createHash as createHash26 } from "crypto";
|
|
47477
48499
|
|
|
47478
48500
|
// src/security/injection-suite/corpus.ts
|
|
47479
|
-
import { createHash as
|
|
48501
|
+
import { createHash as createHash24 } from "crypto";
|
|
47480
48502
|
function sha2564(value) {
|
|
47481
|
-
return
|
|
48503
|
+
return createHash24("sha256").update(value).digest("hex");
|
|
47482
48504
|
}
|
|
47483
48505
|
function buildInjectionSuiteCorpusManifest(stage, variantsPerFamily, seed = 71) {
|
|
47484
48506
|
const variants = generateSuiteVariants(variantsPerFamily, seed, stage);
|
|
@@ -47523,7 +48545,7 @@ ${failures.join("\n")}`);
|
|
|
47523
48545
|
}
|
|
47524
48546
|
|
|
47525
48547
|
// src/security/injection-suite/decision-rule.ts
|
|
47526
|
-
import { createHash as
|
|
48548
|
+
import { createHash as createHash25 } from "crypto";
|
|
47527
48549
|
var H5_DECISION_RULE = Object.freeze({
|
|
47528
48550
|
schemaVersion: 1,
|
|
47529
48551
|
ruleId: "h5-origin-authority-decision-v1",
|
|
@@ -47573,7 +48595,7 @@ var H5_DECISION_RULE = Object.freeze({
|
|
|
47573
48595
|
});
|
|
47574
48596
|
var H5_DECISION_RULE_BYTES = `${JSON.stringify(H5_DECISION_RULE, null, 2)}
|
|
47575
48597
|
`;
|
|
47576
|
-
var H5_DECISION_RULE_SHA256 =
|
|
48598
|
+
var H5_DECISION_RULE_SHA256 = createHash25("sha256").update(H5_DECISION_RULE_BYTES).digest("hex");
|
|
47577
48599
|
|
|
47578
48600
|
// src/security/injection-suite/freeze.ts
|
|
47579
48601
|
function canonicalize(value) {
|
|
@@ -47588,7 +48610,7 @@ function stableInjectionSuiteJson(value) {
|
|
|
47588
48610
|
return JSON.stringify(canonicalize(value));
|
|
47589
48611
|
}
|
|
47590
48612
|
function sha2565(value) {
|
|
47591
|
-
return
|
|
48613
|
+
return createHash26("sha256").update(value).digest("hex");
|
|
47592
48614
|
}
|
|
47593
48615
|
function verifyFrozenDecisionRule(metadata) {
|
|
47594
48616
|
if (!metadata.decisionRuleHash) {
|
|
@@ -48043,8 +49065,8 @@ async function completeChatResult(options, prompt) {
|
|
|
48043
49065
|
|
|
48044
49066
|
// src/security/injection-suite/product-lifecycle.ts
|
|
48045
49067
|
import { appendFileSync as appendFileSync2 } from "fs";
|
|
48046
|
-
import
|
|
48047
|
-
import { createHash as
|
|
49068
|
+
import path44 from "path";
|
|
49069
|
+
import { createHash as createHash28 } from "crypto";
|
|
48048
49070
|
|
|
48049
49071
|
// src/security/injection-suite/store-image.ts
|
|
48050
49072
|
import {
|
|
@@ -48053,10 +49075,10 @@ import {
|
|
|
48053
49075
|
StorageManager as StorageManager5
|
|
48054
49076
|
} from "@remnic/core";
|
|
48055
49077
|
import { writeFileAtomically as writeFileAtomically2 } from "@remnic/core/maintenance/atomic-file";
|
|
48056
|
-
import { createHash as
|
|
49078
|
+
import { createHash as createHash27 } from "crypto";
|
|
48057
49079
|
import { cp, mkdir as mkdir18, mkdtemp as mkdtemp13, readFile as readFile26, rename as rename7, rm as rm18 } from "fs/promises";
|
|
48058
49080
|
import os9 from "os";
|
|
48059
|
-
import
|
|
49081
|
+
import path43 from "path";
|
|
48060
49082
|
function seedIdentity(identity, screenEnabled) {
|
|
48061
49083
|
const arm = screenEnabled ? "quarantine" : "none";
|
|
48062
49084
|
return defaultSuiteIdentity({
|
|
@@ -48069,7 +49091,7 @@ function seedIdentity(identity, screenEnabled) {
|
|
|
48069
49091
|
});
|
|
48070
49092
|
}
|
|
48071
49093
|
function buildSeedKey(identity, variant, input, screenEnabled) {
|
|
48072
|
-
return
|
|
49094
|
+
return createHash27("sha256").update(
|
|
48073
49095
|
JSON.stringify({
|
|
48074
49096
|
suiteVersion: identity.suiteVersion,
|
|
48075
49097
|
stage: identity.stage,
|
|
@@ -48081,7 +49103,7 @@ function buildSeedKey(identity, variant, input, screenEnabled) {
|
|
|
48081
49103
|
executor: input.executor ?? "local",
|
|
48082
49104
|
baseUrl: input.baseUrl ?? "",
|
|
48083
49105
|
model: input.model ?? "",
|
|
48084
|
-
scenarioSha256:
|
|
49106
|
+
scenarioSha256: createHash27("sha256").update(JSON.stringify(variant)).digest("hex"),
|
|
48085
49107
|
modelDigest: input.modelDigest ?? ""
|
|
48086
49108
|
})
|
|
48087
49109
|
).digest("hex");
|
|
@@ -48089,7 +49111,7 @@ function buildSeedKey(identity, variant, input, screenEnabled) {
|
|
|
48089
49111
|
async function readManifest(directory) {
|
|
48090
49112
|
try {
|
|
48091
49113
|
return JSON.parse(
|
|
48092
|
-
await readFile26(
|
|
49114
|
+
await readFile26(path43.join(directory, "h5-seed.json"), "utf8")
|
|
48093
49115
|
);
|
|
48094
49116
|
} catch (error) {
|
|
48095
49117
|
if (error.code === "ENOENT") return void 0;
|
|
@@ -48110,7 +49132,7 @@ async function waitForSeed(directory, timeoutMs) {
|
|
|
48110
49132
|
);
|
|
48111
49133
|
}
|
|
48112
49134
|
function benchMemorySource(sessionId) {
|
|
48113
|
-
return `bench-replay-${
|
|
49135
|
+
return `bench-replay-${createHash27("sha256").update(sessionId).digest("hex").slice(0, 16)}`;
|
|
48114
49136
|
}
|
|
48115
49137
|
function plantOrigin(variant) {
|
|
48116
49138
|
const roles = new Set(variant.plantTurns.map((turn) => turn.originRole));
|
|
@@ -48120,9 +49142,9 @@ function plantOrigin(variant) {
|
|
|
48120
49142
|
return role === "tool" ? "tool_output" : role;
|
|
48121
49143
|
}
|
|
48122
49144
|
async function generateSeed(input) {
|
|
48123
|
-
const parent =
|
|
49145
|
+
const parent = path43.dirname(input.directory);
|
|
48124
49146
|
await mkdir18(parent, { recursive: true });
|
|
48125
|
-
const temporary = await mkdtemp13(
|
|
49147
|
+
const temporary = await mkdtemp13(path43.join(parent, ".h5-seed-"));
|
|
48126
49148
|
const plantSession = `h5-seed-${input.seedKey}`;
|
|
48127
49149
|
let adapter;
|
|
48128
49150
|
try {
|
|
@@ -48174,7 +49196,7 @@ async function generateSeed(input) {
|
|
|
48174
49196
|
await adapter.destroy();
|
|
48175
49197
|
adapter = void 0;
|
|
48176
49198
|
await writeFileAtomically2(
|
|
48177
|
-
|
|
49199
|
+
path43.join(temporary, "h5-seed.json"),
|
|
48178
49200
|
`${JSON.stringify(manifest, null, 2)}
|
|
48179
49201
|
`
|
|
48180
49202
|
);
|
|
@@ -48195,11 +49217,11 @@ async function prepareInjectionSuiteStore(input) {
|
|
|
48195
49217
|
input.run,
|
|
48196
49218
|
screenEnabled
|
|
48197
49219
|
);
|
|
48198
|
-
const storeRoot =
|
|
48199
|
-
const directory =
|
|
49220
|
+
const storeRoot = path43.join(input.run.outputDir, "store-images");
|
|
49221
|
+
const directory = path43.join(storeRoot, seedKey);
|
|
48200
49222
|
let manifest = await readManifest(directory);
|
|
48201
49223
|
if (!manifest) {
|
|
48202
|
-
const claims = new InjectionSuiteClaimLock(
|
|
49224
|
+
const claims = new InjectionSuiteClaimLock(path43.join(storeRoot, "claims"));
|
|
48203
49225
|
const claim = await claims.tryClaim(
|
|
48204
49226
|
seedIdentity(input.identity, screenEnabled)
|
|
48205
49227
|
);
|
|
@@ -48231,7 +49253,7 @@ async function prepareInjectionSuiteStore(input) {
|
|
|
48231
49253
|
if (manifest.seedKey !== seedKey || manifest.screenEnabled !== screenEnabled) {
|
|
48232
49254
|
throw new Error("H5 extraction seed manifest drifted");
|
|
48233
49255
|
}
|
|
48234
|
-
const memoryDir = await mkdtemp13(
|
|
49256
|
+
const memoryDir = await mkdtemp13(path43.join(os9.tmpdir(), "h5-arm-store-"));
|
|
48235
49257
|
await cp(directory, memoryDir, { recursive: true, force: false });
|
|
48236
49258
|
return {
|
|
48237
49259
|
memoryDir,
|
|
@@ -48252,7 +49274,7 @@ var DEFAULT_DEPS = {
|
|
|
48252
49274
|
complete: completeChatResult
|
|
48253
49275
|
};
|
|
48254
49276
|
function sha2566(value) {
|
|
48255
|
-
return
|
|
49277
|
+
return createHash28("sha256").update(value).digest("hex");
|
|
48256
49278
|
}
|
|
48257
49279
|
function buildInjectionSuiteAdapterOptions(arm, input) {
|
|
48258
49280
|
const executor = input.executor ?? "local";
|
|
@@ -48511,7 +49533,7 @@ ${variant.livenessPrompt}`;
|
|
|
48511
49533
|
}
|
|
48512
49534
|
if (input.captureResponses) {
|
|
48513
49535
|
appendFileSync2(
|
|
48514
|
-
|
|
49536
|
+
path44.join(input.outputDir, "responses.jsonl"),
|
|
48515
49537
|
`${JSON.stringify({
|
|
48516
49538
|
rowKey: buildInjectionSuiteRowKey(identity),
|
|
48517
49539
|
renderedRecall: recalled,
|
|
@@ -48569,7 +49591,7 @@ ${variant.livenessPrompt}`;
|
|
|
48569
49591
|
// src/security/injection-suite/runner.ts
|
|
48570
49592
|
var INJECTION_SUITE_RESUME_CONTRACT = "h5-injection-suite-resume-v3";
|
|
48571
49593
|
function injectionSuiteResumeContractHash(metadata) {
|
|
48572
|
-
return
|
|
49594
|
+
return createHash29("sha256").update(
|
|
48573
49595
|
JSON.stringify({
|
|
48574
49596
|
contract: INJECTION_SUITE_RESUME_CONTRACT,
|
|
48575
49597
|
suiteVersion: metadata.suiteVersion,
|
|
@@ -48719,7 +49741,7 @@ async function executeRow(identity, variant, input) {
|
|
|
48719
49741
|
async function readRunMetadata(outputDir) {
|
|
48720
49742
|
try {
|
|
48721
49743
|
return JSON.parse(
|
|
48722
|
-
await readFile27(
|
|
49744
|
+
await readFile27(path45.join(outputDir, "run.json"), "utf8")
|
|
48723
49745
|
);
|
|
48724
49746
|
} catch (error) {
|
|
48725
49747
|
if (error.code === "ENOENT") return void 0;
|
|
@@ -48727,7 +49749,7 @@ async function readRunMetadata(outputDir) {
|
|
|
48727
49749
|
}
|
|
48728
49750
|
}
|
|
48729
49751
|
async function writeNewRunMetadata(outputDir, metadata) {
|
|
48730
|
-
const filePath =
|
|
49752
|
+
const filePath = path45.join(outputDir, "run.json");
|
|
48731
49753
|
let handle;
|
|
48732
49754
|
try {
|
|
48733
49755
|
handle = await open3(filePath, "wx");
|
|
@@ -48753,7 +49775,7 @@ async function writeNewRunMetadata(outputDir, metadata) {
|
|
|
48753
49775
|
}
|
|
48754
49776
|
}
|
|
48755
49777
|
async function ensureFrozenArtifact(outputDir, fileName, content) {
|
|
48756
|
-
const filePath =
|
|
49778
|
+
const filePath = path45.join(outputDir, fileName);
|
|
48757
49779
|
try {
|
|
48758
49780
|
const existing = await readFile27(filePath, "utf8");
|
|
48759
49781
|
if (existing !== content)
|
|
@@ -48771,7 +49793,7 @@ async function ensureFrozenArtifact(outputDir, fileName, content) {
|
|
|
48771
49793
|
}
|
|
48772
49794
|
async function appendEpisode(outputDir, row) {
|
|
48773
49795
|
await writeFile17(
|
|
48774
|
-
|
|
49796
|
+
path45.join(outputDir, "episodes.jsonl"),
|
|
48775
49797
|
`${JSON.stringify(row)}
|
|
48776
49798
|
`,
|
|
48777
49799
|
{ flag: "a" }
|
|
@@ -48780,7 +49802,7 @@ async function appendEpisode(outputDir, row) {
|
|
|
48780
49802
|
async function ensureEpisode(outputDir, row) {
|
|
48781
49803
|
try {
|
|
48782
49804
|
const existing = await readFile27(
|
|
48783
|
-
|
|
49805
|
+
path45.join(outputDir, "episodes.jsonl"),
|
|
48784
49806
|
"utf8"
|
|
48785
49807
|
);
|
|
48786
49808
|
if (existing.includes(row.rowKey)) return;
|
|
@@ -48851,7 +49873,7 @@ async function runInjectionSuiteCliCommand(input) {
|
|
|
48851
49873
|
frozen.decisionRuleBytes
|
|
48852
49874
|
)
|
|
48853
49875
|
]);
|
|
48854
|
-
await writeFile17(
|
|
49876
|
+
await writeFile17(path45.join(input.outputDir, "deviations.jsonl"), "", {
|
|
48855
49877
|
flag: "a"
|
|
48856
49878
|
});
|
|
48857
49879
|
if (!existing) {
|
|
@@ -49027,23 +50049,23 @@ async function runInjectionSuiteCliCommand(input) {
|
|
|
49027
50049
|
// src/security/injection-suite/online-adaptive.ts
|
|
49028
50050
|
import { writeFileAtomically as writeFileAtomically6 } from "@remnic/core/maintenance/atomic-file";
|
|
49029
50051
|
import { renderAuthorityFence as renderAuthorityFence3 } from "@remnic/core";
|
|
49030
|
-
import { createHash as
|
|
49031
|
-
import { existsSync as
|
|
50052
|
+
import { createHash as createHash31 } from "crypto";
|
|
50053
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
|
|
49032
50054
|
import { appendFile, mkdir as mkdir20, readFile as readFile30, writeFile as writeFile18 } from "fs/promises";
|
|
49033
|
-
import
|
|
50055
|
+
import path48 from "path";
|
|
49034
50056
|
import { setTimeout as delay4 } from "timers/promises";
|
|
49035
50057
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
49036
50058
|
|
|
49037
50059
|
// src/security/injection-suite/online-adaptive-analysis.ts
|
|
49038
50060
|
import { writeFileAtomically as writeFileAtomically5 } from "@remnic/core/maintenance/atomic-file";
|
|
49039
|
-
import { createHash as
|
|
50061
|
+
import { createHash as createHash30 } from "crypto";
|
|
49040
50062
|
import { readFile as readFile29 } from "fs/promises";
|
|
49041
|
-
import
|
|
50063
|
+
import path47 from "path";
|
|
49042
50064
|
|
|
49043
50065
|
// src/security/injection-suite/publication-stats.ts
|
|
49044
50066
|
import { writeFileAtomically as writeFileAtomically4 } from "@remnic/core/maintenance/atomic-file";
|
|
49045
50067
|
import { readFile as readFile28 } from "fs/promises";
|
|
49046
|
-
import
|
|
50068
|
+
import path46 from "path";
|
|
49047
50069
|
var H5_PUBLICATION_ANALYSIS_RULE = Object.freeze({
|
|
49048
50070
|
schemaVersion: 1,
|
|
49049
50071
|
analysisId: "h5-publication-analysis-v1",
|
|
@@ -49326,13 +50348,13 @@ function analyzeInjectionSuitePublicationRows(rows, metadata, design, completene
|
|
|
49326
50348
|
}
|
|
49327
50349
|
async function analyzeInjectionSuitePublicationRun(runDir) {
|
|
49328
50350
|
const [metadata, design, episodeText] = await Promise.all([
|
|
49329
|
-
readFile28(
|
|
50351
|
+
readFile28(path46.join(runDir, "run.json"), "utf8").then(
|
|
49330
50352
|
(text) => JSON.parse(text)
|
|
49331
50353
|
),
|
|
49332
|
-
readFile28(
|
|
50354
|
+
readFile28(path46.join(runDir, "expected-design.json"), "utf8").then(
|
|
49333
50355
|
(text) => JSON.parse(text)
|
|
49334
50356
|
),
|
|
49335
|
-
readFile28(
|
|
50357
|
+
readFile28(path46.join(runDir, "episodes.jsonl"), "utf8")
|
|
49336
50358
|
]);
|
|
49337
50359
|
verifyFrozenDesign(design, metadata);
|
|
49338
50360
|
verifyFrozenDecisionRule(metadata);
|
|
@@ -49360,7 +50382,7 @@ async function analyzeInjectionSuitePublicationRun(runDir) {
|
|
|
49360
50382
|
}
|
|
49361
50383
|
);
|
|
49362
50384
|
await writeFileAtomically4(
|
|
49363
|
-
|
|
50385
|
+
path46.join(runDir, "publication-statistics.json"),
|
|
49364
50386
|
`${JSON.stringify(analysis, null, 2)}
|
|
49365
50387
|
`
|
|
49366
50388
|
);
|
|
@@ -49654,7 +50676,7 @@ function analyzeInjectionSuiteOnlineAdaptiveRows(args) {
|
|
|
49654
50676
|
};
|
|
49655
50677
|
}
|
|
49656
50678
|
async function readCorpusManifest(runDir) {
|
|
49657
|
-
const text = await readFile29(
|
|
50679
|
+
const text = await readFile29(path47.join(runDir, "online-corpus-manifest.json"), "utf8").catch(
|
|
49658
50680
|
(error) => error.code === "ENOENT" ? void 0 : Promise.reject(error)
|
|
49659
50681
|
);
|
|
49660
50682
|
if (text === void 0) return { manifest: void 0, hashVerified: false };
|
|
@@ -49664,16 +50686,16 @@ async function readCorpusManifest(runDir) {
|
|
|
49664
50686
|
} catch {
|
|
49665
50687
|
return { manifest: void 0, hashVerified: false };
|
|
49666
50688
|
}
|
|
49667
|
-
const corpusBytes = await readFile29(
|
|
50689
|
+
const corpusBytes = await readFile29(path47.join(runDir, "online-corpus.jsonl")).catch(
|
|
49668
50690
|
(error) => error.code === "ENOENT" ? Buffer.alloc(0) : Promise.reject(error)
|
|
49669
50691
|
);
|
|
49670
|
-
const hashVerified = corpusBytes.length > 0 &&
|
|
50692
|
+
const hashVerified = corpusBytes.length > 0 && createHash30("sha256").update(corpusBytes).digest("hex") === manifest.corpusSha256;
|
|
49671
50693
|
return { manifest, hashVerified };
|
|
49672
50694
|
}
|
|
49673
50695
|
var INJECTION_SUITE_ONLINE_RESUME_CONTRACT = "h5-injection-suite-online-resume-v1";
|
|
49674
50696
|
var INJECTION_SUITE_ONLINE_CONTRACT_GENERATION = 1;
|
|
49675
50697
|
function injectionSuiteResumeContractHashForOnline(metadata) {
|
|
49676
|
-
return
|
|
50698
|
+
return createHash30("sha256").update(
|
|
49677
50699
|
JSON.stringify({
|
|
49678
50700
|
contract: INJECTION_SUITE_ONLINE_RESUME_CONTRACT,
|
|
49679
50701
|
suiteVersion: metadata.suiteVersion,
|
|
@@ -49712,9 +50734,9 @@ function injectionSuiteResumeContractHashForOnline(metadata) {
|
|
|
49712
50734
|
}
|
|
49713
50735
|
async function analyzeInjectionSuiteOnlineAdaptiveRun(runDir) {
|
|
49714
50736
|
const [metadataText, designText, episodeLines, corpus] = await Promise.all([
|
|
49715
|
-
readFile29(
|
|
49716
|
-
readFile29(
|
|
49717
|
-
readJsonlLines(
|
|
50737
|
+
readFile29(path47.join(runDir, "run.json"), "utf8"),
|
|
50738
|
+
readFile29(path47.join(runDir, "expected-design.json"), "utf8"),
|
|
50739
|
+
readJsonlLines(path47.join(runDir, "episodes.jsonl")),
|
|
49718
50740
|
readCorpusManifest(runDir)
|
|
49719
50741
|
]);
|
|
49720
50742
|
const metadata = JSON.parse(metadataText);
|
|
@@ -49759,7 +50781,7 @@ async function analyzeInjectionSuiteOnlineAdaptiveRun(runDir) {
|
|
|
49759
50781
|
unexpectedRows += 1;
|
|
49760
50782
|
byRowKey.delete(key);
|
|
49761
50783
|
}
|
|
49762
|
-
const corpusLines = await readJsonlLines(
|
|
50784
|
+
const corpusLines = await readJsonlLines(path47.join(runDir, "online-corpus.jsonl"));
|
|
49763
50785
|
const corpusValid = /* @__PURE__ */ new Map();
|
|
49764
50786
|
let corpusSeenCount = 0;
|
|
49765
50787
|
for (const line of corpusLines ?? []) {
|
|
@@ -49866,7 +50888,7 @@ async function analyzeInjectionSuiteOnlineAdaptiveRun(runDir) {
|
|
|
49866
50888
|
manifestCountMatch
|
|
49867
50889
|
};
|
|
49868
50890
|
await writeFileAtomically5(
|
|
49869
|
-
|
|
50891
|
+
path47.join(runDir, "online-adaptive-statistics.json"),
|
|
49870
50892
|
`${JSON.stringify(statistics, null, 2)}
|
|
49871
50893
|
`
|
|
49872
50894
|
);
|
|
@@ -49879,17 +50901,17 @@ var ONLINE_ADAPTIVE_ARMS = [
|
|
|
49879
50901
|
"layered-fence-quarantine"
|
|
49880
50902
|
];
|
|
49881
50903
|
function benchPackageRoot() {
|
|
49882
|
-
let dir =
|
|
50904
|
+
let dir = path48.dirname(fileURLToPath3(import.meta.url));
|
|
49883
50905
|
for (let depth = 0; depth < 8; depth += 1) {
|
|
49884
|
-
const candidate =
|
|
49885
|
-
if (
|
|
50906
|
+
const candidate = path48.join(dir, "package.json");
|
|
50907
|
+
if (existsSync4(candidate)) {
|
|
49886
50908
|
try {
|
|
49887
|
-
const manifest = JSON.parse(
|
|
50909
|
+
const manifest = JSON.parse(readFileSync4(candidate, "utf8"));
|
|
49888
50910
|
if (manifest.name === "@remnic/bench") return dir;
|
|
49889
50911
|
} catch {
|
|
49890
50912
|
}
|
|
49891
50913
|
}
|
|
49892
|
-
const parent =
|
|
50914
|
+
const parent = path48.dirname(dir);
|
|
49893
50915
|
if (parent === dir) break;
|
|
49894
50916
|
dir = parent;
|
|
49895
50917
|
}
|
|
@@ -49903,8 +50925,8 @@ function resolveDefaultAttackerPromptPath() {
|
|
|
49903
50925
|
);
|
|
49904
50926
|
}
|
|
49905
50927
|
const relative = "fixtures/h5-injection/online-adaptive/attacker-prompt-v1.md";
|
|
49906
|
-
const absolute =
|
|
49907
|
-
if (!
|
|
50928
|
+
const absolute = path48.join(root, relative);
|
|
50929
|
+
if (!existsSync4(absolute)) {
|
|
49908
50930
|
throw new Error(
|
|
49909
50931
|
`bench fixture ${relative} is not published; pass --attacker-prompt explicitly`
|
|
49910
50932
|
);
|
|
@@ -49919,29 +50941,29 @@ function resolveDefaultScreenSourcePath() {
|
|
|
49919
50941
|
"cannot resolve @remnic/core package root for the disclosed write-path screen source"
|
|
49920
50942
|
);
|
|
49921
50943
|
}
|
|
49922
|
-
const tsCandidate =
|
|
50944
|
+
const tsCandidate = path48.join(
|
|
49923
50945
|
coreRoot,
|
|
49924
50946
|
"src",
|
|
49925
50947
|
"security",
|
|
49926
50948
|
"injection-screen.ts"
|
|
49927
50949
|
);
|
|
49928
|
-
if (
|
|
49929
|
-
const jsCandidate =
|
|
50950
|
+
if (existsSync4(tsCandidate)) return tsCandidate;
|
|
50951
|
+
const jsCandidate = path48.join(
|
|
49930
50952
|
coreRoot,
|
|
49931
50953
|
"dist",
|
|
49932
50954
|
"security",
|
|
49933
50955
|
"injection-screen.js"
|
|
49934
50956
|
);
|
|
49935
|
-
if (
|
|
50957
|
+
if (existsSync4(jsCandidate)) return jsCandidate;
|
|
49936
50958
|
throw new Error(
|
|
49937
50959
|
"cannot locate @remnic/core/src/security/injection-screen.ts or its dist build"
|
|
49938
50960
|
);
|
|
49939
50961
|
}
|
|
49940
50962
|
function walkUpToPackageRoot(startPath) {
|
|
49941
|
-
let dir =
|
|
50963
|
+
let dir = path48.dirname(fileURLToPath3(startPath));
|
|
49942
50964
|
for (let depth = 0; depth < 8; depth += 1) {
|
|
49943
|
-
if (
|
|
49944
|
-
const parent =
|
|
50965
|
+
if (existsSync4(path48.join(dir, "package.json"))) return dir;
|
|
50966
|
+
const parent = path48.dirname(dir);
|
|
49945
50967
|
if (parent === dir) break;
|
|
49946
50968
|
dir = parent;
|
|
49947
50969
|
}
|
|
@@ -50108,7 +51130,7 @@ function buildOnlineAdaptiveAttackerInput(args) {
|
|
|
50108
51130
|
);
|
|
50109
51131
|
}
|
|
50110
51132
|
function sha2567(value) {
|
|
50111
|
-
return
|
|
51133
|
+
return createHash31("sha256").update(value).digest("hex");
|
|
50112
51134
|
}
|
|
50113
51135
|
function regenBaseVariant(family, index, seed) {
|
|
50114
51136
|
const variants = generateFamilyVariants(family, index, seed, "base");
|
|
@@ -50123,18 +51145,18 @@ async function loadOnlineAdaptiveHistory(outputDir) {
|
|
|
50123
51145
|
episodes: /* @__PURE__ */ new Map(),
|
|
50124
51146
|
episodeKeys: /* @__PURE__ */ new Set()
|
|
50125
51147
|
};
|
|
50126
|
-
for (const line of await readJsonlLines(
|
|
51148
|
+
for (const line of await readJsonlLines(path48.join(outputDir, "online-corpus.jsonl")) ?? []) {
|
|
50127
51149
|
const parsed = JSON.parse(line);
|
|
50128
51150
|
history.corpus.set(corpusKey(parsed.arm, parsed.variantId), parsed);
|
|
50129
51151
|
}
|
|
50130
|
-
for (const line of await readJsonlLines(
|
|
51152
|
+
for (const line of await readJsonlLines(path48.join(outputDir, "responses.jsonl")) ?? []) {
|
|
50131
51153
|
const parsed = JSON.parse(line);
|
|
50132
51154
|
history.responses.set(parsed.rowKey, {
|
|
50133
51155
|
responseText: parsed.responseText,
|
|
50134
51156
|
toolCalls: parsed.toolCalls
|
|
50135
51157
|
});
|
|
50136
51158
|
}
|
|
50137
|
-
for (const line of await readJsonlLines(
|
|
51159
|
+
for (const line of await readJsonlLines(path48.join(outputDir, "episodes.jsonl")) ?? []) {
|
|
50138
51160
|
const parsed = JSON.parse(line);
|
|
50139
51161
|
history.episodes.set(parsed.rowKey, parsed);
|
|
50140
51162
|
history.episodeKeys.add(parsed.rowKey);
|
|
@@ -50177,7 +51199,7 @@ function previousIterationFor(args) {
|
|
|
50177
51199
|
}
|
|
50178
51200
|
async function appendCorpusLine(outputDir, line) {
|
|
50179
51201
|
await appendFile(
|
|
50180
|
-
|
|
51202
|
+
path48.join(outputDir, "online-corpus.jsonl"),
|
|
50181
51203
|
`${JSON.stringify(line)}
|
|
50182
51204
|
`
|
|
50183
51205
|
);
|
|
@@ -50186,7 +51208,7 @@ async function appendEpisodeLine(outputDir, row, seen) {
|
|
|
50186
51208
|
if (seen.has(row.rowKey)) return;
|
|
50187
51209
|
seen.add(row.rowKey);
|
|
50188
51210
|
await appendFile(
|
|
50189
|
-
|
|
51211
|
+
path48.join(outputDir, "episodes.jsonl"),
|
|
50190
51212
|
`${JSON.stringify(row)}
|
|
50191
51213
|
`
|
|
50192
51214
|
);
|
|
@@ -50323,7 +51345,7 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
|
|
|
50323
51345
|
ensureFrozenArtifact(input.outputDir, "expected-design.json", frozen.designBytes),
|
|
50324
51346
|
ensureFrozenArtifact(input.outputDir, "decision-rule.json", frozen.decisionRuleBytes)
|
|
50325
51347
|
]);
|
|
50326
|
-
await writeFile18(
|
|
51348
|
+
await writeFile18(path48.join(input.outputDir, "deviations.jsonl"), "", { flag: "a" });
|
|
50327
51349
|
if (!existing) {
|
|
50328
51350
|
const metadata = {
|
|
50329
51351
|
schemaVersion: 3,
|
|
@@ -50621,7 +51643,7 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
|
|
|
50621
51643
|
}
|
|
50622
51644
|
}
|
|
50623
51645
|
const corpusBytes = await readFile30(
|
|
50624
|
-
|
|
51646
|
+
path48.join(input.outputDir, "online-corpus.jsonl"),
|
|
50625
51647
|
"utf8"
|
|
50626
51648
|
).catch(
|
|
50627
51649
|
(error) => error.code === "ENOENT" ? "" : Promise.reject(error)
|
|
@@ -50630,7 +51652,7 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
|
|
|
50630
51652
|
const parsedLines = lines.map(
|
|
50631
51653
|
(line) => JSON.parse(line)
|
|
50632
51654
|
);
|
|
50633
|
-
const episodeCount = (await readJsonlLines(
|
|
51655
|
+
const episodeCount = (await readJsonlLines(path48.join(input.outputDir, "episodes.jsonl")))?.length ?? 0;
|
|
50634
51656
|
const manifest = {
|
|
50635
51657
|
schemaVersion: 1,
|
|
50636
51658
|
stage: ONLINE_ADAPTIVE_STAGE,
|
|
@@ -50648,7 +51670,7 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
|
|
|
50648
51670
|
attackerSeedBase: input.seedBase ?? 71
|
|
50649
51671
|
};
|
|
50650
51672
|
await writeFileAtomically6(
|
|
50651
|
-
|
|
51673
|
+
path48.join(input.outputDir, "online-corpus-manifest.json"),
|
|
50652
51674
|
`${JSON.stringify(manifest, null, 2)}
|
|
50653
51675
|
`
|
|
50654
51676
|
);
|
|
@@ -50664,7 +51686,7 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
|
|
|
50664
51686
|
// src/security/injection-suite/stats.ts
|
|
50665
51687
|
import { writeFileAtomically as writeFileAtomically7 } from "@remnic/core/maintenance/atomic-file";
|
|
50666
51688
|
import { readFile as readFile31 } from "fs/promises";
|
|
50667
|
-
import
|
|
51689
|
+
import path49 from "path";
|
|
50668
51690
|
var ONE_SIDED_95_Z = 1.6448536269514722;
|
|
50669
51691
|
function oneSidedWilsonLower95(successes, total) {
|
|
50670
51692
|
if (!Number.isInteger(successes) || !Number.isInteger(total) || total < 1 || successes < 0 || successes > total) {
|
|
@@ -50846,13 +51868,13 @@ function parseEpisodes(text) {
|
|
|
50846
51868
|
}
|
|
50847
51869
|
async function computeInjectionSuiteRun(runDir) {
|
|
50848
51870
|
const [metadata, design, episodeText] = await Promise.all([
|
|
50849
|
-
readFile31(
|
|
51871
|
+
readFile31(path49.join(runDir, "run.json"), "utf8").then(
|
|
50850
51872
|
(text) => JSON.parse(text)
|
|
50851
51873
|
),
|
|
50852
|
-
readFile31(
|
|
51874
|
+
readFile31(path49.join(runDir, "expected-design.json"), "utf8").then(
|
|
50853
51875
|
(text) => JSON.parse(text)
|
|
50854
51876
|
),
|
|
50855
|
-
readFile31(
|
|
51877
|
+
readFile31(path49.join(runDir, "episodes.jsonl"), "utf8")
|
|
50856
51878
|
]);
|
|
50857
51879
|
verifyFrozenDesign(design, metadata);
|
|
50858
51880
|
verifyFrozenDecisionRule(metadata);
|
|
@@ -50874,14 +51896,14 @@ async function computeInjectionSuiteRun(runDir) {
|
|
|
50874
51896
|
async function analyzeInjectionSuiteRun(runDir) {
|
|
50875
51897
|
const analysis = await computeInjectionSuiteRun(runDir);
|
|
50876
51898
|
await writeFileAtomically7(
|
|
50877
|
-
|
|
51899
|
+
path49.join(runDir, "statistics.json"),
|
|
50878
51900
|
`${JSON.stringify(analysis, null, 2)}
|
|
50879
51901
|
`
|
|
50880
51902
|
);
|
|
50881
51903
|
return analysis;
|
|
50882
51904
|
}
|
|
50883
51905
|
async function replayInjectionSuiteStatistics(runDir) {
|
|
50884
|
-
const frozen = await readFile31(
|
|
51906
|
+
const frozen = await readFile31(path49.join(runDir, "statistics.json"), "utf8");
|
|
50885
51907
|
const analysis = await computeInjectionSuiteRun(runDir);
|
|
50886
51908
|
const replayed = `${JSON.stringify(analysis, null, 2)}
|
|
50887
51909
|
`;
|
|
@@ -51032,15 +52054,15 @@ function analyzeInjectionSuiteUtility(observations, plan) {
|
|
|
51032
52054
|
|
|
51033
52055
|
// src/security/injection-suite/utility-runner.ts
|
|
51034
52056
|
import { writeFileAtomically as writeFileAtomically8 } from "@remnic/core/maintenance/atomic-file";
|
|
51035
|
-
import { createHash as
|
|
52057
|
+
import { createHash as createHash32, randomUUID as randomUUID38 } from "crypto";
|
|
51036
52058
|
import {
|
|
51037
52059
|
closeSync,
|
|
51038
|
-
existsSync as
|
|
52060
|
+
existsSync as existsSync5,
|
|
51039
52061
|
fsyncSync,
|
|
51040
52062
|
mkdirSync as mkdirSync2,
|
|
51041
52063
|
openSync,
|
|
51042
|
-
readdirSync,
|
|
51043
|
-
readFileSync as
|
|
52064
|
+
readdirSync as readdirSync2,
|
|
52065
|
+
readFileSync as readFileSync5,
|
|
51044
52066
|
realpathSync as realpathSync2,
|
|
51045
52067
|
statSync,
|
|
51046
52068
|
renameSync,
|
|
@@ -51048,12 +52070,12 @@ import {
|
|
|
51048
52070
|
writeFileSync
|
|
51049
52071
|
} from "fs";
|
|
51050
52072
|
import { readFile as readFile32 } from "fs/promises";
|
|
51051
|
-
import
|
|
52073
|
+
import path50 from "path";
|
|
51052
52074
|
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
51053
52075
|
var UTILITY_ARMS = ["none", "fencing"];
|
|
51054
52076
|
var UTILITY_SEEDS = [1, 2, 3, 4, 5];
|
|
51055
52077
|
var UTILITY_SEED_CONCURRENCY = 2;
|
|
51056
|
-
var DRIFT_ROOT =
|
|
52078
|
+
var DRIFT_ROOT = path50.resolve(
|
|
51057
52079
|
fileURLToPath4(new URL("../../fixtures/drift-gen-core/11", import.meta.url))
|
|
51058
52080
|
);
|
|
51059
52081
|
function isRetryableUtilityFailure(task) {
|
|
@@ -51063,36 +52085,36 @@ function isRetryableUtilityFailure(task) {
|
|
|
51063
52085
|
var UtilityCheckpointStore = class {
|
|
51064
52086
|
directory;
|
|
51065
52087
|
constructor(root, arm, seed, benchmark) {
|
|
51066
|
-
this.directory =
|
|
52088
|
+
this.directory = path50.join(root, "utility-checkpoints", arm, String(seed), benchmark);
|
|
51067
52089
|
mkdirSync2(this.directory, { recursive: true });
|
|
51068
52090
|
}
|
|
51069
52091
|
key(taskId) {
|
|
51070
|
-
return
|
|
52092
|
+
return createHash32("sha256").update(taskId).digest("hex");
|
|
51071
52093
|
}
|
|
51072
52094
|
checkpointPath(taskId) {
|
|
51073
|
-
return
|
|
52095
|
+
return path50.join(this.directory, `${this.key(taskId)}.json`);
|
|
51074
52096
|
}
|
|
51075
52097
|
inFlightPath(taskId) {
|
|
51076
|
-
return
|
|
52098
|
+
return path50.join(this.directory, `${this.key(taskId)}.inflight`);
|
|
51077
52099
|
}
|
|
51078
52100
|
loadTasks() {
|
|
51079
52101
|
const tasks = /* @__PURE__ */ new Map();
|
|
51080
|
-
for (const entry of
|
|
52102
|
+
for (const entry of readdirSync2(this.directory)) {
|
|
51081
52103
|
if (!entry.endsWith(".json")) continue;
|
|
51082
|
-
const task = JSON.parse(
|
|
52104
|
+
const task = JSON.parse(readFileSync5(path50.join(this.directory, entry), "utf8"));
|
|
51083
52105
|
if (!isRetryableUtilityFailure(task)) tasks.set(task.taskId, task);
|
|
51084
52106
|
}
|
|
51085
52107
|
return tasks;
|
|
51086
52108
|
}
|
|
51087
52109
|
ambiguousTaskIds() {
|
|
51088
52110
|
const ambiguous = [];
|
|
51089
|
-
for (const entry of
|
|
52111
|
+
for (const entry of readdirSync2(this.directory)) {
|
|
51090
52112
|
if (!entry.endsWith(".inflight")) continue;
|
|
51091
|
-
const taskId =
|
|
52113
|
+
const taskId = readFileSync5(path50.join(this.directory, entry), "utf8").trim();
|
|
51092
52114
|
if (!taskId) continue;
|
|
51093
52115
|
const checkpoint = this.checkpointPath(taskId);
|
|
51094
|
-
if (
|
|
51095
|
-
rmSync(
|
|
52116
|
+
if (existsSync5(checkpoint) && !isRetryableUtilityFailure(JSON.parse(readFileSync5(checkpoint, "utf8")))) {
|
|
52117
|
+
rmSync(path50.join(this.directory, entry), { force: true });
|
|
51096
52118
|
continue;
|
|
51097
52119
|
}
|
|
51098
52120
|
ambiguous.push(taskId);
|
|
@@ -51119,8 +52141,8 @@ var UtilityCheckpointStore = class {
|
|
|
51119
52141
|
}
|
|
51120
52142
|
commit(task) {
|
|
51121
52143
|
const target = this.checkpointPath(task.taskId);
|
|
51122
|
-
if (
|
|
51123
|
-
const existing = JSON.parse(
|
|
52144
|
+
if (existsSync5(target)) {
|
|
52145
|
+
const existing = JSON.parse(readFileSync5(target, "utf8"));
|
|
51124
52146
|
if (isRetryableUtilityFailure(existing)) renameSync(target, `${target}.failed-${randomUUID38()}`);
|
|
51125
52147
|
}
|
|
51126
52148
|
this.writeAtomicSync(target, `${JSON.stringify(task, null, 2)}
|
|
@@ -51137,7 +52159,7 @@ var UtilityCheckpointStore = class {
|
|
|
51137
52159
|
closeSync(handle);
|
|
51138
52160
|
}
|
|
51139
52161
|
renameSync(temporary, target);
|
|
51140
|
-
const directory = openSync(
|
|
52162
|
+
const directory = openSync(path50.dirname(target), "r");
|
|
51141
52163
|
try {
|
|
51142
52164
|
fsyncSync(directory);
|
|
51143
52165
|
} finally {
|
|
@@ -51276,16 +52298,16 @@ async function runPublishedUtility(input, arm, seed, benchmark, datasetDir, plan
|
|
|
51276
52298
|
}
|
|
51277
52299
|
}
|
|
51278
52300
|
function readJsonLines(file) {
|
|
51279
|
-
return
|
|
52301
|
+
return readFileSync5(file, "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
51280
52302
|
}
|
|
51281
52303
|
async function runDriftUtility(input, arm, seed, plannedItems) {
|
|
51282
52304
|
const store = new UtilityCheckpointStore(input.outputDir, arm, seed, "drift-gen");
|
|
51283
52305
|
assertNoAmbiguousUtilityTask(store, input.retryAmbiguous === true);
|
|
51284
52306
|
const completed = store.loadTasks();
|
|
51285
|
-
const probes = readJsonLines(
|
|
52307
|
+
const probes = readJsonLines(path50.join(DRIFT_ROOT, "gold", "probes.jsonl"));
|
|
51286
52308
|
for (const probe of probes) plannedItems.add(`drift-gen\0${probe.id}`);
|
|
51287
52309
|
const sessions = ["u1", "u2"].flatMap(
|
|
51288
|
-
(userId) => readJsonLines(
|
|
52310
|
+
(userId) => readJsonLines(path50.join(DRIFT_ROOT, "users", userId, "sessions.jsonl"))
|
|
51289
52311
|
);
|
|
51290
52312
|
const adapter = await createUtilityAdapter(input, arm, seed);
|
|
51291
52313
|
const observations = [];
|
|
@@ -51341,7 +52363,7 @@ async function runDriftUtility(input, arm, seed, plannedItems) {
|
|
|
51341
52363
|
var UTILITY_CONTRACT_FILE = "utility-contract.json";
|
|
51342
52364
|
function datasetDigest(directory) {
|
|
51343
52365
|
if (!directory) return null;
|
|
51344
|
-
const root =
|
|
52366
|
+
const root = path50.resolve(directory);
|
|
51345
52367
|
const files = [];
|
|
51346
52368
|
const visited = /* @__PURE__ */ new Set();
|
|
51347
52369
|
const walk = (dir) => {
|
|
@@ -51353,8 +52375,8 @@ function datasetDigest(directory) {
|
|
|
51353
52375
|
}
|
|
51354
52376
|
if (visited.has(real)) return;
|
|
51355
52377
|
visited.add(real);
|
|
51356
|
-
for (const entry of
|
|
51357
|
-
const full =
|
|
52378
|
+
for (const entry of readdirSync2(dir, { withFileTypes: true }).sort((a, b) => a.name < b.name ? -1 : 1)) {
|
|
52379
|
+
const full = path50.join(dir, entry.name);
|
|
51358
52380
|
if (entry.isFile()) {
|
|
51359
52381
|
files.push(full);
|
|
51360
52382
|
continue;
|
|
@@ -51375,11 +52397,11 @@ function datasetDigest(directory) {
|
|
|
51375
52397
|
}
|
|
51376
52398
|
};
|
|
51377
52399
|
walk(root);
|
|
51378
|
-
const digest =
|
|
52400
|
+
const digest = createHash32("sha256");
|
|
51379
52401
|
for (const file of files.sort()) {
|
|
51380
|
-
digest.update(
|
|
52402
|
+
digest.update(path50.relative(root, file).split(path50.sep).join("/"));
|
|
51381
52403
|
digest.update("\0");
|
|
51382
|
-
digest.update(
|
|
52404
|
+
digest.update(createHash32("sha256").update(readFileSync5(file)).digest("hex"));
|
|
51383
52405
|
digest.update("\0");
|
|
51384
52406
|
}
|
|
51385
52407
|
return `sha256:${digest.digest("hex")}:${files.length}`;
|
|
@@ -51408,7 +52430,7 @@ function utilityContract(input, benchmarks) {
|
|
|
51408
52430
|
};
|
|
51409
52431
|
}
|
|
51410
52432
|
async function assertUtilityContract(input, benchmarks) {
|
|
51411
|
-
const file =
|
|
52433
|
+
const file = path50.join(input.outputDir, UTILITY_CONTRACT_FILE);
|
|
51412
52434
|
const current = `${JSON.stringify(utilityContract(input, benchmarks), null, 2)}
|
|
51413
52435
|
`;
|
|
51414
52436
|
let existing;
|
|
@@ -51418,7 +52440,7 @@ async function assertUtilityContract(input, benchmarks) {
|
|
|
51418
52440
|
if (error.code !== "ENOENT") throw error;
|
|
51419
52441
|
}
|
|
51420
52442
|
if (existing === void 0) {
|
|
51421
|
-
if (
|
|
52443
|
+
if (existsSync5(path50.join(input.outputDir, "utility-checkpoints"))) {
|
|
51422
52444
|
throw new Error(
|
|
51423
52445
|
`${input.outputDir} holds utility checkpoints but no ${UTILITY_CONTRACT_FILE}; use a fresh output directory`
|
|
51424
52446
|
);
|
|
@@ -51444,7 +52466,7 @@ async function runInjectionSuiteUtility(input) {
|
|
|
51444
52466
|
throw new Error("H5 LongMemEval utility requires --longmemeval-dataset-dir");
|
|
51445
52467
|
}
|
|
51446
52468
|
if (benchmarks.includes("drift-gen")) {
|
|
51447
|
-
await readFile32(
|
|
52469
|
+
await readFile32(path50.join(DRIFT_ROOT, "..", "dataset.manifest.json"), "utf8");
|
|
51448
52470
|
}
|
|
51449
52471
|
enforceRunIdentityGate(input, !captureBenchmarkExecutionProvenance().gitDirty);
|
|
51450
52472
|
mkdirSync2(input.outputDir, { recursive: true });
|
|
@@ -51470,7 +52492,7 @@ async function runInjectionSuiteUtility(input) {
|
|
|
51470
52492
|
}));
|
|
51471
52493
|
observations.push(...bySeed.flat());
|
|
51472
52494
|
await writeFileAtomically8(
|
|
51473
|
-
|
|
52495
|
+
path50.join(input.outputDir, "utility-observations.json"),
|
|
51474
52496
|
`${JSON.stringify(observations, null, 2)}
|
|
51475
52497
|
`
|
|
51476
52498
|
);
|
|
@@ -51485,13 +52507,13 @@ async function runInjectionSuiteUtility(input) {
|
|
|
51485
52507
|
})
|
|
51486
52508
|
};
|
|
51487
52509
|
await writeFileAtomically8(
|
|
51488
|
-
|
|
52510
|
+
path50.join(input.outputDir, "utility-plan.json"),
|
|
51489
52511
|
`${JSON.stringify(plan, null, 2)}
|
|
51490
52512
|
`
|
|
51491
52513
|
);
|
|
51492
52514
|
const analysis = analyzeInjectionSuiteUtility(observations, plan);
|
|
51493
52515
|
await writeFileAtomically8(
|
|
51494
|
-
|
|
52516
|
+
path50.join(input.outputDir, "utility-statistics.json"),
|
|
51495
52517
|
`${JSON.stringify(analysis, null, 2)}
|
|
51496
52518
|
`
|
|
51497
52519
|
);
|
|
@@ -51501,13 +52523,13 @@ async function runInjectionSuiteUtility(input) {
|
|
|
51501
52523
|
// src/security/injection-suite/campaign.ts
|
|
51502
52524
|
import { writeFileAtomically as writeFileAtomically10 } from "@remnic/core/maintenance/atomic-file";
|
|
51503
52525
|
import { readFile as readFile33 } from "fs/promises";
|
|
51504
|
-
import
|
|
52526
|
+
import path52 from "path";
|
|
51505
52527
|
|
|
51506
52528
|
// src/security/injection-suite/report.ts
|
|
51507
52529
|
import { writeFileAtomically as writeFileAtomically9 } from "@remnic/core/maintenance/atomic-file";
|
|
51508
|
-
import { createHash as
|
|
52530
|
+
import { createHash as createHash33 } from "crypto";
|
|
51509
52531
|
import { mkdir as mkdir21 } from "fs/promises";
|
|
51510
|
-
import
|
|
52532
|
+
import path51 from "path";
|
|
51511
52533
|
function csvCell(value) {
|
|
51512
52534
|
const text = value === null || value === void 0 ? "" : String(value);
|
|
51513
52535
|
return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
|
|
@@ -51516,7 +52538,7 @@ function format(value) {
|
|
|
51516
52538
|
return value === null ? "NA" : value.toFixed(4);
|
|
51517
52539
|
}
|
|
51518
52540
|
function sha2568(value) {
|
|
51519
|
-
return
|
|
52541
|
+
return createHash33("sha256").update(value).digest("hex");
|
|
51520
52542
|
}
|
|
51521
52543
|
function familyCsv(base) {
|
|
51522
52544
|
const rows = [[
|
|
@@ -51631,10 +52653,10 @@ ${adaptiveRows}
|
|
|
51631
52653
|
`;
|
|
51632
52654
|
}
|
|
51633
52655
|
async function writeInjectionSuiteReport(input) {
|
|
51634
|
-
const paperDir =
|
|
52656
|
+
const paperDir = path51.join(input.outputDir, "paper");
|
|
51635
52657
|
await Promise.all([
|
|
51636
|
-
mkdir21(
|
|
51637
|
-
mkdir21(
|
|
52658
|
+
mkdir21(path51.join(paperDir, "tables"), { recursive: true }),
|
|
52659
|
+
mkdir21(path51.join(paperDir, "figures"), { recursive: true })
|
|
51638
52660
|
]);
|
|
51639
52661
|
const artifacts = {
|
|
51640
52662
|
"report.md": reportMarkdown(input),
|
|
@@ -51642,7 +52664,7 @@ async function writeInjectionSuiteReport(input) {
|
|
|
51642
52664
|
"figures/block-rates.svg": blockRateSvg(input.base)
|
|
51643
52665
|
};
|
|
51644
52666
|
for (const [relative, content] of Object.entries(artifacts)) {
|
|
51645
|
-
await writeFileAtomically9(
|
|
52667
|
+
await writeFileAtomically9(path51.join(paperDir, relative), content);
|
|
51646
52668
|
}
|
|
51647
52669
|
const manifest = {
|
|
51648
52670
|
schemaVersion: 1,
|
|
@@ -51652,7 +52674,7 @@ async function writeInjectionSuiteReport(input) {
|
|
|
51652
52674
|
)
|
|
51653
52675
|
};
|
|
51654
52676
|
await writeFileAtomically9(
|
|
51655
|
-
|
|
52677
|
+
path51.join(paperDir, "report-manifest.json"),
|
|
51656
52678
|
`${JSON.stringify(manifest, null, 2)}
|
|
51657
52679
|
`
|
|
51658
52680
|
);
|
|
@@ -51718,7 +52740,7 @@ async function decideInjectionSuiteCampaign(input) {
|
|
|
51718
52740
|
const adaptive = await Promise.all((input.adaptiveRunDirs ?? []).map(analyzeInjectionSuiteRun));
|
|
51719
52741
|
const decision = decideInjectionSuiteCampaignResults(base, utility, adaptive);
|
|
51720
52742
|
await writeFileAtomically10(
|
|
51721
|
-
|
|
52743
|
+
path52.join(input.outputDir, "campaign-decision.json"),
|
|
51722
52744
|
`${JSON.stringify(decision, null, 2)}
|
|
51723
52745
|
`
|
|
51724
52746
|
);
|
|
@@ -51733,7 +52755,7 @@ async function decideInjectionSuiteCampaign(input) {
|
|
|
51733
52755
|
}
|
|
51734
52756
|
|
|
51735
52757
|
// src/coding-graph/generator.ts
|
|
51736
|
-
import { createHash as
|
|
52758
|
+
import { createHash as createHash34 } from "crypto";
|
|
51737
52759
|
function createSeededRng3(seed) {
|
|
51738
52760
|
let state = seed >>> 0;
|
|
51739
52761
|
return function rng() {
|
|
@@ -51762,7 +52784,7 @@ var EDGE_TYPE_WEIGHTS = [
|
|
|
51762
52784
|
var PROVENANCE_VALUES = ["heuristic", "heuristic", "heuristic", "trace"];
|
|
51763
52785
|
var AVG_BYTES_PER_LINE = 40;
|
|
51764
52786
|
function hashContent(input) {
|
|
51765
|
-
return
|
|
52787
|
+
return createHash34("sha256").update(input).digest("hex").slice(0, 16);
|
|
51766
52788
|
}
|
|
51767
52789
|
function generateSyntheticRepo(config) {
|
|
51768
52790
|
const rng = createSeededRng3(config.seed);
|
|
@@ -51860,7 +52882,7 @@ import { performance as performance2 } from "perf_hooks";
|
|
|
51860
52882
|
import { mkdtemp as mkdtemp14, rm as rm19 } from "fs/promises";
|
|
51861
52883
|
import { statSync as statSync2 } from "fs";
|
|
51862
52884
|
import { tmpdir as tmpdir7 } from "os";
|
|
51863
|
-
import
|
|
52885
|
+
import path53 from "path";
|
|
51864
52886
|
import os10 from "os";
|
|
51865
52887
|
import {
|
|
51866
52888
|
GraphStore
|
|
@@ -51957,15 +52979,15 @@ async function runCodingGraphBenchmark(config = {}) {
|
|
|
51957
52979
|
const sampleRss = () => {
|
|
51958
52980
|
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
|
51959
52981
|
};
|
|
51960
|
-
const dir = await mkdtemp14(
|
|
51961
|
-
const dbPath =
|
|
52982
|
+
const dir = await mkdtemp14(path53.join(tmpdir7(), "coding-graph-bench-"));
|
|
52983
|
+
const dbPath = path53.join(dir, "bench.sqlite");
|
|
51962
52984
|
try {
|
|
51963
52985
|
const store = await GraphStore.open({ dbPath });
|
|
51964
52986
|
try {
|
|
51965
52987
|
const FULL_INDEX_SAMPLES = 3;
|
|
51966
52988
|
const fullIndexSamples = [];
|
|
51967
52989
|
for (let s = 0; s < FULL_INDEX_SAMPLES; s++) {
|
|
51968
|
-
const sampleStore = s === 0 ? store : await GraphStore.open({ dbPath:
|
|
52990
|
+
const sampleStore = s === 0 ? store : await GraphStore.open({ dbPath: path53.join(dir, `bench-warm-${s}.sqlite`) });
|
|
51969
52991
|
const fi = await timeAsync(() => sampleStore.upsertFileBatch(storeFiles));
|
|
51970
52992
|
if (!fi.result.ok) {
|
|
51971
52993
|
if (sampleStore !== store) await sampleStore.close();
|
|
@@ -52288,12 +53310,12 @@ function buildBaselineFromReport(report, note) {
|
|
|
52288
53310
|
// src/coding-graph/repeated-failure-report.ts
|
|
52289
53311
|
import { constants } from "fs";
|
|
52290
53312
|
import { lstat as lstat7, mkdir as mkdir22, open as open4 } from "fs/promises";
|
|
52291
|
-
import
|
|
53313
|
+
import path54 from "path";
|
|
52292
53314
|
import { writeFileAtomically as writeFileAtomically11 } from "@remnic/core/maintenance/atomic-file";
|
|
52293
53315
|
import { z as z4 } from "zod";
|
|
52294
53316
|
|
|
52295
53317
|
// src/coding-graph/repeated-failure-report-rendering.ts
|
|
52296
|
-
import { createHash as
|
|
53318
|
+
import { createHash as createHash35 } from "crypto";
|
|
52297
53319
|
import { z as z3 } from "zod";
|
|
52298
53320
|
var SHA2562 = /^[a-f0-9]{64}$/;
|
|
52299
53321
|
var IntervalSchema = z3.object({
|
|
@@ -52577,7 +53599,7 @@ function assessClaimEligibility(run, statistics, audit, evidence) {
|
|
|
52577
53599
|
};
|
|
52578
53600
|
}
|
|
52579
53601
|
function sha2569(value) {
|
|
52580
|
-
return
|
|
53602
|
+
return createHash35("sha256").update(value).digest("hex");
|
|
52581
53603
|
}
|
|
52582
53604
|
function formatNumber(value, digits = 6) {
|
|
52583
53605
|
return value === null ? "NA" : value.toFixed(digits);
|
|
@@ -52619,22 +53641,22 @@ function aggregateArmOutcomes(rows) {
|
|
|
52619
53641
|
const first = group[0];
|
|
52620
53642
|
if (!first) throw new Error("arm outcome group cannot be empty");
|
|
52621
53643
|
const valid = group.filter((row) => row.status === "VALID");
|
|
52622
|
-
const
|
|
53644
|
+
const mean6 = (values) => values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
52623
53645
|
return {
|
|
52624
53646
|
modelProfileId: first.identity.modelProfileId,
|
|
52625
53647
|
modelProfileHash: first.identity.modelProfileHash,
|
|
52626
53648
|
arm: first.identity.arm,
|
|
52627
53649
|
validRows: valid.length,
|
|
52628
53650
|
invalidRows: group.length - valid.length,
|
|
52629
|
-
repeatedFailureRate:
|
|
52630
|
-
taskPassRate:
|
|
52631
|
-
meanSteps:
|
|
53651
|
+
repeatedFailureRate: mean6(valid.map((row) => row.repeatedFailure ? 1 : 0)),
|
|
53652
|
+
taskPassRate: mean6(valid.map((row) => row.taskPassed ? 1 : 0)),
|
|
53653
|
+
meanSteps: mean6(valid.map(
|
|
52632
53654
|
(row) => requireNonnegativeInteger(row.steps, "steps", row.rowKey)
|
|
52633
53655
|
)),
|
|
52634
|
-
warningRate:
|
|
53656
|
+
warningRate: mean6(valid.map(
|
|
52635
53657
|
(row) => requireNonnegativeInteger(row.warningCount, "warningCount", row.rowKey) > 0 ? 1 : 0
|
|
52636
53658
|
)),
|
|
52637
|
-
falseWarningRate:
|
|
53659
|
+
falseWarningRate: mean6(valid.map(
|
|
52638
53660
|
(row) => requireNonnegativeInteger(row.falseWarningCount, "falseWarningCount", row.rowKey) > 0 ? 1 : 0
|
|
52639
53661
|
))
|
|
52640
53662
|
};
|
|
@@ -52916,20 +53938,20 @@ var SOURCE_ARTIFACTS = [
|
|
|
52916
53938
|
async function readArtifactLeaf(filePath) {
|
|
52917
53939
|
const leaf = await lstat7(filePath);
|
|
52918
53940
|
if (!leaf.isFile()) {
|
|
52919
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
53941
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path54.basename(filePath)}`);
|
|
52920
53942
|
}
|
|
52921
53943
|
let handle;
|
|
52922
53944
|
try {
|
|
52923
53945
|
handle = await open4(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
52924
53946
|
} catch (error) {
|
|
52925
53947
|
if (error.code === "ELOOP") {
|
|
52926
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
53948
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path54.basename(filePath)}`);
|
|
52927
53949
|
}
|
|
52928
53950
|
throw error;
|
|
52929
53951
|
}
|
|
52930
53952
|
try {
|
|
52931
53953
|
if (!(await handle.stat()).isFile()) {
|
|
52932
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
53954
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path54.basename(filePath)}`);
|
|
52933
53955
|
}
|
|
52934
53956
|
return await handle.readFile();
|
|
52935
53957
|
} finally {
|
|
@@ -53133,7 +54155,7 @@ async function assertArtifactMayBeWritten(filePath, content) {
|
|
|
53133
54155
|
try {
|
|
53134
54156
|
const prior = await readArtifactLeaf(filePath);
|
|
53135
54157
|
if (!prior.equals(Buffer.from(content))) {
|
|
53136
|
-
throw new Error(`paper writer refuses to overwrite changed paper artifact: ${
|
|
54158
|
+
throw new Error(`paper writer refuses to overwrite changed paper artifact: ${path54.basename(filePath)}`);
|
|
53137
54159
|
}
|
|
53138
54160
|
return false;
|
|
53139
54161
|
} catch (error) {
|
|
@@ -53142,7 +54164,7 @@ async function assertArtifactMayBeWritten(filePath, content) {
|
|
|
53142
54164
|
}
|
|
53143
54165
|
}
|
|
53144
54166
|
async function writeRepeatedFailurePaperArtifacts(options) {
|
|
53145
|
-
const runDir =
|
|
54167
|
+
const runDir = path54.resolve(options.runDir);
|
|
53146
54168
|
const reproManifest = await verifyRunManifest(runDir);
|
|
53147
54169
|
const source = await readSourceArtifacts(runDir);
|
|
53148
54170
|
const runJson = JSON.parse(source["run.json"]);
|
|
@@ -53160,7 +54182,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
53160
54182
|
throw new Error("paper report preregistration does not match run metadata");
|
|
53161
54183
|
}
|
|
53162
54184
|
const committedFixtureDir = await resolveCommittedH6FixtureDirectory();
|
|
53163
|
-
const committedDecisionRuleBytes = (await readArtifactLeaf(
|
|
54185
|
+
const committedDecisionRuleBytes = (await readArtifactLeaf(path54.join(committedFixtureDir, "decision-rule.json"))).toString("utf8");
|
|
53164
54186
|
if (decisionRuleBytes !== committedDecisionRuleBytes) {
|
|
53165
54187
|
throw new Error("paper report decision rule differs from the frozen committed artifact");
|
|
53166
54188
|
}
|
|
@@ -53519,7 +54541,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
53519
54541
|
await Promise.all(writeStates.map(async (artifact) => {
|
|
53520
54542
|
const bytes = await readArtifactLeaf(artifact.artifactPath);
|
|
53521
54543
|
if (!bytes.equals(Buffer.from(artifact.content))) {
|
|
53522
|
-
throw new Error(`paper artifact verification failed: ${
|
|
54544
|
+
throw new Error(`paper artifact verification failed: ${path54.basename(artifact.artifactPath)}`);
|
|
53523
54545
|
}
|
|
53524
54546
|
}));
|
|
53525
54547
|
const artifactPaths = writeStates.map((artifact) => artifact.artifactPath);
|
|
@@ -53539,8 +54561,8 @@ async function runRepeatedFailurePaperReportCliCommand(options) {
|
|
|
53539
54561
|
return {
|
|
53540
54562
|
exitCode: 0,
|
|
53541
54563
|
output: JSON.stringify({
|
|
53542
|
-
reportPath:
|
|
53543
|
-
manifestPath:
|
|
54564
|
+
reportPath: path54.relative(path54.resolve(options.runDir), result.reportPath),
|
|
54565
|
+
manifestPath: path54.relative(path54.resolve(options.runDir), result.manifestPath)
|
|
53544
54566
|
})
|
|
53545
54567
|
};
|
|
53546
54568
|
} catch (error) {
|
|
@@ -53551,7 +54573,7 @@ async function runRepeatedFailurePaperReportCliCommand(options) {
|
|
|
53551
54573
|
// src/attribute-cli.ts
|
|
53552
54574
|
import { QmdClient } from "@remnic/core";
|
|
53553
54575
|
import { lstat as lstat8, readdir as readdir9, readFile as readFile34 } from "fs/promises";
|
|
53554
|
-
import
|
|
54576
|
+
import path55 from "path";
|
|
53555
54577
|
function parseFrontmatter2(fileContent) {
|
|
53556
54578
|
const lines = fileContent.split(/\r?\n/);
|
|
53557
54579
|
if (lines.length > 0 && lines[0].trim() === "---") {
|
|
@@ -53615,7 +54637,7 @@ async function scanMemoryDir(dirPath) {
|
|
|
53615
54637
|
if (entry.isSymbolicLink()) {
|
|
53616
54638
|
continue;
|
|
53617
54639
|
}
|
|
53618
|
-
const fullPath =
|
|
54640
|
+
const fullPath = path55.join(currentDir, entry.name);
|
|
53619
54641
|
try {
|
|
53620
54642
|
const stats = await lstat8(fullPath);
|
|
53621
54643
|
if (stats.isSymbolicLink()) {
|
|
@@ -53629,7 +54651,7 @@ async function scanMemoryDir(dirPath) {
|
|
|
53629
54651
|
} else if (stats.isFile() && entry.name.endsWith(".md")) {
|
|
53630
54652
|
const content = await readFile34(fullPath, "utf8");
|
|
53631
54653
|
const { id, body } = parseFrontmatter2(content);
|
|
53632
|
-
const relPath =
|
|
54654
|
+
const relPath = path55.relative(dirPath, fullPath);
|
|
53633
54655
|
memories.push({
|
|
53634
54656
|
id: id ?? relPath,
|
|
53635
54657
|
content: body.trim()
|
|
@@ -53647,24 +54669,24 @@ async function scanMemoryDir(dirPath) {
|
|
|
53647
54669
|
return memories;
|
|
53648
54670
|
}
|
|
53649
54671
|
async function resolveQmdMemory(memoryDir, collection, resultPath) {
|
|
53650
|
-
const root =
|
|
54672
|
+
const root = path55.resolve(memoryDir);
|
|
53651
54673
|
const candidates = /* @__PURE__ */ new Set();
|
|
53652
54674
|
const addCandidate = (candidate) => {
|
|
53653
|
-
const resolved =
|
|
53654
|
-
const relative =
|
|
53655
|
-
if (relative !== ".." && !relative.startsWith(`..${
|
|
54675
|
+
const resolved = path55.resolve(candidate);
|
|
54676
|
+
const relative = path55.relative(root, resolved);
|
|
54677
|
+
if (relative !== ".." && !relative.startsWith(`..${path55.sep}`) && !path55.isAbsolute(relative)) {
|
|
53656
54678
|
candidates.add(resolved);
|
|
53657
54679
|
}
|
|
53658
54680
|
};
|
|
53659
54681
|
const addRelative = (relativePath) => {
|
|
53660
54682
|
const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
53661
54683
|
if (!normalized) return;
|
|
53662
|
-
addCandidate(
|
|
54684
|
+
addCandidate(path55.join(root, normalized));
|
|
53663
54685
|
if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
|
|
53664
|
-
addCandidate(
|
|
54686
|
+
addCandidate(path55.join(root, "facts", normalized));
|
|
53665
54687
|
}
|
|
53666
54688
|
};
|
|
53667
|
-
if (
|
|
54689
|
+
if (path55.isAbsolute(resultPath)) {
|
|
53668
54690
|
addCandidate(resultPath);
|
|
53669
54691
|
} else {
|
|
53670
54692
|
addRelative(resultPath);
|
|
@@ -53908,6 +54930,7 @@ export {
|
|
|
53908
54930
|
ONLINE_ADAPTIVE_STAGE,
|
|
53909
54931
|
OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION,
|
|
53910
54932
|
OTHER_NAMESPACE_MEMORIES,
|
|
54933
|
+
OUTCOME_PRIOR_CLI_COMMAND,
|
|
53911
54934
|
OpenAiResponsesJudgeError,
|
|
53912
54935
|
OpenAiResponsesProvider,
|
|
53913
54936
|
PROCEDURAL_REAL_SCENARIOS,
|
|
@@ -54057,6 +55080,7 @@ export {
|
|
|
54057
55080
|
discoveryEndpointFor,
|
|
54058
55081
|
emailFixture,
|
|
54059
55082
|
entityRecall,
|
|
55083
|
+
evaluateH1Decision,
|
|
54060
55084
|
evaluateTaskState,
|
|
54061
55085
|
exactMatch,
|
|
54062
55086
|
executeLocalRow,
|
|
@@ -54125,6 +55149,8 @@ export {
|
|
|
54125
55149
|
loadLoCoMo10,
|
|
54126
55150
|
loadLocalLabManifest,
|
|
54127
55151
|
loadLongMemEvalS,
|
|
55152
|
+
loadOutcomePriorArms,
|
|
55153
|
+
loadOutcomePriorDecisionRule,
|
|
54128
55154
|
loadSealKeyFromEnv,
|
|
54129
55155
|
loadSealedQrels,
|
|
54130
55156
|
loadSealedRubric,
|
|
@@ -54203,6 +55229,8 @@ export {
|
|
|
54203
55229
|
runInjectionSuiteUtility,
|
|
54204
55230
|
runJudgeCalibration,
|
|
54205
55231
|
runMitigatedBaseline,
|
|
55232
|
+
runOutcomePriorGatesCli,
|
|
55233
|
+
runOutcomePriorScaffoldCli,
|
|
54206
55234
|
runProceduralAblation,
|
|
54207
55235
|
runProceduralAblationCli,
|
|
54208
55236
|
runRepeatedFailureCliCommand,
|