@remnic/bench 9.69.37 → 9.69.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +383 -341
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -13963,7 +13963,7 @@ async function resolveLocalLabRuntimeProfile(options) {
|
|
|
13963
13963
|
|
|
13964
13964
|
// src/benchmark.ts
|
|
13965
13965
|
import fs from "fs";
|
|
13966
|
-
import
|
|
13966
|
+
import path33 from "path";
|
|
13967
13967
|
import { createHash as createHash17 } from "crypto";
|
|
13968
13968
|
import { expandTildePath as expandTildePath3 } from "@remnic/core";
|
|
13969
13969
|
|
|
@@ -23994,8 +23994,8 @@ function isPlainObject4(value) {
|
|
|
23994
23994
|
|
|
23995
23995
|
// src/benchmarks/published/memoryagentbench/runner.ts
|
|
23996
23996
|
import { randomUUID as randomUUID9 } from "crypto";
|
|
23997
|
-
import {
|
|
23998
|
-
import
|
|
23997
|
+
import { readFile as readFile16 } from "fs/promises";
|
|
23998
|
+
import path16 from "path";
|
|
23999
23999
|
|
|
24000
24000
|
// src/benchmarks/published/memoryagentbench/fixture.ts
|
|
24001
24001
|
var MEMORY_AGENT_BENCH_SMOKE_FIXTURE = [
|
|
@@ -24067,6 +24067,172 @@ var MEMORY_AGENT_BENCH_SMOKE_FIXTURE = [
|
|
|
24067
24067
|
}
|
|
24068
24068
|
];
|
|
24069
24069
|
|
|
24070
|
+
// src/benchmarks/published/memoryagentbench/recsys-entity-mapping.ts
|
|
24071
|
+
import { lstatSync, realpathSync } from "fs";
|
|
24072
|
+
import { access, readFile as readFile15 } from "fs/promises";
|
|
24073
|
+
import path15 from "path";
|
|
24074
|
+
async function loadRecSysEntityMapping(datasetDir) {
|
|
24075
|
+
const candidates = recsysEntityMappingCandidates(datasetDir);
|
|
24076
|
+
for (const candidate of candidates) {
|
|
24077
|
+
if (!isSafeRecsysMappingCandidate(candidate, datasetDir)) {
|
|
24078
|
+
continue;
|
|
24079
|
+
}
|
|
24080
|
+
if (!await fileExists(candidate)) {
|
|
24081
|
+
continue;
|
|
24082
|
+
}
|
|
24083
|
+
let parsed;
|
|
24084
|
+
try {
|
|
24085
|
+
parsed = JSON.parse(await readFile15(candidate, "utf8"));
|
|
24086
|
+
} catch (error) {
|
|
24087
|
+
console.error(
|
|
24088
|
+
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} is invalid JSON; trying the next candidate: ${error instanceof Error ? error.message : String(error)}`
|
|
24089
|
+
);
|
|
24090
|
+
continue;
|
|
24091
|
+
}
|
|
24092
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
24093
|
+
console.error(
|
|
24094
|
+
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} must be an object; trying the next candidate.`
|
|
24095
|
+
);
|
|
24096
|
+
continue;
|
|
24097
|
+
}
|
|
24098
|
+
const idToName = /* @__PURE__ */ new Map();
|
|
24099
|
+
let invalidMapping = false;
|
|
24100
|
+
for (const [rawName, rawId] of Object.entries(parsed)) {
|
|
24101
|
+
const id = typeof rawId === "number" ? rawId : Number(rawId);
|
|
24102
|
+
if (!Number.isInteger(id)) {
|
|
24103
|
+
console.error(
|
|
24104
|
+
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} has non-integer id for ${rawName}; trying the next candidate.`
|
|
24105
|
+
);
|
|
24106
|
+
invalidMapping = true;
|
|
24107
|
+
break;
|
|
24108
|
+
}
|
|
24109
|
+
idToName.set(id, extractMovieName(rawName));
|
|
24110
|
+
}
|
|
24111
|
+
if (invalidMapping) {
|
|
24112
|
+
continue;
|
|
24113
|
+
}
|
|
24114
|
+
if (idToName.size === 0) {
|
|
24115
|
+
console.error(
|
|
24116
|
+
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} is empty; trying the next candidate.`
|
|
24117
|
+
);
|
|
24118
|
+
continue;
|
|
24119
|
+
}
|
|
24120
|
+
return {
|
|
24121
|
+
idToName,
|
|
24122
|
+
movieCandidates: [...new Set(idToName.values())],
|
|
24123
|
+
aliasCounts: countMovieAliases([...new Set(idToName.values())]),
|
|
24124
|
+
sourcePath: candidate
|
|
24125
|
+
};
|
|
24126
|
+
}
|
|
24127
|
+
return null;
|
|
24128
|
+
}
|
|
24129
|
+
async function requireRecSysEntityMapping(datasetDir) {
|
|
24130
|
+
const mapping = await loadRecSysEntityMapping(datasetDir);
|
|
24131
|
+
if (!mapping) {
|
|
24132
|
+
throw new Error(
|
|
24133
|
+
`MemoryAgentBench ReDial samples require a valid ReDial entity mapping. Expected one of: ${recsysEntityMappingCandidates(datasetDir).join(", ") || "entity2id.json under the dataset directory"}.`
|
|
24134
|
+
);
|
|
24135
|
+
}
|
|
24136
|
+
return mapping;
|
|
24137
|
+
}
|
|
24138
|
+
function movieAliases(movie) {
|
|
24139
|
+
const aliases = [movie];
|
|
24140
|
+
const titleWithoutYear = movie.replace(/\s*\(\d{4}\)\s*$/, "").trim();
|
|
24141
|
+
if (titleWithoutYear.length >= 2 && titleWithoutYear !== movie) {
|
|
24142
|
+
aliases.push(titleWithoutYear);
|
|
24143
|
+
}
|
|
24144
|
+
const titleWithoutArticle = titleWithoutYear.replace(/^(?:the|a|an)\s+/i, "").trim();
|
|
24145
|
+
if (titleWithoutArticle.length >= 2 && titleWithoutArticle !== titleWithoutYear) {
|
|
24146
|
+
aliases.push(titleWithoutArticle);
|
|
24147
|
+
}
|
|
24148
|
+
return aliases;
|
|
24149
|
+
}
|
|
24150
|
+
function recsysEntityMappingCandidates(datasetDir) {
|
|
24151
|
+
if (!datasetDir) {
|
|
24152
|
+
return [];
|
|
24153
|
+
}
|
|
24154
|
+
const absoluteDatasetDir = path15.resolve(datasetDir);
|
|
24155
|
+
const roots = [
|
|
24156
|
+
absoluteDatasetDir,
|
|
24157
|
+
path15.dirname(absoluteDatasetDir)
|
|
24158
|
+
];
|
|
24159
|
+
const canonicalSuffixes = [
|
|
24160
|
+
path15.join("processed_data", "Recsys_Redial", "entity2id.json"),
|
|
24161
|
+
path15.join("Recsys_Redial", "entity2id.json")
|
|
24162
|
+
];
|
|
24163
|
+
const looseSuffixes = ["entity2id.json"];
|
|
24164
|
+
return [
|
|
24165
|
+
...roots.flatMap(
|
|
24166
|
+
(root) => canonicalSuffixes.map((suffix) => path15.join(root, suffix))
|
|
24167
|
+
),
|
|
24168
|
+
...looseSuffixes.map((suffix) => path15.join(absoluteDatasetDir, suffix))
|
|
24169
|
+
];
|
|
24170
|
+
}
|
|
24171
|
+
function isSafeRecsysMappingCandidate(candidate, datasetDir) {
|
|
24172
|
+
if (!datasetDir) {
|
|
24173
|
+
return false;
|
|
24174
|
+
}
|
|
24175
|
+
const root = path15.dirname(path15.resolve(datasetDir));
|
|
24176
|
+
const resolvedRoot = path15.resolve(root);
|
|
24177
|
+
const resolvedCandidate = path15.resolve(candidate);
|
|
24178
|
+
const rel = path15.relative(resolvedRoot, resolvedCandidate);
|
|
24179
|
+
if (rel === "" || rel === ".." || rel.startsWith(`..${path15.sep}`) || path15.isAbsolute(rel)) {
|
|
24180
|
+
return false;
|
|
24181
|
+
}
|
|
24182
|
+
let current = resolvedRoot;
|
|
24183
|
+
for (const part of rel.split(path15.sep)) {
|
|
24184
|
+
if (part === "" || part === ".") {
|
|
24185
|
+
continue;
|
|
24186
|
+
}
|
|
24187
|
+
current = path15.join(current, part);
|
|
24188
|
+
try {
|
|
24189
|
+
if (lstatSync(current).isSymbolicLink()) {
|
|
24190
|
+
return false;
|
|
24191
|
+
}
|
|
24192
|
+
} catch {
|
|
24193
|
+
return false;
|
|
24194
|
+
}
|
|
24195
|
+
}
|
|
24196
|
+
try {
|
|
24197
|
+
const rootReal = realpathSync(resolvedRoot);
|
|
24198
|
+
const candidateReal = realpathSync(resolvedCandidate);
|
|
24199
|
+
const realRel = path15.relative(rootReal, candidateReal);
|
|
24200
|
+
return realRel !== "" && realRel !== ".." && !realRel.startsWith(`..${path15.sep}`) && !path15.isAbsolute(realRel);
|
|
24201
|
+
} catch {
|
|
24202
|
+
return false;
|
|
24203
|
+
}
|
|
24204
|
+
}
|
|
24205
|
+
async function fileExists(filePath) {
|
|
24206
|
+
try {
|
|
24207
|
+
await access(filePath);
|
|
24208
|
+
return true;
|
|
24209
|
+
} catch {
|
|
24210
|
+
return false;
|
|
24211
|
+
}
|
|
24212
|
+
}
|
|
24213
|
+
function extractMovieName(rawName) {
|
|
24214
|
+
const filename = rawName.split("/").pop() ?? rawName;
|
|
24215
|
+
const decodedFilename = decodeUrlComponentSafely(filename);
|
|
24216
|
+
return decodedFilename.replace(/[_>]+/g, " ").replace(/\((\d{4})\s+film\)$/i, "($1)").replace(/\s+/g, " ").trim();
|
|
24217
|
+
}
|
|
24218
|
+
function decodeUrlComponentSafely(value) {
|
|
24219
|
+
try {
|
|
24220
|
+
return decodeURIComponent(value);
|
|
24221
|
+
} catch {
|
|
24222
|
+
return value;
|
|
24223
|
+
}
|
|
24224
|
+
}
|
|
24225
|
+
function countMovieAliases(movieCandidates) {
|
|
24226
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
24227
|
+
for (const movie of movieCandidates) {
|
|
24228
|
+
for (const alias of movieAliases(movie)) {
|
|
24229
|
+
const normalizedAlias = alias.toLowerCase();
|
|
24230
|
+
counts2.set(normalizedAlias, (counts2.get(normalizedAlias) ?? 0) + 1);
|
|
24231
|
+
}
|
|
24232
|
+
}
|
|
24233
|
+
return counts2;
|
|
24234
|
+
}
|
|
24235
|
+
|
|
24070
24236
|
// src/benchmarks/published/memoryagentbench/runner.ts
|
|
24071
24237
|
var DATASET_SPLITS2 = [
|
|
24072
24238
|
{
|
|
@@ -24926,28 +25092,6 @@ function findMovieCandidateMention(normalizedLine, movie, aliasCounts) {
|
|
|
24926
25092
|
function stripTrailingRecommendationPunctuation(value) {
|
|
24927
25093
|
return value.replace(/^["'`]+/g, "").replace(/["'`.!?;:]+$/g, "").trim();
|
|
24928
25094
|
}
|
|
24929
|
-
function countMovieAliases(movieCandidates) {
|
|
24930
|
-
const counts2 = /* @__PURE__ */ new Map();
|
|
24931
|
-
for (const movie of movieCandidates) {
|
|
24932
|
-
for (const alias of movieAliases(movie)) {
|
|
24933
|
-
const normalizedAlias = alias.toLowerCase();
|
|
24934
|
-
counts2.set(normalizedAlias, (counts2.get(normalizedAlias) ?? 0) + 1);
|
|
24935
|
-
}
|
|
24936
|
-
}
|
|
24937
|
-
return counts2;
|
|
24938
|
-
}
|
|
24939
|
-
function movieAliases(movie) {
|
|
24940
|
-
const aliases = [movie];
|
|
24941
|
-
const titleWithoutYear = movie.replace(/\s*\(\d{4}\)\s*$/, "").trim();
|
|
24942
|
-
if (titleWithoutYear.length >= 2 && titleWithoutYear !== movie) {
|
|
24943
|
-
aliases.push(titleWithoutYear);
|
|
24944
|
-
}
|
|
24945
|
-
const titleWithoutArticle = titleWithoutYear.replace(/^(?:the|a|an)\s+/i, "").trim();
|
|
24946
|
-
if (titleWithoutArticle.length >= 2 && titleWithoutArticle !== titleWithoutYear) {
|
|
24947
|
-
aliases.push(titleWithoutArticle);
|
|
24948
|
-
}
|
|
24949
|
-
return aliases;
|
|
24950
|
-
}
|
|
24951
25095
|
function findDelimitedIndex(haystack, needle) {
|
|
24952
25096
|
let start = 0;
|
|
24953
25097
|
while (start < haystack.length) {
|
|
@@ -25009,108 +25153,6 @@ function editDistance(a, b) {
|
|
|
25009
25153
|
}
|
|
25010
25154
|
return previous[b.length] ?? 0;
|
|
25011
25155
|
}
|
|
25012
|
-
async function loadRecSysEntityMapping(datasetDir) {
|
|
25013
|
-
const candidates = recsysEntityMappingCandidates(datasetDir);
|
|
25014
|
-
for (const candidate of candidates) {
|
|
25015
|
-
if (!await fileExists(candidate)) {
|
|
25016
|
-
continue;
|
|
25017
|
-
}
|
|
25018
|
-
let parsed;
|
|
25019
|
-
try {
|
|
25020
|
-
parsed = JSON.parse(await readFile15(candidate, "utf8"));
|
|
25021
|
-
} catch (error) {
|
|
25022
|
-
console.error(
|
|
25023
|
-
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} is invalid JSON; trying the next candidate: ${error instanceof Error ? error.message : String(error)}`
|
|
25024
|
-
);
|
|
25025
|
-
continue;
|
|
25026
|
-
}
|
|
25027
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
25028
|
-
console.error(
|
|
25029
|
-
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} must be an object; trying the next candidate.`
|
|
25030
|
-
);
|
|
25031
|
-
continue;
|
|
25032
|
-
}
|
|
25033
|
-
const idToName = /* @__PURE__ */ new Map();
|
|
25034
|
-
let invalidMapping = false;
|
|
25035
|
-
for (const [rawName, rawId] of Object.entries(parsed)) {
|
|
25036
|
-
const id = typeof rawId === "number" ? rawId : Number(rawId);
|
|
25037
|
-
if (!Number.isInteger(id)) {
|
|
25038
|
-
console.error(
|
|
25039
|
-
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} has non-integer id for ${rawName}; trying the next candidate.`
|
|
25040
|
-
);
|
|
25041
|
-
invalidMapping = true;
|
|
25042
|
-
break;
|
|
25043
|
-
}
|
|
25044
|
-
idToName.set(id, extractMovieName(rawName));
|
|
25045
|
-
}
|
|
25046
|
-
if (invalidMapping) {
|
|
25047
|
-
continue;
|
|
25048
|
-
}
|
|
25049
|
-
if (idToName.size === 0) {
|
|
25050
|
-
console.error(
|
|
25051
|
-
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} is empty; trying the next candidate.`
|
|
25052
|
-
);
|
|
25053
|
-
continue;
|
|
25054
|
-
}
|
|
25055
|
-
return {
|
|
25056
|
-
idToName,
|
|
25057
|
-
movieCandidates: [...new Set(idToName.values())],
|
|
25058
|
-
aliasCounts: countMovieAliases([...new Set(idToName.values())]),
|
|
25059
|
-
sourcePath: candidate
|
|
25060
|
-
};
|
|
25061
|
-
}
|
|
25062
|
-
return null;
|
|
25063
|
-
}
|
|
25064
|
-
async function requireRecSysEntityMapping(datasetDir) {
|
|
25065
|
-
const mapping = await loadRecSysEntityMapping(datasetDir);
|
|
25066
|
-
if (!mapping) {
|
|
25067
|
-
throw new Error(
|
|
25068
|
-
`MemoryAgentBench ReDial samples require a valid ReDial entity mapping. Expected one of: ${recsysEntityMappingCandidates(datasetDir).join(", ") || "entity2id.json under the dataset directory"}.`
|
|
25069
|
-
);
|
|
25070
|
-
}
|
|
25071
|
-
return mapping;
|
|
25072
|
-
}
|
|
25073
|
-
function recsysEntityMappingCandidates(datasetDir) {
|
|
25074
|
-
if (!datasetDir) {
|
|
25075
|
-
return [];
|
|
25076
|
-
}
|
|
25077
|
-
const absoluteDatasetDir = path15.resolve(datasetDir);
|
|
25078
|
-
const roots = [
|
|
25079
|
-
absoluteDatasetDir,
|
|
25080
|
-
path15.dirname(absoluteDatasetDir)
|
|
25081
|
-
];
|
|
25082
|
-
const canonicalSuffixes = [
|
|
25083
|
-
path15.join("processed_data", "Recsys_Redial", "entity2id.json"),
|
|
25084
|
-
path15.join("Recsys_Redial", "entity2id.json")
|
|
25085
|
-
];
|
|
25086
|
-
const looseSuffixes = ["entity2id.json"];
|
|
25087
|
-
return [
|
|
25088
|
-
...roots.flatMap(
|
|
25089
|
-
(root) => canonicalSuffixes.map((suffix) => path15.join(root, suffix))
|
|
25090
|
-
),
|
|
25091
|
-
...looseSuffixes.map((suffix) => path15.join(absoluteDatasetDir, suffix))
|
|
25092
|
-
];
|
|
25093
|
-
}
|
|
25094
|
-
async function fileExists(filePath) {
|
|
25095
|
-
try {
|
|
25096
|
-
await access(filePath);
|
|
25097
|
-
return true;
|
|
25098
|
-
} catch {
|
|
25099
|
-
return false;
|
|
25100
|
-
}
|
|
25101
|
-
}
|
|
25102
|
-
function extractMovieName(rawName) {
|
|
25103
|
-
const filename = rawName.split("/").pop() ?? rawName;
|
|
25104
|
-
const decodedFilename = decodeUrlComponentSafely(filename);
|
|
25105
|
-
return decodedFilename.replace(/[_>]+/g, " ").replace(/\((\d{4})\s+film\)$/i, "($1)").replace(/\s+/g, " ").trim();
|
|
25106
|
-
}
|
|
25107
|
-
function decodeUrlComponentSafely(value) {
|
|
25108
|
-
try {
|
|
25109
|
-
return decodeURIComponent(value);
|
|
25110
|
-
} catch {
|
|
25111
|
-
return value;
|
|
25112
|
-
}
|
|
25113
|
-
}
|
|
25114
25156
|
async function loadDataset9(mode, datasetDir, limit) {
|
|
25115
25157
|
const normalizedLimit = normalizeLimit8(limit);
|
|
25116
25158
|
const ensureDatasetItems = (items) => {
|
|
@@ -25125,7 +25167,7 @@ async function loadDataset9(mode, datasetDir, limit) {
|
|
|
25125
25167
|
const datasetErrors = [];
|
|
25126
25168
|
for (const filename of DATASET_BUNDLE_CANDIDATES) {
|
|
25127
25169
|
const parsed = await tryReadDatasetFile(
|
|
25128
|
-
|
|
25170
|
+
path16.join(datasetDir, filename),
|
|
25129
25171
|
filename,
|
|
25130
25172
|
datasetErrors
|
|
25131
25173
|
);
|
|
@@ -25142,7 +25184,7 @@ async function loadDataset9(mode, datasetDir, limit) {
|
|
|
25142
25184
|
let splitData;
|
|
25143
25185
|
for (const filename of splitConfig.candidates) {
|
|
25144
25186
|
try {
|
|
25145
|
-
splitData = await readDatasetFile(
|
|
25187
|
+
splitData = await readDatasetFile(path16.join(datasetDir, filename), filename);
|
|
25146
25188
|
break;
|
|
25147
25189
|
} catch (error) {
|
|
25148
25190
|
if (!isFileNotFoundError2(error)) {
|
|
@@ -25180,7 +25222,7 @@ async function loadDataset9(mode, datasetDir, limit) {
|
|
|
25180
25222
|
return ensureDatasetItems(applyLimit8(MEMORY_AGENT_BENCH_SMOKE_FIXTURE, normalizedLimit));
|
|
25181
25223
|
}
|
|
25182
25224
|
async function readDatasetFile(filePath, filename) {
|
|
25183
|
-
const raw = await
|
|
25225
|
+
const raw = await readFile16(filePath, "utf8");
|
|
25184
25226
|
const parsed = filename.endsWith(".jsonl") ? parseJsonLines(raw, filename) : parseJsonArray(raw, filename);
|
|
25185
25227
|
return parsed.map(
|
|
25186
25228
|
(item, index) => parseMemoryAgentBenchItem(item, `${filename} item ${index + 1}`)
|
|
@@ -25791,7 +25833,7 @@ function loadCases(mode, limit) {
|
|
|
25791
25833
|
// src/benchmarks/remnic/extraction-judge-calibration/runner.ts
|
|
25792
25834
|
import { randomUUID as randomUUID11 } from "crypto";
|
|
25793
25835
|
import os3 from "os";
|
|
25794
|
-
import
|
|
25836
|
+
import path17 from "path";
|
|
25795
25837
|
import {
|
|
25796
25838
|
createVerdictCache,
|
|
25797
25839
|
judgeFactDurability,
|
|
@@ -25901,8 +25943,8 @@ var extractionJudgeCalibrationDefinition = {
|
|
|
25901
25943
|
async function runExtractionJudgeCalibrationBenchmark(options) {
|
|
25902
25944
|
const cases = loadCases2(options.mode, options.limit);
|
|
25903
25945
|
const config = parseConfig2({
|
|
25904
|
-
memoryDir:
|
|
25905
|
-
workspaceDir:
|
|
25946
|
+
memoryDir: path17.join(os3.tmpdir(), "remnic-bench-extraction-judge"),
|
|
25947
|
+
workspaceDir: path17.join(os3.tmpdir(), "remnic-bench-extraction-judge-workspace"),
|
|
25906
25948
|
openaiApiKey: "bench-test-key",
|
|
25907
25949
|
extractionJudgeEnabled: true,
|
|
25908
25950
|
extractionJudgeBatchSize: 4,
|
|
@@ -27634,7 +27676,7 @@ function constantAggregate2(value) {
|
|
|
27634
27676
|
|
|
27635
27677
|
// src/benchmarks/remnic/entity-consolidation/runner.ts
|
|
27636
27678
|
import os4 from "os";
|
|
27637
|
-
import
|
|
27679
|
+
import path18 from "path";
|
|
27638
27680
|
import { randomUUID as randomUUID14 } from "crypto";
|
|
27639
27681
|
import { mkdtemp as mkdtemp4, rm as rm5 } from "fs/promises";
|
|
27640
27682
|
import { StorageManager as StorageManager2 } from "@remnic/core";
|
|
@@ -27797,7 +27839,7 @@ function loadCases4(mode, limit) {
|
|
|
27797
27839
|
return limited;
|
|
27798
27840
|
}
|
|
27799
27841
|
async function executeCase(sample) {
|
|
27800
|
-
const tmpDir = await mkdtemp4(
|
|
27842
|
+
const tmpDir = await mkdtemp4(path18.join(os4.tmpdir(), "remnic-bench-entity-consolidation-"));
|
|
27801
27843
|
try {
|
|
27802
27844
|
const storage = new StorageManager2(tmpDir);
|
|
27803
27845
|
await storage.ensureDirectories();
|
|
@@ -27976,9 +28018,9 @@ function parseNonNegativeInt(rawValue) {
|
|
|
27976
28018
|
|
|
27977
28019
|
// src/benchmarks/remnic/page-versioning/runner.ts
|
|
27978
28020
|
import { randomUUID as randomUUID15 } from "crypto";
|
|
27979
|
-
import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as
|
|
28021
|
+
import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as readFile17, rm as rm6, writeFile as writeFile5 } from "fs/promises";
|
|
27980
28022
|
import os5 from "os";
|
|
27981
|
-
import
|
|
28023
|
+
import path19 from "path";
|
|
27982
28024
|
import {
|
|
27983
28025
|
createVersion,
|
|
27984
28026
|
diffVersions,
|
|
@@ -28142,10 +28184,10 @@ function loadCases5(mode, limit) {
|
|
|
28142
28184
|
return limited;
|
|
28143
28185
|
}
|
|
28144
28186
|
async function executeCase2(sample, dependencies) {
|
|
28145
|
-
const tmpDir = await mkdtemp5(
|
|
28187
|
+
const tmpDir = await mkdtemp5(path19.join(os5.tmpdir(), "remnic-bench-page-versioning-"));
|
|
28146
28188
|
try {
|
|
28147
|
-
const factsDir =
|
|
28148
|
-
const pagePath =
|
|
28189
|
+
const factsDir = path19.join(tmpDir, "facts");
|
|
28190
|
+
const pagePath = path19.join(factsDir, `${sample.id}.md`);
|
|
28149
28191
|
await mkdir5(factsDir, { recursive: true });
|
|
28150
28192
|
const config = versioningConfig();
|
|
28151
28193
|
switch (sample.scenario) {
|
|
@@ -28156,7 +28198,7 @@ async function executeCase2(sample, dependencies) {
|
|
|
28156
28198
|
await dependencies.createVersion(pagePath, "modified content", "write", config, void 0, void 0, tmpDir);
|
|
28157
28199
|
await dependencies.revertToVersion(pagePath, "1", config, void 0, tmpDir);
|
|
28158
28200
|
const history = await dependencies.listVersions(pagePath, config, tmpDir);
|
|
28159
|
-
const pageContent = await
|
|
28201
|
+
const pageContent = await readFile17(pagePath, "utf-8");
|
|
28160
28202
|
const observed = await dependencies.getVersion(pagePath, "3", config, tmpDir);
|
|
28161
28203
|
return {
|
|
28162
28204
|
versionIds: history.versions.map((version) => version.versionId),
|
|
@@ -28173,7 +28215,7 @@ async function executeCase2(sample, dependencies) {
|
|
|
28173
28215
|
await dependencies.createVersion(pagePath, content, "write", pruningConfig, void 0, void 0, tmpDir);
|
|
28174
28216
|
}
|
|
28175
28217
|
const history = await dependencies.listVersions(pagePath, pruningConfig, tmpDir);
|
|
28176
|
-
const pageContent = await
|
|
28218
|
+
const pageContent = await readFile17(pagePath, "utf-8");
|
|
28177
28219
|
const prunedIds = [];
|
|
28178
28220
|
for (const versionId of ["1", "2"]) {
|
|
28179
28221
|
try {
|
|
@@ -28214,7 +28256,7 @@ async function executeCase2(sample, dependencies) {
|
|
|
28214
28256
|
tmpDir
|
|
28215
28257
|
);
|
|
28216
28258
|
const history = await dependencies.listVersions(pagePath, config, tmpDir);
|
|
28217
|
-
const pageContent = await
|
|
28259
|
+
const pageContent = await readFile17(pagePath, "utf-8");
|
|
28218
28260
|
const diff = await dependencies.diffVersions(pagePath, "1", "2", config, tmpDir);
|
|
28219
28261
|
const observedLines = normalizeDiffChangedLines(diff);
|
|
28220
28262
|
return {
|
|
@@ -30501,7 +30543,7 @@ function loadCases9(mode, limit) {
|
|
|
30501
30543
|
import { randomUUID as randomUUID22 } from "crypto";
|
|
30502
30544
|
import { mkdtemp as mkdtemp6, rm as rm7 } from "fs/promises";
|
|
30503
30545
|
import os6 from "os";
|
|
30504
|
-
import
|
|
30546
|
+
import path20 from "path";
|
|
30505
30547
|
import {
|
|
30506
30548
|
StorageManager as StorageManager3,
|
|
30507
30549
|
parseConfig as parseConfig3,
|
|
@@ -30632,7 +30674,7 @@ async function runProceduralRecallBenchmark(options) {
|
|
|
30632
30674
|
}
|
|
30633
30675
|
for (const sample of e2eCases) {
|
|
30634
30676
|
const startedAt = performance.now();
|
|
30635
|
-
const dir = await mkdtemp6(
|
|
30677
|
+
const dir = await mkdtemp6(path20.join(os6.tmpdir(), "remnic-bench-procedural-recall-"));
|
|
30636
30678
|
let section = null;
|
|
30637
30679
|
try {
|
|
30638
30680
|
const storage = new StorageManager3(dir);
|
|
@@ -30653,7 +30695,7 @@ ${body}`,
|
|
|
30653
30695
|
);
|
|
30654
30696
|
const config = parseConfig3({
|
|
30655
30697
|
memoryDir: dir,
|
|
30656
|
-
workspaceDir:
|
|
30698
|
+
workspaceDir: path20.join(dir, "ws"),
|
|
30657
30699
|
openaiApiKey: "bench-key",
|
|
30658
30700
|
procedural: {
|
|
30659
30701
|
enabled: sample.proceduralEnabled !== false,
|
|
@@ -30725,7 +30767,7 @@ ${body}`,
|
|
|
30725
30767
|
import { randomUUID as randomUUID23 } from "crypto";
|
|
30726
30768
|
import { mkdtemp as mkdtemp7, writeFile as writeFile6, rm as rm8, mkdir as mkdir6, realpath as realpath5 } from "fs/promises";
|
|
30727
30769
|
import { tmpdir as tmpdir2 } from "os";
|
|
30728
|
-
import
|
|
30770
|
+
import path21 from "path";
|
|
30729
30771
|
|
|
30730
30772
|
// src/ingestion-scorer.ts
|
|
30731
30773
|
function normalize(value) {
|
|
@@ -31227,12 +31269,12 @@ async function runIngestionEntityRecallBenchmark(options) {
|
|
|
31227
31269
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31228
31270
|
}
|
|
31229
31271
|
const fixture = emailFixture.generate();
|
|
31230
|
-
const fixtureDir = await mkdtemp7(
|
|
31272
|
+
const fixtureDir = await mkdtemp7(path21.join(tmpdir2(), "bench-email-"));
|
|
31231
31273
|
try {
|
|
31232
31274
|
await options.ingestionAdapter.reset();
|
|
31233
31275
|
for (const file of fixture.files) {
|
|
31234
|
-
const filePath =
|
|
31235
|
-
await mkdir6(
|
|
31276
|
+
const filePath = path21.join(fixtureDir, file.relativePath);
|
|
31277
|
+
await mkdir6(path21.dirname(filePath), { recursive: true });
|
|
31236
31278
|
await writeFile6(filePath, file.content, "utf8");
|
|
31237
31279
|
}
|
|
31238
31280
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -31362,7 +31404,7 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
31362
31404
|
import { randomUUID as randomUUID24 } from "crypto";
|
|
31363
31405
|
import { mkdtemp as mkdtemp8, writeFile as writeFile7, rm as rm9, mkdir as mkdir7, realpath as realpath6 } from "fs/promises";
|
|
31364
31406
|
import { tmpdir as tmpdir3 } from "os";
|
|
31365
|
-
import
|
|
31407
|
+
import path22 from "path";
|
|
31366
31408
|
var ingestionSchemaCompletenessDefinition = {
|
|
31367
31409
|
id: "ingestion-schema-completeness",
|
|
31368
31410
|
title: "Ingestion: Schema Completeness",
|
|
@@ -31381,12 +31423,12 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
31381
31423
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31382
31424
|
}
|
|
31383
31425
|
const fixture = emailFixture.generate();
|
|
31384
|
-
const fixtureDir = await mkdtemp8(
|
|
31426
|
+
const fixtureDir = await mkdtemp8(path22.join(tmpdir3(), "bench-email-"));
|
|
31385
31427
|
try {
|
|
31386
31428
|
await options.ingestionAdapter.reset();
|
|
31387
31429
|
for (const file of fixture.files) {
|
|
31388
|
-
const filePath =
|
|
31389
|
-
await mkdir7(
|
|
31430
|
+
const filePath = path22.join(fixtureDir, file.relativePath);
|
|
31431
|
+
await mkdir7(path22.dirname(filePath), { recursive: true });
|
|
31390
31432
|
await writeFile7(filePath, file.content, "utf8");
|
|
31391
31433
|
}
|
|
31392
31434
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -31535,7 +31577,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
31535
31577
|
import { randomUUID as randomUUID25 } from "crypto";
|
|
31536
31578
|
import { mkdtemp as mkdtemp9, writeFile as writeFile8, rm as rm10, mkdir as mkdir8, realpath as realpath7 } from "fs/promises";
|
|
31537
31579
|
import { tmpdir as tmpdir4 } from "os";
|
|
31538
|
-
import
|
|
31580
|
+
import path23 from "path";
|
|
31539
31581
|
var ingestionBacklinkF1Definition = {
|
|
31540
31582
|
id: "ingestion-backlink-f1",
|
|
31541
31583
|
title: "Ingestion: Backlink F1",
|
|
@@ -31554,12 +31596,12 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
31554
31596
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31555
31597
|
}
|
|
31556
31598
|
const fixture = emailFixture.generate();
|
|
31557
|
-
const fixtureDir = await mkdtemp9(
|
|
31599
|
+
const fixtureDir = await mkdtemp9(path23.join(tmpdir4(), "bench-email-"));
|
|
31558
31600
|
try {
|
|
31559
31601
|
await options.ingestionAdapter.reset();
|
|
31560
31602
|
for (const file of fixture.files) {
|
|
31561
|
-
const filePath =
|
|
31562
|
-
await mkdir8(
|
|
31603
|
+
const filePath = path23.join(fixtureDir, file.relativePath);
|
|
31604
|
+
await mkdir8(path23.dirname(filePath), { recursive: true });
|
|
31563
31605
|
await writeFile8(filePath, file.content, "utf8");
|
|
31564
31606
|
}
|
|
31565
31607
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -31636,7 +31678,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
31636
31678
|
import { randomUUID as randomUUID26 } from "crypto";
|
|
31637
31679
|
import { mkdtemp as mkdtemp10, writeFile as writeFile9, rm as rm11, mkdir as mkdir9, realpath as realpath8 } from "fs/promises";
|
|
31638
31680
|
import { tmpdir as tmpdir5 } from "os";
|
|
31639
|
-
import
|
|
31681
|
+
import path24 from "path";
|
|
31640
31682
|
var INGESTION_SETUP_FRICTION_LOWER_IS_BETTER = /* @__PURE__ */ new Set(["setup_friction", "commands_count", "prompts_count", "errors_count"]);
|
|
31641
31683
|
var ingestionSetupFrictionDefinition = {
|
|
31642
31684
|
id: "ingestion-setup-friction",
|
|
@@ -31656,12 +31698,12 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
31656
31698
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31657
31699
|
}
|
|
31658
31700
|
const fixture = emailFixture.generate();
|
|
31659
|
-
const fixtureDir = await mkdtemp10(
|
|
31701
|
+
const fixtureDir = await mkdtemp10(path24.join(tmpdir5(), "bench-friction-"));
|
|
31660
31702
|
try {
|
|
31661
31703
|
await options.ingestionAdapter.reset();
|
|
31662
31704
|
for (const file of fixture.files) {
|
|
31663
|
-
const filePath =
|
|
31664
|
-
await mkdir9(
|
|
31705
|
+
const filePath = path24.join(fixtureDir, file.relativePath);
|
|
31706
|
+
await mkdir9(path24.dirname(filePath), { recursive: true });
|
|
31665
31707
|
await writeFile9(filePath, file.content, "utf8");
|
|
31666
31708
|
}
|
|
31667
31709
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -31742,7 +31784,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
31742
31784
|
import { randomUUID as randomUUID27 } from "crypto";
|
|
31743
31785
|
import { mkdtemp as mkdtemp11, writeFile as writeFile10, rm as rm12, mkdir as mkdir10, realpath as realpath9 } from "fs/promises";
|
|
31744
31786
|
import { tmpdir as tmpdir6 } from "os";
|
|
31745
|
-
import
|
|
31787
|
+
import path25 from "path";
|
|
31746
31788
|
var CITATION_SUPPORT_THRESHOLD = 0.72;
|
|
31747
31789
|
var ingestionCitationAccuracyDefinition = {
|
|
31748
31790
|
id: "ingestion-citation-accuracy",
|
|
@@ -31801,10 +31843,10 @@ function resolveCitedSources(sourceRefs, seeAlso, pageRef, sourceContentMap) {
|
|
|
31801
31843
|
return "";
|
|
31802
31844
|
}
|
|
31803
31845
|
for (const ref of normalizedRefs) {
|
|
31804
|
-
const refBase =
|
|
31846
|
+
const refBase = path25.basename(ref).toLowerCase();
|
|
31805
31847
|
let matched = false;
|
|
31806
31848
|
for (const [relativePath, content] of sourceContentMap) {
|
|
31807
|
-
if (relativePath === ref || relativePath.endsWith(ref) ||
|
|
31849
|
+
if (relativePath === ref || relativePath.endsWith(ref) || path25.basename(relativePath).toLowerCase() === refBase) {
|
|
31808
31850
|
resolved.push(content);
|
|
31809
31851
|
matched = true;
|
|
31810
31852
|
break;
|
|
@@ -31820,9 +31862,9 @@ function resolveCitedSources(sourceRefs, seeAlso, pageRef, sourceContentMap) {
|
|
|
31820
31862
|
if (normalizedRefs.length > 0) {
|
|
31821
31863
|
return "";
|
|
31822
31864
|
}
|
|
31823
|
-
const pageBase =
|
|
31865
|
+
const pageBase = path25.basename(pageRef).toLowerCase();
|
|
31824
31866
|
for (const [relativePath, content] of sourceContentMap) {
|
|
31825
|
-
if (
|
|
31867
|
+
if (path25.basename(relativePath).toLowerCase() === pageBase) {
|
|
31826
31868
|
return content;
|
|
31827
31869
|
}
|
|
31828
31870
|
}
|
|
@@ -31833,12 +31875,12 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
31833
31875
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
31834
31876
|
}
|
|
31835
31877
|
const fixture = emailFixture.generate();
|
|
31836
|
-
const fixtureDir = await mkdtemp11(
|
|
31878
|
+
const fixtureDir = await mkdtemp11(path25.join(tmpdir6(), "bench-citation-"));
|
|
31837
31879
|
try {
|
|
31838
31880
|
await options.ingestionAdapter.reset();
|
|
31839
31881
|
for (const file of fixture.files) {
|
|
31840
|
-
const filePath =
|
|
31841
|
-
await mkdir10(
|
|
31882
|
+
const filePath = path25.join(fixtureDir, file.relativePath);
|
|
31883
|
+
await mkdir10(path25.dirname(filePath), { recursive: true });
|
|
31842
31884
|
await writeFile10(filePath, file.content, "utf8");
|
|
31843
31885
|
}
|
|
31844
31886
|
const benchmarkStart = performance.now();
|
|
@@ -32224,7 +32266,7 @@ var ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS = ASSISTANT_MORNING_BRIEF_SCENARIOS.
|
|
|
32224
32266
|
|
|
32225
32267
|
// src/benchmarks/remnic/_assistant-common/runner.ts
|
|
32226
32268
|
import { randomUUID as randomUUID28 } from "crypto";
|
|
32227
|
-
import
|
|
32269
|
+
import path27 from "path";
|
|
32228
32270
|
|
|
32229
32271
|
// src/run-seeds.ts
|
|
32230
32272
|
function buildBenchmarkRunSeeds(runCount, baseSeed) {
|
|
@@ -32316,7 +32358,7 @@ function pairedDeltaConfidenceInterval(candidateValues, baselineValues, options
|
|
|
32316
32358
|
// src/judges/sealed-rubric.ts
|
|
32317
32359
|
import { createHash as createHash10 } from "crypto";
|
|
32318
32360
|
import { appendFileSync, mkdirSync } from "fs";
|
|
32319
|
-
import
|
|
32361
|
+
import path26 from "path";
|
|
32320
32362
|
|
|
32321
32363
|
// src/judges/sealed-prompts/assistant-rubric-v1.ts
|
|
32322
32364
|
var ASSISTANT_RUBRIC_V1 = `# Assistant rubric v1 (sealed)
|
|
@@ -32596,7 +32638,7 @@ function createSpotCheckFileLogger(options) {
|
|
|
32596
32638
|
return { log() {
|
|
32597
32639
|
} };
|
|
32598
32640
|
}
|
|
32599
|
-
const logPath =
|
|
32641
|
+
const logPath = path26.join(directory, `${runId}.jsonl`);
|
|
32600
32642
|
let written = 0;
|
|
32601
32643
|
let warnedOnWriteFailure = false;
|
|
32602
32644
|
const cap = typeof sampleSize === "number" && sampleSize > 0 ? sampleSize : 5;
|
|
@@ -32685,7 +32727,7 @@ async function runAssistantBenchmark(definition, scenarios, resolved, runnerOpti
|
|
|
32685
32727
|
const runId = buildRunId(definition.id);
|
|
32686
32728
|
const spotCheckLogger = createSpotCheckFileLogger({
|
|
32687
32729
|
runId,
|
|
32688
|
-
directory: runnerOptions.spotCheckDir ??
|
|
32730
|
+
directory: runnerOptions.spotCheckDir ?? path27.join(process.cwd(), "benchmarks", "results", "spot-checks"),
|
|
32689
32731
|
sampleRate: 0.35,
|
|
32690
32732
|
sampleSize: 5
|
|
32691
32733
|
});
|
|
@@ -33334,7 +33376,7 @@ async function runAssistantSynthesisBenchmark(options) {
|
|
|
33334
33376
|
|
|
33335
33377
|
// src/benchmarks/remnic/buffer-surprise-trigger/runner.ts
|
|
33336
33378
|
import { randomUUID as randomUUID29 } from "crypto";
|
|
33337
|
-
import
|
|
33379
|
+
import path28 from "path";
|
|
33338
33380
|
import os7 from "os";
|
|
33339
33381
|
import { mkdir as mkdir11, rm as rm13 } from "fs/promises";
|
|
33340
33382
|
import {
|
|
@@ -33565,7 +33607,7 @@ function hasExplicitTopicPivotCue(text) {
|
|
|
33565
33607
|
}
|
|
33566
33608
|
async function runBufferSurpriseTriggerBenchmark(options) {
|
|
33567
33609
|
const cases = loadCases10(options.mode, options.limit);
|
|
33568
|
-
const tmpRoot =
|
|
33610
|
+
const tmpRoot = path28.join(
|
|
33569
33611
|
os7.tmpdir(),
|
|
33570
33612
|
`remnic-bench-buffer-surprise-${randomUUID29()}`
|
|
33571
33613
|
);
|
|
@@ -33634,11 +33676,11 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
33634
33676
|
};
|
|
33635
33677
|
}
|
|
33636
33678
|
async function runSingleCase(caseDef, options) {
|
|
33637
|
-
const memoryDir =
|
|
33679
|
+
const memoryDir = path28.join(
|
|
33638
33680
|
options.tmpRoot,
|
|
33639
33681
|
`${caseDef.id}-${options.label}`
|
|
33640
33682
|
);
|
|
33641
|
-
const workspaceDir =
|
|
33683
|
+
const workspaceDir = path28.join(memoryDir, "workspace");
|
|
33642
33684
|
await mkdir11(workspaceDir, { recursive: true });
|
|
33643
33685
|
const config = parseConfig4({
|
|
33644
33686
|
memoryDir,
|
|
@@ -35904,7 +35946,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
35904
35946
|
// src/benchmarks/remnic/bounded-memory-contracts/runner.ts
|
|
35905
35947
|
import { randomUUID as randomUUID33 } from "crypto";
|
|
35906
35948
|
import { mkdir as mkdir12, writeFile as writeFile11 } from "fs/promises";
|
|
35907
|
-
import
|
|
35949
|
+
import path29 from "path";
|
|
35908
35950
|
|
|
35909
35951
|
// src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
|
|
35910
35952
|
import { createHash as createHash12 } from "crypto";
|
|
@@ -37136,23 +37178,23 @@ async function runBoundedMemoryContractsBenchmark(options) {
|
|
|
37136
37178
|
};
|
|
37137
37179
|
}
|
|
37138
37180
|
async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks) {
|
|
37139
|
-
const root =
|
|
37140
|
-
await mkdir12(
|
|
37141
|
-
await mkdir12(
|
|
37142
|
-
await mkdir12(
|
|
37143
|
-
await mkdir12(
|
|
37181
|
+
const root = path29.resolve(outputDir);
|
|
37182
|
+
await mkdir12(path29.join(root, "conditions"), { recursive: true });
|
|
37183
|
+
await mkdir12(path29.join(root, "prompts"), { recursive: true });
|
|
37184
|
+
await mkdir12(path29.join(root, "retrieval"), { recursive: true });
|
|
37185
|
+
await mkdir12(path29.join(root, "scores"), { recursive: true });
|
|
37144
37186
|
const csvRows = [
|
|
37145
37187
|
"task_id,condition,family,scope,task_success,should_ask_accuracy,relevant_memory_recall,stale_memory_harm_rate,wrong_scope_retrieval_rate,supersession_respected_rate,citation_coverage,memory_tokens_injected,retrieved_item_count,compression_ratio_vs_raw_transcript"
|
|
37146
37188
|
];
|
|
37147
37189
|
for (const condition of BOUNDED_MEMORY_CONDITIONS) {
|
|
37148
37190
|
const results = byCondition.get(condition);
|
|
37149
|
-
const condDir =
|
|
37191
|
+
const condDir = path29.join(root, "conditions", condition);
|
|
37150
37192
|
await mkdir12(condDir, { recursive: true });
|
|
37151
37193
|
for (const { task, pack, decision } of results) {
|
|
37152
37194
|
const scores = scoreTaskPair(task, pack, decision);
|
|
37153
37195
|
const promptMd = renderPromptPack(task, condition, pack);
|
|
37154
|
-
const promptPath =
|
|
37155
|
-
await mkdir12(
|
|
37196
|
+
const promptPath = path29.join(root, "prompts", `${task.id}.${condition}.md`);
|
|
37197
|
+
await mkdir12(path29.dirname(promptPath), { recursive: true });
|
|
37156
37198
|
await writeFile11(promptPath, promptMd, "utf8");
|
|
37157
37199
|
const retrievalJson = `${JSON.stringify(
|
|
37158
37200
|
{
|
|
@@ -37176,7 +37218,7 @@ async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks
|
|
|
37176
37218
|
2
|
|
37177
37219
|
)}
|
|
37178
37220
|
`;
|
|
37179
|
-
const retrievalPath =
|
|
37221
|
+
const retrievalPath = path29.join(root, "retrieval", `${task.id}.${condition}.json`);
|
|
37180
37222
|
await writeFile11(retrievalPath, retrievalJson, "utf8");
|
|
37181
37223
|
csvRows.push(
|
|
37182
37224
|
[
|
|
@@ -37198,23 +37240,23 @@ async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks
|
|
|
37198
37240
|
);
|
|
37199
37241
|
}
|
|
37200
37242
|
await writeFile11(
|
|
37201
|
-
|
|
37243
|
+
path29.join(condDir, "summary.json"),
|
|
37202
37244
|
`${JSON.stringify(conditionAggregates[condition], null, 2)}
|
|
37203
37245
|
`,
|
|
37204
37246
|
"utf8"
|
|
37205
37247
|
);
|
|
37206
37248
|
}
|
|
37207
|
-
await writeFile11(
|
|
37249
|
+
await writeFile11(path29.join(root, "scores", "per-task.csv"), `${csvRows.join("\n")}
|
|
37208
37250
|
`, "utf8");
|
|
37209
37251
|
await writeFile11(
|
|
37210
|
-
|
|
37252
|
+
path29.join(root, "scores", "aggregate.json"),
|
|
37211
37253
|
`${JSON.stringify(conditionAggregates, null, 2)}
|
|
37212
37254
|
`,
|
|
37213
37255
|
"utf8"
|
|
37214
37256
|
);
|
|
37215
37257
|
const report = renderReportMarkdown(tasks, conditionAggregates);
|
|
37216
|
-
await writeFile11(
|
|
37217
|
-
return
|
|
37258
|
+
await writeFile11(path29.join(root, "report.md"), report, "utf8");
|
|
37259
|
+
return path29.join(root, "report.md");
|
|
37218
37260
|
}
|
|
37219
37261
|
function renderPromptPack(task, condition, pack) {
|
|
37220
37262
|
const lines = [];
|
|
@@ -37264,14 +37306,14 @@ import { createHash as createHash16, randomUUID as randomUUID34 } from "crypto";
|
|
|
37264
37306
|
|
|
37265
37307
|
// src/benchmarks/remnic/staged-memory/fixture.ts
|
|
37266
37308
|
import { createHash as createHash15 } from "crypto";
|
|
37267
|
-
import { lstat as lstat5, mkdir as mkdir14, readFile as
|
|
37268
|
-
import
|
|
37309
|
+
import { lstat as lstat5, mkdir as mkdir14, readFile as readFile19, rename as rename4, rm as rm15, writeFile as writeFile13 } from "fs/promises";
|
|
37310
|
+
import path32 from "path";
|
|
37269
37311
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
37270
37312
|
|
|
37271
37313
|
// src/generators/drift-gen/index.ts
|
|
37272
37314
|
import { createHash as createHash14 } from "crypto";
|
|
37273
37315
|
import { mkdir as mkdir13, readdir as readdir7, rename as rename3, rm as rm14, writeFile as writeFile12 } from "fs/promises";
|
|
37274
|
-
import
|
|
37316
|
+
import path31 from "path";
|
|
37275
37317
|
|
|
37276
37318
|
// src/generators/drift-gen/names.ts
|
|
37277
37319
|
var PERSON_NAMES = Object.freeze([
|
|
@@ -37997,8 +38039,8 @@ function renderUserSessions(rng, user, epochs) {
|
|
|
37997
38039
|
|
|
37998
38040
|
// src/generators/drift-gen/validate.ts
|
|
37999
38041
|
import { createHash as createHash13 } from "crypto";
|
|
38000
|
-
import { lstat as lstat4, readFile as
|
|
38001
|
-
import
|
|
38042
|
+
import { lstat as lstat4, readFile as readFile18, readdir as readdir6 } from "fs/promises";
|
|
38043
|
+
import path30 from "path";
|
|
38002
38044
|
var FACT_COUNT_TOLERANCE = 0.1;
|
|
38003
38045
|
var RATIO_TOLERANCE = 0.05;
|
|
38004
38046
|
var MAX_QUESTION_ANSWER_LEAKAGE = 0.6;
|
|
@@ -38086,7 +38128,7 @@ async function readJsonl(filePath, errors, isShape) {
|
|
|
38086
38128
|
errors.push(`symlinked corpus file rejected: ${filePath}`);
|
|
38087
38129
|
return [];
|
|
38088
38130
|
}
|
|
38089
|
-
raw = await
|
|
38131
|
+
raw = await readFile18(filePath, "utf8");
|
|
38090
38132
|
} catch {
|
|
38091
38133
|
errors.push(`missing file: ${filePath}`);
|
|
38092
38134
|
return [];
|
|
@@ -38130,31 +38172,31 @@ async function isNonSymlinkDirectory(dirPath, errors) {
|
|
|
38130
38172
|
}
|
|
38131
38173
|
async function hasNoSymlinkComponents(rootDir, targetPath, errors, description) {
|
|
38132
38174
|
let current = rootDir;
|
|
38133
|
-
for (const part of
|
|
38175
|
+
for (const part of path30.relative(rootDir, targetPath).split(path30.sep)) {
|
|
38134
38176
|
if (part.length === 0 || part === ".") continue;
|
|
38135
|
-
current =
|
|
38177
|
+
current = path30.join(current, part);
|
|
38136
38178
|
try {
|
|
38137
38179
|
if ((await lstat4(current)).isSymbolicLink()) {
|
|
38138
|
-
errors.push(`${description} contains a symlinked path component: ${
|
|
38180
|
+
errors.push(`${description} contains a symlinked path component: ${path30.relative(rootDir, current)}`);
|
|
38139
38181
|
return false;
|
|
38140
38182
|
}
|
|
38141
38183
|
} catch {
|
|
38142
|
-
errors.push(`${description} is missing: ${
|
|
38184
|
+
errors.push(`${description} is missing: ${path30.relative(rootDir, current)}`);
|
|
38143
38185
|
return false;
|
|
38144
38186
|
}
|
|
38145
38187
|
}
|
|
38146
38188
|
return true;
|
|
38147
38189
|
}
|
|
38148
38190
|
function corpusRelativePath(corpusDir, targetPath) {
|
|
38149
|
-
return
|
|
38191
|
+
return path30.relative(corpusDir, targetPath).split(path30.sep).join("/");
|
|
38150
38192
|
}
|
|
38151
38193
|
async function loadSeedDir(corpusDir, seed, errors) {
|
|
38152
|
-
const seedDir =
|
|
38194
|
+
const seedDir = path30.join(corpusDir, String(seed));
|
|
38153
38195
|
const empty = { seed, facts: [], probes: [], sessions: [], consumedFiles: [] };
|
|
38154
38196
|
if (!await isNonSymlinkDirectory(seedDir, errors)) return empty;
|
|
38155
|
-
const goldDir =
|
|
38156
|
-
const factsPath =
|
|
38157
|
-
const probesPath =
|
|
38197
|
+
const goldDir = path30.join(seedDir, "gold");
|
|
38198
|
+
const factsPath = path30.join(goldDir, "facts.jsonl");
|
|
38199
|
+
const probesPath = path30.join(goldDir, "probes.jsonl");
|
|
38158
38200
|
let facts = [];
|
|
38159
38201
|
let probes = [];
|
|
38160
38202
|
const consumedFiles = [];
|
|
@@ -38167,7 +38209,7 @@ async function loadSeedDir(corpusDir, seed, errors) {
|
|
|
38167
38209
|
probes = await readJsonl(probesPath, errors, isGoldProbeShape);
|
|
38168
38210
|
}
|
|
38169
38211
|
const sessions = [];
|
|
38170
|
-
const usersDir =
|
|
38212
|
+
const usersDir = path30.join(seedDir, "users");
|
|
38171
38213
|
if (!await isNonSymlinkDirectory(usersDir, errors)) {
|
|
38172
38214
|
return { seed, facts, probes, sessions, consumedFiles };
|
|
38173
38215
|
}
|
|
@@ -38175,7 +38217,7 @@ async function loadSeedDir(corpusDir, seed, errors) {
|
|
|
38175
38217
|
try {
|
|
38176
38218
|
const entries = await readdir6(usersDir, { withFileTypes: true });
|
|
38177
38219
|
for (const entry of entries) {
|
|
38178
|
-
const userDir =
|
|
38220
|
+
const userDir = path30.join(usersDir, entry.name);
|
|
38179
38221
|
if (entry.isSymbolicLink()) {
|
|
38180
38222
|
errors.push(`symlinked corpus entry rejected: ${userDir}`);
|
|
38181
38223
|
continue;
|
|
@@ -38187,8 +38229,8 @@ async function loadSeedDir(corpusDir, seed, errors) {
|
|
|
38187
38229
|
return { seed, facts, probes, sessions, consumedFiles };
|
|
38188
38230
|
}
|
|
38189
38231
|
for (const userId of userIds.sort()) {
|
|
38190
|
-
const userDir =
|
|
38191
|
-
const sessionsPath =
|
|
38232
|
+
const userDir = path30.join(usersDir, userId);
|
|
38233
|
+
const sessionsPath = path30.join(userDir, "sessions.jsonl");
|
|
38192
38234
|
consumedFiles.push(corpusRelativePath(corpusDir, sessionsPath));
|
|
38193
38235
|
for (const session of await readJsonl(sessionsPath, errors, isDriftSessionShape)) {
|
|
38194
38236
|
if (session.userId !== userId) {
|
|
@@ -38499,10 +38541,10 @@ function isManifestShape(value) {
|
|
|
38499
38541
|
);
|
|
38500
38542
|
}
|
|
38501
38543
|
async function checkFileHashes(corpusDir, manifest, errors) {
|
|
38502
|
-
const resolvedRoot =
|
|
38544
|
+
const resolvedRoot = path30.resolve(corpusDir);
|
|
38503
38545
|
for (const [relPath, expected] of Object.entries(manifest.files)) {
|
|
38504
|
-
const absPath =
|
|
38505
|
-
if (absPath !== resolvedRoot && !absPath.startsWith(resolvedRoot +
|
|
38546
|
+
const absPath = path30.resolve(corpusDir, relPath);
|
|
38547
|
+
if (absPath !== resolvedRoot && !absPath.startsWith(resolvedRoot + path30.sep)) {
|
|
38506
38548
|
errors.push(`manifest lists a path outside the corpus root: ${relPath}`);
|
|
38507
38549
|
continue;
|
|
38508
38550
|
}
|
|
@@ -38511,7 +38553,7 @@ async function checkFileHashes(corpusDir, manifest, errors) {
|
|
|
38511
38553
|
}
|
|
38512
38554
|
let data;
|
|
38513
38555
|
try {
|
|
38514
|
-
data = await
|
|
38556
|
+
data = await readFile18(absPath);
|
|
38515
38557
|
} catch {
|
|
38516
38558
|
errors.push(`manifest lists missing file: ${relPath}`);
|
|
38517
38559
|
continue;
|
|
@@ -38560,13 +38602,13 @@ async function validateDriftCorpus(corpusDir) {
|
|
|
38560
38602
|
} catch {
|
|
38561
38603
|
return { ok: false, errors: [`corpus directory not found: ${corpusDir}`], warnings, stats: emptyStats };
|
|
38562
38604
|
}
|
|
38563
|
-
const manifestPath =
|
|
38605
|
+
const manifestPath = path30.join(corpusDir, "dataset.manifest.json");
|
|
38564
38606
|
if (!await hasNoSymlinkComponents(corpusDir, manifestPath, errors, "dataset manifest")) {
|
|
38565
38607
|
return { ok: false, errors, warnings, stats: emptyStats };
|
|
38566
38608
|
}
|
|
38567
38609
|
let manifestRaw;
|
|
38568
38610
|
try {
|
|
38569
|
-
manifestRaw = JSON.parse(await
|
|
38611
|
+
manifestRaw = JSON.parse(await readFile18(manifestPath, "utf8"));
|
|
38570
38612
|
} catch {
|
|
38571
38613
|
return {
|
|
38572
38614
|
ok: false,
|
|
@@ -38683,11 +38725,11 @@ async function generateDriftCorpus(options) {
|
|
|
38683
38725
|
const seedDir = String(options.seed);
|
|
38684
38726
|
const written = /* @__PURE__ */ new Map();
|
|
38685
38727
|
written.set(
|
|
38686
|
-
|
|
38728
|
+
path31.posix.join(seedDir, "gold", "facts.jsonl"),
|
|
38687
38729
|
toJsonl(corpus.facts)
|
|
38688
38730
|
);
|
|
38689
38731
|
written.set(
|
|
38690
|
-
|
|
38732
|
+
path31.posix.join(seedDir, "gold", "probes.jsonl"),
|
|
38691
38733
|
toJsonl(corpus.probes)
|
|
38692
38734
|
);
|
|
38693
38735
|
const sessionsByUser = /* @__PURE__ */ new Map();
|
|
@@ -38698,7 +38740,7 @@ async function generateDriftCorpus(options) {
|
|
|
38698
38740
|
}
|
|
38699
38741
|
for (const [userId, sessions] of [...sessionsByUser.entries()].sort()) {
|
|
38700
38742
|
written.set(
|
|
38701
|
-
|
|
38743
|
+
path31.posix.join(seedDir, "users", userId, "sessions.jsonl"),
|
|
38702
38744
|
toJsonl(sessions)
|
|
38703
38745
|
);
|
|
38704
38746
|
}
|
|
@@ -38727,23 +38769,23 @@ async function generateDriftCorpus(options) {
|
|
|
38727
38769
|
licenses: [{ source: "synthetic", license: "MIT (repo)" }],
|
|
38728
38770
|
...options.audit ? { audit: options.audit } : {}
|
|
38729
38771
|
};
|
|
38730
|
-
const stagingDir =
|
|
38772
|
+
const stagingDir = path31.join(options.outDir, `.staging-${options.seed}`);
|
|
38731
38773
|
await rm14(stagingDir, { recursive: true, force: true });
|
|
38732
38774
|
for (const [relPath, content] of written) {
|
|
38733
|
-
const absPath =
|
|
38734
|
-
await mkdir13(
|
|
38775
|
+
const absPath = path31.join(stagingDir, path31.relative(seedDir, relPath));
|
|
38776
|
+
await mkdir13(path31.dirname(absPath), { recursive: true });
|
|
38735
38777
|
await writeFile12(absPath, content, "utf8");
|
|
38736
38778
|
}
|
|
38737
|
-
const finalSeedDir =
|
|
38738
|
-
const backupDir =
|
|
38779
|
+
const finalSeedDir = path31.join(options.outDir, seedDir);
|
|
38780
|
+
const backupDir = path31.join(options.outDir, `.backup-${options.seed}-${process.pid}`);
|
|
38739
38781
|
const staleSeedDirs = (await readdir7(options.outDir, { withFileTypes: true })).filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name) && entry.name !== seedDir).map((entry) => ({
|
|
38740
|
-
source:
|
|
38741
|
-
backup:
|
|
38782
|
+
source: path31.join(options.outDir, entry.name),
|
|
38783
|
+
backup: path31.join(options.outDir, `.backup-stale-${entry.name}-${process.pid}`)
|
|
38742
38784
|
}));
|
|
38743
38785
|
const quarantinedStaleDirs = [];
|
|
38744
|
-
const manifestPath =
|
|
38745
|
-
const manifestStaging =
|
|
38746
|
-
const manifestBackup =
|
|
38786
|
+
const manifestPath = path31.join(options.outDir, "dataset.manifest.json");
|
|
38787
|
+
const manifestStaging = path31.join(options.outDir, ".staging-manifest.json");
|
|
38788
|
+
const manifestBackup = path31.join(options.outDir, `.backup-manifest-${process.pid}.json`);
|
|
38747
38789
|
let hadPrevious = false;
|
|
38748
38790
|
let replacementInstalled = false;
|
|
38749
38791
|
let hadPreviousManifest = false;
|
|
@@ -38986,7 +39028,7 @@ var MAX_QUESTION_ANSWER_LEAKAGE2 = 0.6;
|
|
|
38986
39028
|
var MANIFEST_FILE = "manifest.json";
|
|
38987
39029
|
var CASES_FILE = "cases.jsonl";
|
|
38988
39030
|
function canonicalDriftDir() {
|
|
38989
|
-
return
|
|
39031
|
+
return path32.resolve(path32.dirname(fileURLToPath2(import.meta.url)), "../../../fixtures/drift-gen-core");
|
|
38990
39032
|
}
|
|
38991
39033
|
var DISTRACTOR_TEMPLATES = Object.freeze([
|
|
38992
39034
|
{ templateId: "dist-employer-1", text: "Marlow Petrov works at Quill Optical." },
|
|
@@ -39029,14 +39071,14 @@ function factUserPrefixOk(factId, userId) {
|
|
|
39029
39071
|
return factId.startsWith(`gf-${userId}-`);
|
|
39030
39072
|
}
|
|
39031
39073
|
async function readJsonlFile(filePath) {
|
|
39032
|
-
const raw = await
|
|
39074
|
+
const raw = await readFile19(filePath, "utf8");
|
|
39033
39075
|
const lines = raw.split("\n").filter((line) => line.trim().length > 0);
|
|
39034
39076
|
return lines.map((line) => JSON.parse(line));
|
|
39035
39077
|
}
|
|
39036
39078
|
async function loadVerifiedDriftCorpus(driftDir, seed) {
|
|
39037
|
-
const label =
|
|
39038
|
-
const manifestPath =
|
|
39039
|
-
const manifest = JSON.parse(await
|
|
39079
|
+
const label = path32.basename(driftDir);
|
|
39080
|
+
const manifestPath = path32.join(driftDir, "dataset.manifest.json");
|
|
39081
|
+
const manifest = JSON.parse(await readFile19(manifestPath, "utf8"));
|
|
39040
39082
|
if (typeof manifest.name !== "string" || manifest.name.length === 0) {
|
|
39041
39083
|
throw new Error(`drift corpus ${label} has no manifest name`);
|
|
39042
39084
|
}
|
|
@@ -39045,17 +39087,17 @@ async function loadVerifiedDriftCorpus(driftDir, seed) {
|
|
|
39045
39087
|
}
|
|
39046
39088
|
const files = manifest.files ?? {};
|
|
39047
39089
|
const seedDir = String(seed);
|
|
39048
|
-
const wanted = [
|
|
39090
|
+
const wanted = [path32.posix.join(seedDir, "gold", "facts.jsonl"), path32.posix.join(seedDir, "gold", "probes.jsonl")];
|
|
39049
39091
|
for (const relPath of wanted) {
|
|
39050
39092
|
const expected = files[relPath];
|
|
39051
39093
|
if (typeof expected !== "string") {
|
|
39052
39094
|
throw new Error(`drift manifest is missing a hash for ${relPath}`);
|
|
39053
39095
|
}
|
|
39054
|
-
if (sha2562(await
|
|
39096
|
+
if (sha2562(await readFile19(path32.join(driftDir, relPath), "utf8")) !== expected) {
|
|
39055
39097
|
throw new Error(`drift corpus file ${relPath} fails its manifest hash`);
|
|
39056
39098
|
}
|
|
39057
39099
|
}
|
|
39058
|
-
const factRows = await readJsonlFile(
|
|
39100
|
+
const factRows = await readJsonlFile(path32.join(driftDir, wanted[0]));
|
|
39059
39101
|
const facts = factRows.map((row) => {
|
|
39060
39102
|
const fact3 = row;
|
|
39061
39103
|
if (typeof fact3.id !== "string" || typeof fact3.userId !== "string" || typeof fact3.statement !== "string" || typeof fact3.subject !== "string" || typeof fact3.attribute !== "string" || typeof fact3.value !== "string" || typeof fact3.introducedEpoch !== "number") {
|
|
@@ -39067,15 +39109,15 @@ async function loadVerifiedDriftCorpus(driftDir, seed) {
|
|
|
39067
39109
|
const sessions = [];
|
|
39068
39110
|
for (const relPath of sessionFiles) {
|
|
39069
39111
|
const expected = files[relPath];
|
|
39070
|
-
if (sha2562(await
|
|
39112
|
+
if (sha2562(await readFile19(path32.join(driftDir, relPath), "utf8")) !== expected) {
|
|
39071
39113
|
throw new Error(`drift corpus file ${relPath} fails its manifest hash`);
|
|
39072
39114
|
}
|
|
39073
|
-
sessions.push(...await readJsonlFile(
|
|
39115
|
+
sessions.push(...await readJsonlFile(path32.join(driftDir, relPath)));
|
|
39074
39116
|
}
|
|
39075
39117
|
sessions.sort((a, b) => a.epoch - b.epoch || compareStrings(a.sessionId, b.sessionId));
|
|
39076
39118
|
return {
|
|
39077
39119
|
manifestName: manifest.name,
|
|
39078
|
-
manifestSha256: sha2562(await
|
|
39120
|
+
manifestSha256: sha2562(await readFile19(manifestPath, "utf8")),
|
|
39079
39121
|
facts,
|
|
39080
39122
|
sessions,
|
|
39081
39123
|
seed
|
|
@@ -39266,7 +39308,7 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39266
39308
|
if (!dirStat?.isDirectory()) {
|
|
39267
39309
|
return {
|
|
39268
39310
|
ok: false,
|
|
39269
|
-
errors: [`fixture directory not found: ${
|
|
39311
|
+
errors: [`fixture directory not found: ${path32.basename(fixtureDir)}`],
|
|
39270
39312
|
warnings,
|
|
39271
39313
|
stats: { users: 0, cases: 0, distractorsPerCase: 0, transitions: 0 }
|
|
39272
39314
|
};
|
|
@@ -39274,8 +39316,8 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39274
39316
|
if (dirStat.isSymbolicLink()) {
|
|
39275
39317
|
errors.push("fixture directory must not be a symlink");
|
|
39276
39318
|
}
|
|
39277
|
-
const manifestPath =
|
|
39278
|
-
const rawManifest = await
|
|
39319
|
+
const manifestPath = path32.join(fixtureDir, MANIFEST_FILE);
|
|
39320
|
+
const rawManifest = await readFile19(manifestPath, "utf8").catch(() => void 0);
|
|
39279
39321
|
if (rawManifest === void 0) {
|
|
39280
39322
|
return {
|
|
39281
39323
|
ok: false,
|
|
@@ -39327,7 +39369,7 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39327
39369
|
errors.push(`manifest.files must hash ${CASES_FILE}`);
|
|
39328
39370
|
}
|
|
39329
39371
|
for (const [relPath, expectedHash] of Object.entries(manifest.files)) {
|
|
39330
|
-
const filePath =
|
|
39372
|
+
const filePath = path32.join(fixtureDir, relPath);
|
|
39331
39373
|
const stat6 = await lstat5(filePath).catch(() => void 0);
|
|
39332
39374
|
if (!stat6) {
|
|
39333
39375
|
errors.push(`fixture file listed in manifest is missing: ${relPath}`);
|
|
@@ -39337,11 +39379,11 @@ async function validateStagedMemoryFixture(fixtureDir) {
|
|
|
39337
39379
|
errors.push(`fixture file must not be a symlink: ${relPath}`);
|
|
39338
39380
|
continue;
|
|
39339
39381
|
}
|
|
39340
|
-
if (sha2562(await
|
|
39382
|
+
if (sha2562(await readFile19(filePath, "utf8")) !== expectedHash) {
|
|
39341
39383
|
errors.push(`fixture file fails its manifest hash: ${relPath}`);
|
|
39342
39384
|
}
|
|
39343
39385
|
}
|
|
39344
|
-
const rawCases = await
|
|
39386
|
+
const rawCases = await readFile19(path32.join(fixtureDir, CASES_FILE), "utf8").catch(() => void 0);
|
|
39345
39387
|
if (rawCases === void 0) {
|
|
39346
39388
|
return {
|
|
39347
39389
|
ok: false,
|
|
@@ -39488,9 +39530,9 @@ async function loadStagedMemoryFixture(fixtureDir) {
|
|
|
39488
39530
|
${report.errors.map((e) => ` - ${e}`).join("\n")}`);
|
|
39489
39531
|
}
|
|
39490
39532
|
const rawManifest = JSON.parse(
|
|
39491
|
-
await
|
|
39533
|
+
await readFile19(path32.join(fixtureDir, MANIFEST_FILE), "utf8")
|
|
39492
39534
|
);
|
|
39493
|
-
const rawCases = await
|
|
39535
|
+
const rawCases = await readFile19(path32.join(fixtureDir, CASES_FILE), "utf8");
|
|
39494
39536
|
const cases = rawCases.split("\n").filter((line) => line.trim()).map((line) => JSON.parse(line));
|
|
39495
39537
|
return {
|
|
39496
39538
|
manifest: rawManifest,
|
|
@@ -40429,8 +40471,8 @@ function finalizeBenchmarkResultConfig(result, options) {
|
|
|
40429
40471
|
}
|
|
40430
40472
|
|
|
40431
40473
|
// src/benchmark.ts
|
|
40432
|
-
var DEFAULT_BASELINE_PATH =
|
|
40433
|
-
var DEFAULT_REPORT_PATH =
|
|
40474
|
+
var DEFAULT_BASELINE_PATH = path33.join(process.cwd(), "benchmarks", "baseline.json");
|
|
40475
|
+
var DEFAULT_REPORT_PATH = path33.join(process.cwd(), "benchmarks", "report.json");
|
|
40434
40476
|
var BASELINE_VERSION = 1;
|
|
40435
40477
|
var DEFAULT_TOLERANCE = 10;
|
|
40436
40478
|
var DEFAULT_FULL_RUN_COUNT = 5;
|
|
@@ -40508,7 +40550,7 @@ async function runBenchmark(benchmarkId, options) {
|
|
|
40508
40550
|
if (!willWrapPrimary && !willWrapCross) {
|
|
40509
40551
|
return void 0;
|
|
40510
40552
|
}
|
|
40511
|
-
const cacheDir = options.judgeCacheDir ?
|
|
40553
|
+
const cacheDir = options.judgeCacheDir ? path33.resolve(expandTildePath3(options.judgeCacheDir)) : options.outputDir ? path33.join(path33.resolve(expandTildePath3(options.outputDir)), "judge-cache") : void 0;
|
|
40512
40554
|
if (cacheDir === void 0) {
|
|
40513
40555
|
return void 0;
|
|
40514
40556
|
}
|
|
@@ -40707,7 +40749,7 @@ function loadBaseline(baselinePath) {
|
|
|
40707
40749
|
return raw;
|
|
40708
40750
|
}
|
|
40709
40751
|
function saveBaseline(baselinePath, baseline) {
|
|
40710
|
-
fs.mkdirSync(
|
|
40752
|
+
fs.mkdirSync(path33.dirname(baselinePath), { recursive: true });
|
|
40711
40753
|
fs.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}
|
|
40712
40754
|
`);
|
|
40713
40755
|
}
|
|
@@ -40937,7 +40979,7 @@ function generateReport(results, reportPath) {
|
|
|
40937
40979
|
totalDurationMs: results.reduce((sum, result) => sum + result.totalDurationMs, 0)
|
|
40938
40980
|
};
|
|
40939
40981
|
if (reportPath) {
|
|
40940
|
-
fs.mkdirSync(
|
|
40982
|
+
fs.mkdirSync(path33.dirname(reportPath), { recursive: true });
|
|
40941
40983
|
fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}
|
|
40942
40984
|
`);
|
|
40943
40985
|
}
|
|
@@ -41361,8 +41403,8 @@ var LOCOMO_CATEGORY_ORDER2 = ["single_hop", "multi_hop", "temporal", "open_domai
|
|
|
41361
41403
|
var LOCOMO_TASK_CATEGORY_PATTERN2 = /-(single_hop|multi_hop|temporal|open_domain|adversarial)$/;
|
|
41362
41404
|
var SOURCE_TURN_PATTERN = /^\[([^,\]\s]+),\s*turn\s+(\d+),\s*([^,\]]+?)(?:,\s*score\s+[^\]]+)?\]/i;
|
|
41363
41405
|
var SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
|
41364
|
-
function sanitizeLoComoResultReference(
|
|
41365
|
-
const reference = basename2(
|
|
41406
|
+
function sanitizeLoComoResultReference(path44) {
|
|
41407
|
+
const reference = basename2(path44).replace(/[\u0000-\u001f\u007f`]/g, "_");
|
|
41366
41408
|
if (!reference) throw new Error("Result path must identify a file.");
|
|
41367
41409
|
return reference;
|
|
41368
41410
|
}
|
|
@@ -42920,50 +42962,50 @@ function assertMemoryIdRef(value) {
|
|
|
42920
42962
|
throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
|
|
42921
42963
|
}
|
|
42922
42964
|
}
|
|
42923
|
-
function assertJsonConfig(value,
|
|
42965
|
+
function assertJsonConfig(value, path44 = "retrievalConfig") {
|
|
42924
42966
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
42925
42967
|
if (typeof value === "number") {
|
|
42926
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
42968
|
+
if (!Number.isFinite(value)) throw new Error(`${path44} must contain only finite JSON numbers.`);
|
|
42927
42969
|
return value;
|
|
42928
42970
|
}
|
|
42929
42971
|
if (Array.isArray(value)) {
|
|
42930
|
-
return value.map((entry, index) => assertJsonConfig(entry, `${
|
|
42972
|
+
return value.map((entry, index) => assertJsonConfig(entry, `${path44}[${index}]`));
|
|
42931
42973
|
}
|
|
42932
42974
|
if (!value || typeof value !== "object") {
|
|
42933
|
-
throw new Error(`${
|
|
42975
|
+
throw new Error(`${path44} must be JSON-serializable and provider-free.`);
|
|
42934
42976
|
}
|
|
42935
42977
|
const output = {};
|
|
42936
42978
|
for (const key of Object.keys(value).sort()) {
|
|
42937
42979
|
const child = value[key];
|
|
42938
42980
|
if (key === "openaiApiKey") {
|
|
42939
42981
|
if (child !== false) {
|
|
42940
|
-
throw new Error(`${
|
|
42982
|
+
throw new Error(`${path44}.${key} must be exactly false for provider-free capture.`);
|
|
42941
42983
|
}
|
|
42942
42984
|
output[key] = false;
|
|
42943
42985
|
continue;
|
|
42944
42986
|
}
|
|
42945
42987
|
if (isSecretKey(key)) {
|
|
42946
|
-
throw new Error(`${
|
|
42988
|
+
throw new Error(`${path44}.${key} contains secret-bearing configuration.`);
|
|
42947
42989
|
}
|
|
42948
42990
|
if (child === void 0) continue;
|
|
42949
42991
|
if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
|
|
42950
|
-
throw new Error(`${
|
|
42992
|
+
throw new Error(`${path44}.${key} is provider-capable configuration.`);
|
|
42951
42993
|
}
|
|
42952
|
-
output[key] = assertJsonConfig(child, `${
|
|
42994
|
+
output[key] = assertJsonConfig(child, `${path44}.${key}`);
|
|
42953
42995
|
}
|
|
42954
42996
|
return output;
|
|
42955
42997
|
}
|
|
42956
|
-
function sanitizeProviderFreeRetrievalConfig(value,
|
|
42998
|
+
function sanitizeProviderFreeRetrievalConfig(value, path44 = "retrievalConfig") {
|
|
42957
42999
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
42958
43000
|
if (typeof value === "number") {
|
|
42959
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
43001
|
+
if (!Number.isFinite(value)) throw new Error(`${path44} must contain only finite JSON numbers.`);
|
|
42960
43002
|
return value;
|
|
42961
43003
|
}
|
|
42962
43004
|
if (Array.isArray(value)) {
|
|
42963
|
-
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${
|
|
43005
|
+
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path44}[${index}]`));
|
|
42964
43006
|
}
|
|
42965
43007
|
if (!value || typeof value !== "object") {
|
|
42966
|
-
throw new Error(`${
|
|
43008
|
+
throw new Error(`${path44} must be JSON-serializable.`);
|
|
42967
43009
|
}
|
|
42968
43010
|
const output = {};
|
|
42969
43011
|
for (const key of Object.keys(value).sort()) {
|
|
@@ -42972,7 +43014,7 @@ function sanitizeProviderFreeRetrievalConfig(value, path43 = "retrievalConfig")
|
|
|
42972
43014
|
if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
|
|
42973
43015
|
continue;
|
|
42974
43016
|
}
|
|
42975
|
-
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${
|
|
43017
|
+
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path44}.${key}`);
|
|
42976
43018
|
}
|
|
42977
43019
|
return output;
|
|
42978
43020
|
}
|
|
@@ -42980,10 +43022,10 @@ function sanitizeProviderFreeRetrievalConfig(value, path43 = "retrievalConfig")
|
|
|
42980
43022
|
// src/result-summary.ts
|
|
42981
43023
|
import { existsSync as existsSync2 } from "fs";
|
|
42982
43024
|
import { readdir as readdir8 } from "fs/promises";
|
|
42983
|
-
import
|
|
43025
|
+
import path34 from "path";
|
|
42984
43026
|
|
|
42985
43027
|
// src/integrity/sealed-qrels.ts
|
|
42986
|
-
import { readFile as
|
|
43028
|
+
import { readFile as readFile20 } from "fs/promises";
|
|
42987
43029
|
function isSealedQrelsArtifact(value) {
|
|
42988
43030
|
if (!value || typeof value !== "object") {
|
|
42989
43031
|
return false;
|
|
@@ -43053,7 +43095,7 @@ function parseSealedQrels(raw, options = {}) {
|
|
|
43053
43095
|
};
|
|
43054
43096
|
}
|
|
43055
43097
|
async function loadSealedQrels(filePath, options = {}) {
|
|
43056
|
-
const raw = await
|
|
43098
|
+
const raw = await readFile20(filePath, "utf8");
|
|
43057
43099
|
return parseSealedQrels(raw, options);
|
|
43058
43100
|
}
|
|
43059
43101
|
function serializeSealedQrels(artifact) {
|
|
@@ -43310,7 +43352,7 @@ async function loadBenchmarkResultSummaries(resultsDir) {
|
|
|
43310
43352
|
if (!entry.isFile() || !entry.name.endsWith(".json")) {
|
|
43311
43353
|
continue;
|
|
43312
43354
|
}
|
|
43313
|
-
const filePath =
|
|
43355
|
+
const filePath = path34.join(resultsDir, entry.name);
|
|
43314
43356
|
try {
|
|
43315
43357
|
const result = await loadBenchmarkResult(filePath);
|
|
43316
43358
|
summaries.push(summarizeBenchmarkResult(result, filePath));
|
|
@@ -43330,7 +43372,7 @@ async function loadBenchmarkResultSummaries(resultsDir) {
|
|
|
43330
43372
|
}
|
|
43331
43373
|
|
|
43332
43374
|
// src/benchmarks/custom/loader.ts
|
|
43333
|
-
import { readFile as
|
|
43375
|
+
import { readFile as readFile21 } from "fs/promises";
|
|
43334
43376
|
import { parse as parseYaml } from "yaml";
|
|
43335
43377
|
var CUSTOM_SCORING_VALUES = /* @__PURE__ */ new Set([
|
|
43336
43378
|
"exact_match",
|
|
@@ -43350,7 +43392,7 @@ function parseCustomBenchmark(source) {
|
|
|
43350
43392
|
async function loadCustomBenchmarkFile(filePath) {
|
|
43351
43393
|
let source;
|
|
43352
43394
|
try {
|
|
43353
|
-
source = await
|
|
43395
|
+
source = await readFile21(filePath, "utf8");
|
|
43354
43396
|
} catch (error) {
|
|
43355
43397
|
throw new Error(
|
|
43356
43398
|
`Failed to read custom benchmark file ${filePath}: ${formatError(error)}`
|
|
@@ -43458,7 +43500,7 @@ function formatError(error) {
|
|
|
43458
43500
|
|
|
43459
43501
|
// src/benchmarks/custom/runner.ts
|
|
43460
43502
|
import { randomUUID as randomUUID35 } from "crypto";
|
|
43461
|
-
import
|
|
43503
|
+
import path35 from "path";
|
|
43462
43504
|
import { expandTildePath as expandTildePath4 } from "@remnic/core";
|
|
43463
43505
|
async function runCustomBenchmarkFile(filePath, options) {
|
|
43464
43506
|
const spec = await loadCustomBenchmarkFile(filePath);
|
|
@@ -43471,7 +43513,7 @@ async function runCustomBenchmarkFile(filePath, options) {
|
|
|
43471
43513
|
let cacheRestore;
|
|
43472
43514
|
let cacheCounters;
|
|
43473
43515
|
if (spec.scoring === "llm_judge" && runOptions.system.judge !== void 0 && !runOptions.noJudgeCache && (runOptions.judgeProvider ?? null) !== null) {
|
|
43474
|
-
const cacheDir = runOptions.judgeCacheDir ?
|
|
43516
|
+
const cacheDir = runOptions.judgeCacheDir ? path35.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path35.join(path35.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
|
|
43475
43517
|
if (cacheDir !== void 0) {
|
|
43476
43518
|
const originalJudge = runOptions.system.judge;
|
|
43477
43519
|
const wrapped = wrapJudgeWithCache({
|
|
@@ -43672,7 +43714,7 @@ async function scoreTask(scoring, options, question, actual, expected) {
|
|
|
43672
43714
|
}
|
|
43673
43715
|
}
|
|
43674
43716
|
function createCustomBenchmarkDefinition(benchmark, filePath) {
|
|
43675
|
-
const id = `custom:${slugify(
|
|
43717
|
+
const id = `custom:${slugify(path35.basename(filePath, path35.extname(filePath)) || benchmark.name)}`;
|
|
43676
43718
|
return {
|
|
43677
43719
|
id,
|
|
43678
43720
|
title: benchmark.name,
|
|
@@ -44543,8 +44585,8 @@ var chatFixture = {
|
|
|
44543
44585
|
|
|
44544
44586
|
// src/judges/calibration-slice.ts
|
|
44545
44587
|
import { createHash as createHash19, randomBytes as randomBytes2 } from "crypto";
|
|
44546
|
-
import { chmod as chmod2, lstat as lstat6, mkdir as mkdir15, open as open2, readFile as
|
|
44547
|
-
import
|
|
44588
|
+
import { chmod as chmod2, lstat as lstat6, mkdir as mkdir15, open as open2, readFile as readFile22, rename as rename5, unlink as unlink3, writeFile as writeFile14 } from "fs/promises";
|
|
44589
|
+
import path36 from "path";
|
|
44548
44590
|
|
|
44549
44591
|
// src/judges/cohen-kappa.ts
|
|
44550
44592
|
var DEFAULT_KAPPA_BOOTSTRAP_SAMPLES = 2e3;
|
|
@@ -44844,7 +44886,7 @@ async function loadOrInitializeCheckpoint(benchmarkId, provenance, sliceQuestion
|
|
|
44844
44886
|
throw new Error("runJudgeCalibration: checkpoint ordered-question-id hash does not match the validated source.");
|
|
44845
44887
|
}
|
|
44846
44888
|
await ensurePrivateDirectory(provenance.dir);
|
|
44847
|
-
const checkpointPath =
|
|
44889
|
+
const checkpointPath = path36.join(provenance.dir, `${sanitizeCalibrationSegment(benchmarkId)}.checkpoint.json`);
|
|
44848
44890
|
const lockPath = `${checkpointPath}.lock`;
|
|
44849
44891
|
let lockHandle;
|
|
44850
44892
|
try {
|
|
@@ -44892,7 +44934,7 @@ async function loadOrInitializeCheckpoint(benchmarkId, provenance, sliceQuestion
|
|
|
44892
44934
|
try {
|
|
44893
44935
|
const info = await lstat6(checkpointPath);
|
|
44894
44936
|
if (!info.isFile() || info.isSymbolicLink()) throw new Error("checkpoint path is not a regular file");
|
|
44895
|
-
raw = await
|
|
44937
|
+
raw = await readFile22(checkpointPath, "utf8");
|
|
44896
44938
|
} catch (error) {
|
|
44897
44939
|
if (error.code !== "ENOENT") throw error;
|
|
44898
44940
|
const state = { schemaVersion: 2, contractHash, contract, completed: {} };
|
|
@@ -44968,7 +45010,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities, pr
|
|
|
44968
45010
|
...provenance ? provenance : {},
|
|
44969
45011
|
...identities ? identities : {}
|
|
44970
45012
|
};
|
|
44971
|
-
const filePath =
|
|
45013
|
+
const filePath = path36.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
|
|
44972
45014
|
const tempPath = `${filePath}.${randomBytes2(6).toString("hex")}.tmp`;
|
|
44973
45015
|
await writeFile14(tempPath, `${JSON.stringify(state, null, 2)}
|
|
44974
45016
|
`, { encoding: "utf8", mode: 384 });
|
|
@@ -44982,10 +45024,10 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities, pr
|
|
|
44982
45024
|
return filePath;
|
|
44983
45025
|
}
|
|
44984
45026
|
async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
|
|
44985
|
-
const filePath =
|
|
45027
|
+
const filePath = path36.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
|
|
44986
45028
|
let raw;
|
|
44987
45029
|
try {
|
|
44988
|
-
raw = await
|
|
45030
|
+
raw = await readFile22(filePath, "utf8");
|
|
44989
45031
|
} catch {
|
|
44990
45032
|
return void 0;
|
|
44991
45033
|
}
|
|
@@ -45077,9 +45119,9 @@ function sanitizeCalibrationSegment(value) {
|
|
|
45077
45119
|
}
|
|
45078
45120
|
|
|
45079
45121
|
// src/benchmarks/remnic/procedural-recall/ablation.ts
|
|
45080
|
-
import { mkdir as mkdir16, mkdtemp as mkdtemp12, rm as rm16, writeFile as writeFile15, readFile as
|
|
45122
|
+
import { mkdir as mkdir16, mkdtemp as mkdtemp12, rm as rm16, writeFile as writeFile15, readFile as readFile23 } from "fs/promises";
|
|
45081
45123
|
import os8 from "os";
|
|
45082
|
-
import
|
|
45124
|
+
import path37 from "path";
|
|
45083
45125
|
import {
|
|
45084
45126
|
StorageManager as StorageManager4,
|
|
45085
45127
|
parseConfig as parseConfig5,
|
|
@@ -45111,7 +45153,7 @@ async function runSide(scenarios, proceduralEnabled) {
|
|
|
45111
45153
|
const observed = [];
|
|
45112
45154
|
for (const scenario of scenarios) {
|
|
45113
45155
|
const dir = await mkdtemp12(
|
|
45114
|
-
|
|
45156
|
+
path37.join(os8.tmpdir(), "remnic-bench-proc-ablation-")
|
|
45115
45157
|
);
|
|
45116
45158
|
try {
|
|
45117
45159
|
const storage = new StorageManager4(dir);
|
|
@@ -45132,7 +45174,7 @@ ${body}`,
|
|
|
45132
45174
|
);
|
|
45133
45175
|
const config = parseConfig5({
|
|
45134
45176
|
memoryDir: dir,
|
|
45135
|
-
workspaceDir:
|
|
45177
|
+
workspaceDir: path37.join(dir, "ws"),
|
|
45136
45178
|
openaiApiKey: "bench-key",
|
|
45137
45179
|
procedural: {
|
|
45138
45180
|
enabled: proceduralEnabled,
|
|
@@ -45207,7 +45249,7 @@ async function runProceduralAblation(options) {
|
|
|
45207
45249
|
};
|
|
45208
45250
|
}
|
|
45209
45251
|
async function loadAblationFixture(fixturePath) {
|
|
45210
|
-
const raw = await
|
|
45252
|
+
const raw = await readFile23(fixturePath, "utf8");
|
|
45211
45253
|
let parsed;
|
|
45212
45254
|
try {
|
|
45213
45255
|
parsed = JSON.parse(raw);
|
|
@@ -45303,7 +45345,7 @@ async function runProceduralAblationCli(args) {
|
|
|
45303
45345
|
random: args.random,
|
|
45304
45346
|
seed: args.seed
|
|
45305
45347
|
});
|
|
45306
|
-
const outDir =
|
|
45348
|
+
const outDir = path37.dirname(path37.resolve(args.outPath));
|
|
45307
45349
|
await mkdir16(outDir, { recursive: true });
|
|
45308
45350
|
await writeFile15(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
|
|
45309
45351
|
return artifact;
|
|
@@ -46420,19 +46462,19 @@ function createMitigatedTarget(config) {
|
|
|
46420
46462
|
|
|
46421
46463
|
// src/security/injection-suite/runner.ts
|
|
46422
46464
|
import { createHash as createHash22 } from "crypto";
|
|
46423
|
-
import { mkdir as mkdir19, readFile as
|
|
46424
|
-
import
|
|
46465
|
+
import { mkdir as mkdir19, readFile as readFile26, writeFile as writeFile18 } from "fs/promises";
|
|
46466
|
+
import path40 from "path";
|
|
46425
46467
|
|
|
46426
46468
|
// src/security/injection-suite/claims.ts
|
|
46427
46469
|
import { hostname } from "os";
|
|
46428
|
-
import { mkdir as mkdir18, readFile as
|
|
46429
|
-
import
|
|
46470
|
+
import { mkdir as mkdir18, readFile as readFile25, rename as rename7, rm as rm17, stat as stat4, utimes, writeFile as writeFile17 } from "fs/promises";
|
|
46471
|
+
import path39 from "path";
|
|
46430
46472
|
import { randomUUID as randomUUID37 } from "crypto";
|
|
46431
46473
|
|
|
46432
46474
|
// src/security/injection-suite/store.ts
|
|
46433
46475
|
import { createHash as createHash20, randomUUID as randomUUID36 } from "crypto";
|
|
46434
|
-
import { mkdir as mkdir17, readFile as
|
|
46435
|
-
import
|
|
46476
|
+
import { mkdir as mkdir17, readFile as readFile24, rename as rename6, writeFile as writeFile16 } from "fs/promises";
|
|
46477
|
+
import path38 from "path";
|
|
46436
46478
|
|
|
46437
46479
|
// src/security/injection-suite/types.ts
|
|
46438
46480
|
var INJECTION_SUITE_VERSION = "h5-injection-suite-v1";
|
|
@@ -46472,17 +46514,17 @@ var InjectionSuiteRowStore = class {
|
|
|
46472
46514
|
outputDir;
|
|
46473
46515
|
checkpointsDir;
|
|
46474
46516
|
constructor(outputDir) {
|
|
46475
|
-
this.outputDir =
|
|
46476
|
-
this.checkpointsDir =
|
|
46517
|
+
this.outputDir = path38.resolve(outputDir);
|
|
46518
|
+
this.checkpointsDir = path38.join(this.outputDir, "checkpoints");
|
|
46477
46519
|
}
|
|
46478
46520
|
checkpointPath(identity) {
|
|
46479
|
-
return
|
|
46521
|
+
return path38.join(this.checkpointsDir, `${buildInjectionSuiteRowKey(identity)}.json`);
|
|
46480
46522
|
}
|
|
46481
46523
|
async load(identity) {
|
|
46482
46524
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
46483
46525
|
let raw;
|
|
46484
46526
|
try {
|
|
46485
|
-
raw = await
|
|
46527
|
+
raw = await readFile24(this.checkpointPath(identity), "utf8");
|
|
46486
46528
|
} catch (error) {
|
|
46487
46529
|
if (error.code === "ENOENT") return { kind: "MISSING" };
|
|
46488
46530
|
return { kind: "MALFORMED", error: error instanceof Error ? error : new Error(String(error)) };
|
|
@@ -46539,7 +46581,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
46539
46581
|
heartbeatMs;
|
|
46540
46582
|
heartbeats = /* @__PURE__ */ new Map();
|
|
46541
46583
|
lockPath(rowKey) {
|
|
46542
|
-
return
|
|
46584
|
+
return path39.join(this.checkpointsDir, `${rowKey}.lock`);
|
|
46543
46585
|
}
|
|
46544
46586
|
async tryClaim(identity) {
|
|
46545
46587
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
@@ -46565,7 +46607,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
46565
46607
|
claimedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
46566
46608
|
};
|
|
46567
46609
|
try {
|
|
46568
|
-
await writeFile17(
|
|
46610
|
+
await writeFile17(path39.join(lockPath, "owner.json"), `${JSON.stringify(owner)}
|
|
46569
46611
|
`, {
|
|
46570
46612
|
flag: "wx"
|
|
46571
46613
|
});
|
|
@@ -46579,8 +46621,8 @@ var InjectionSuiteClaimLock = class {
|
|
|
46579
46621
|
async release(claim) {
|
|
46580
46622
|
this.stopHeartbeat(claim.lockPath);
|
|
46581
46623
|
try {
|
|
46582
|
-
const ownerPath =
|
|
46583
|
-
const owner = JSON.parse(await
|
|
46624
|
+
const ownerPath = path39.join(claim.lockPath, "owner.json");
|
|
46625
|
+
const owner = JSON.parse(await readFile25(ownerPath, "utf8"));
|
|
46584
46626
|
if (owner.ownerToken !== claim.ownerToken) return;
|
|
46585
46627
|
await rename7(ownerPath, `${ownerPath}.released-${claim.ownerToken}`);
|
|
46586
46628
|
} catch {
|
|
@@ -46595,7 +46637,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
46595
46637
|
await rm17(released, { recursive: true, force: true });
|
|
46596
46638
|
}
|
|
46597
46639
|
async assertOwner(claim) {
|
|
46598
|
-
const owner = JSON.parse(await
|
|
46640
|
+
const owner = JSON.parse(await readFile25(path39.join(claim.lockPath, "owner.json"), "utf8"));
|
|
46599
46641
|
if (owner.ownerToken !== claim.ownerToken) {
|
|
46600
46642
|
throw new Error(`lost injection-suite claim ${claim.rowKey}`);
|
|
46601
46643
|
}
|
|
@@ -46603,7 +46645,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
46603
46645
|
startHeartbeat(lockPath) {
|
|
46604
46646
|
this.stopHeartbeat(lockPath);
|
|
46605
46647
|
const timer = setInterval(() => {
|
|
46606
|
-
void utimes(
|
|
46648
|
+
void utimes(path39.join(lockPath, "owner.json"), /* @__PURE__ */ new Date(), /* @__PURE__ */ new Date()).catch(() => void 0);
|
|
46607
46649
|
}, this.heartbeatMs);
|
|
46608
46650
|
timer.unref?.();
|
|
46609
46651
|
this.heartbeats.set(lockPath, timer);
|
|
@@ -46614,11 +46656,11 @@ var InjectionSuiteClaimLock = class {
|
|
|
46614
46656
|
clearInterval(timer);
|
|
46615
46657
|
}
|
|
46616
46658
|
async reclaimIfExpired(lockPath) {
|
|
46617
|
-
const ownerPath =
|
|
46659
|
+
const ownerPath = path39.join(lockPath, "owner.json");
|
|
46618
46660
|
let leaseMs = this.leaseMs;
|
|
46619
46661
|
let stampMs;
|
|
46620
46662
|
try {
|
|
46621
|
-
const owner = JSON.parse(await
|
|
46663
|
+
const owner = JSON.parse(await readFile25(ownerPath, "utf8"));
|
|
46622
46664
|
if (typeof owner.leaseMs === "number" && owner.leaseMs > 0) leaseMs = owner.leaseMs;
|
|
46623
46665
|
stampMs = (await stat4(ownerPath)).mtimeMs;
|
|
46624
46666
|
} catch {
|
|
@@ -46897,19 +46939,19 @@ async function executeRow(identity, variant, input) {
|
|
|
46897
46939
|
}
|
|
46898
46940
|
async function readRunMetadata(outputDir) {
|
|
46899
46941
|
try {
|
|
46900
|
-
return JSON.parse(await
|
|
46942
|
+
return JSON.parse(await readFile26(path40.join(outputDir, "run.json"), "utf8"));
|
|
46901
46943
|
} catch (error) {
|
|
46902
46944
|
if (error.code === "ENOENT") return void 0;
|
|
46903
46945
|
throw error;
|
|
46904
46946
|
}
|
|
46905
46947
|
}
|
|
46906
46948
|
async function appendEpisode(outputDir, row) {
|
|
46907
|
-
await writeFile18(
|
|
46949
|
+
await writeFile18(path40.join(outputDir, "episodes.jsonl"), `${JSON.stringify(row)}
|
|
46908
46950
|
`, { flag: "a" });
|
|
46909
46951
|
}
|
|
46910
46952
|
async function ensureEpisode(outputDir, row) {
|
|
46911
46953
|
try {
|
|
46912
|
-
const existing = await
|
|
46954
|
+
const existing = await readFile26(path40.join(outputDir, "episodes.jsonl"), "utf8");
|
|
46913
46955
|
if (existing.includes(row.rowKey)) return;
|
|
46914
46956
|
} catch (error) {
|
|
46915
46957
|
if (error.code !== "ENOENT") throw error;
|
|
@@ -46948,7 +46990,7 @@ async function runInjectionSuiteCliCommand(input) {
|
|
|
46948
46990
|
limit: input.limit ?? null
|
|
46949
46991
|
};
|
|
46950
46992
|
try {
|
|
46951
|
-
await writeFile18(
|
|
46993
|
+
await writeFile18(path40.join(input.outputDir, "run.json"), `${JSON.stringify(metadata, null, 2)}
|
|
46952
46994
|
`, {
|
|
46953
46995
|
flag: "wx"
|
|
46954
46996
|
});
|
|
@@ -47194,7 +47236,7 @@ import { performance as performance2 } from "perf_hooks";
|
|
|
47194
47236
|
import { mkdtemp as mkdtemp13, rm as rm18 } from "fs/promises";
|
|
47195
47237
|
import { statSync } from "fs";
|
|
47196
47238
|
import { tmpdir as tmpdir7 } from "os";
|
|
47197
|
-
import
|
|
47239
|
+
import path41 from "path";
|
|
47198
47240
|
import os9 from "os";
|
|
47199
47241
|
import {
|
|
47200
47242
|
GraphStore
|
|
@@ -47291,15 +47333,15 @@ async function runCodingGraphBenchmark(config = {}) {
|
|
|
47291
47333
|
const sampleRss = () => {
|
|
47292
47334
|
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
|
47293
47335
|
};
|
|
47294
|
-
const dir = await mkdtemp13(
|
|
47295
|
-
const dbPath =
|
|
47336
|
+
const dir = await mkdtemp13(path41.join(tmpdir7(), "coding-graph-bench-"));
|
|
47337
|
+
const dbPath = path41.join(dir, "bench.sqlite");
|
|
47296
47338
|
try {
|
|
47297
47339
|
const store = await GraphStore.open({ dbPath });
|
|
47298
47340
|
try {
|
|
47299
47341
|
const FULL_INDEX_SAMPLES = 3;
|
|
47300
47342
|
const fullIndexSamples = [];
|
|
47301
47343
|
for (let s = 0; s < FULL_INDEX_SAMPLES; s++) {
|
|
47302
|
-
const sampleStore = s === 0 ? store : await GraphStore.open({ dbPath:
|
|
47344
|
+
const sampleStore = s === 0 ? store : await GraphStore.open({ dbPath: path41.join(dir, `bench-warm-${s}.sqlite`) });
|
|
47303
47345
|
const fi = await timeAsync(() => sampleStore.upsertFileBatch(storeFiles));
|
|
47304
47346
|
if (!fi.result.ok) {
|
|
47305
47347
|
if (sampleStore !== store) await sampleStore.close();
|
|
@@ -47622,7 +47664,7 @@ function buildBaselineFromReport(report, note) {
|
|
|
47622
47664
|
// src/coding-graph/repeated-failure-report.ts
|
|
47623
47665
|
import { constants } from "fs";
|
|
47624
47666
|
import { lstat as lstat7, mkdir as mkdir20, open as open3 } from "fs/promises";
|
|
47625
|
-
import
|
|
47667
|
+
import path42 from "path";
|
|
47626
47668
|
import { writeFileAtomically } from "@remnic/core/maintenance/atomic-file";
|
|
47627
47669
|
import { z as z4 } from "zod";
|
|
47628
47670
|
|
|
@@ -48250,20 +48292,20 @@ var SOURCE_ARTIFACTS = [
|
|
|
48250
48292
|
async function readArtifactLeaf(filePath) {
|
|
48251
48293
|
const leaf = await lstat7(filePath);
|
|
48252
48294
|
if (!leaf.isFile()) {
|
|
48253
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
48295
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path42.basename(filePath)}`);
|
|
48254
48296
|
}
|
|
48255
48297
|
let handle;
|
|
48256
48298
|
try {
|
|
48257
48299
|
handle = await open3(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
48258
48300
|
} catch (error) {
|
|
48259
48301
|
if (error.code === "ELOOP") {
|
|
48260
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
48302
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path42.basename(filePath)}`);
|
|
48261
48303
|
}
|
|
48262
48304
|
throw error;
|
|
48263
48305
|
}
|
|
48264
48306
|
try {
|
|
48265
48307
|
if (!(await handle.stat()).isFile()) {
|
|
48266
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
48308
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path42.basename(filePath)}`);
|
|
48267
48309
|
}
|
|
48268
48310
|
return await handle.readFile();
|
|
48269
48311
|
} finally {
|
|
@@ -48467,7 +48509,7 @@ async function assertArtifactMayBeWritten(filePath, content) {
|
|
|
48467
48509
|
try {
|
|
48468
48510
|
const prior = await readArtifactLeaf(filePath);
|
|
48469
48511
|
if (!prior.equals(Buffer.from(content))) {
|
|
48470
|
-
throw new Error(`paper writer refuses to overwrite changed paper artifact: ${
|
|
48512
|
+
throw new Error(`paper writer refuses to overwrite changed paper artifact: ${path42.basename(filePath)}`);
|
|
48471
48513
|
}
|
|
48472
48514
|
return false;
|
|
48473
48515
|
} catch (error) {
|
|
@@ -48476,7 +48518,7 @@ async function assertArtifactMayBeWritten(filePath, content) {
|
|
|
48476
48518
|
}
|
|
48477
48519
|
}
|
|
48478
48520
|
async function writeRepeatedFailurePaperArtifacts(options) {
|
|
48479
|
-
const runDir =
|
|
48521
|
+
const runDir = path42.resolve(options.runDir);
|
|
48480
48522
|
const reproManifest = await verifyRunManifest(runDir);
|
|
48481
48523
|
const source = await readSourceArtifacts(runDir);
|
|
48482
48524
|
const runJson = JSON.parse(source["run.json"]);
|
|
@@ -48494,7 +48536,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
48494
48536
|
throw new Error("paper report preregistration does not match run metadata");
|
|
48495
48537
|
}
|
|
48496
48538
|
const committedFixtureDir = await resolveCommittedH6FixtureDirectory();
|
|
48497
|
-
const committedDecisionRuleBytes = (await readArtifactLeaf(
|
|
48539
|
+
const committedDecisionRuleBytes = (await readArtifactLeaf(path42.join(committedFixtureDir, "decision-rule.json"))).toString("utf8");
|
|
48498
48540
|
if (decisionRuleBytes !== committedDecisionRuleBytes) {
|
|
48499
48541
|
throw new Error("paper report decision rule differs from the frozen committed artifact");
|
|
48500
48542
|
}
|
|
@@ -48853,7 +48895,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
48853
48895
|
await Promise.all(writeStates.map(async (artifact) => {
|
|
48854
48896
|
const bytes = await readArtifactLeaf(artifact.artifactPath);
|
|
48855
48897
|
if (!bytes.equals(Buffer.from(artifact.content))) {
|
|
48856
|
-
throw new Error(`paper artifact verification failed: ${
|
|
48898
|
+
throw new Error(`paper artifact verification failed: ${path42.basename(artifact.artifactPath)}`);
|
|
48857
48899
|
}
|
|
48858
48900
|
}));
|
|
48859
48901
|
const artifactPaths = writeStates.map((artifact) => artifact.artifactPath);
|
|
@@ -48873,8 +48915,8 @@ async function runRepeatedFailurePaperReportCliCommand(options) {
|
|
|
48873
48915
|
return {
|
|
48874
48916
|
exitCode: 0,
|
|
48875
48917
|
output: JSON.stringify({
|
|
48876
|
-
reportPath:
|
|
48877
|
-
manifestPath:
|
|
48918
|
+
reportPath: path42.relative(path42.resolve(options.runDir), result.reportPath),
|
|
48919
|
+
manifestPath: path42.relative(path42.resolve(options.runDir), result.manifestPath)
|
|
48878
48920
|
})
|
|
48879
48921
|
};
|
|
48880
48922
|
} catch (error) {
|
|
@@ -48884,8 +48926,8 @@ async function runRepeatedFailurePaperReportCliCommand(options) {
|
|
|
48884
48926
|
|
|
48885
48927
|
// src/attribute-cli.ts
|
|
48886
48928
|
import { QmdClient } from "@remnic/core";
|
|
48887
|
-
import { lstat as lstat8, readdir as readdir9, readFile as
|
|
48888
|
-
import
|
|
48929
|
+
import { lstat as lstat8, readdir as readdir9, readFile as readFile27 } from "fs/promises";
|
|
48930
|
+
import path43 from "path";
|
|
48889
48931
|
function parseFrontmatter2(fileContent) {
|
|
48890
48932
|
const lines = fileContent.split(/\r?\n/);
|
|
48891
48933
|
if (lines.length > 0 && lines[0].trim() === "---") {
|
|
@@ -48949,7 +48991,7 @@ async function scanMemoryDir(dirPath) {
|
|
|
48949
48991
|
if (entry.isSymbolicLink()) {
|
|
48950
48992
|
continue;
|
|
48951
48993
|
}
|
|
48952
|
-
const fullPath =
|
|
48994
|
+
const fullPath = path43.join(currentDir, entry.name);
|
|
48953
48995
|
try {
|
|
48954
48996
|
const stats = await lstat8(fullPath);
|
|
48955
48997
|
if (stats.isSymbolicLink()) {
|
|
@@ -48961,9 +49003,9 @@ async function scanMemoryDir(dirPath) {
|
|
|
48961
49003
|
}
|
|
48962
49004
|
await walk(fullPath, depth + 1);
|
|
48963
49005
|
} else if (stats.isFile() && entry.name.endsWith(".md")) {
|
|
48964
|
-
const content = await
|
|
49006
|
+
const content = await readFile27(fullPath, "utf8");
|
|
48965
49007
|
const { id, body } = parseFrontmatter2(content);
|
|
48966
|
-
const relPath =
|
|
49008
|
+
const relPath = path43.relative(dirPath, fullPath);
|
|
48967
49009
|
memories.push({
|
|
48968
49010
|
id: id ?? relPath,
|
|
48969
49011
|
content: body.trim()
|
|
@@ -48981,24 +49023,24 @@ async function scanMemoryDir(dirPath) {
|
|
|
48981
49023
|
return memories;
|
|
48982
49024
|
}
|
|
48983
49025
|
async function resolveQmdMemory(memoryDir, collection, resultPath) {
|
|
48984
|
-
const root =
|
|
49026
|
+
const root = path43.resolve(memoryDir);
|
|
48985
49027
|
const candidates = /* @__PURE__ */ new Set();
|
|
48986
49028
|
const addCandidate = (candidate) => {
|
|
48987
|
-
const resolved =
|
|
48988
|
-
const relative =
|
|
48989
|
-
if (relative !== ".." && !relative.startsWith(`..${
|
|
49029
|
+
const resolved = path43.resolve(candidate);
|
|
49030
|
+
const relative = path43.relative(root, resolved);
|
|
49031
|
+
if (relative !== ".." && !relative.startsWith(`..${path43.sep}`) && !path43.isAbsolute(relative)) {
|
|
48990
49032
|
candidates.add(resolved);
|
|
48991
49033
|
}
|
|
48992
49034
|
};
|
|
48993
49035
|
const addRelative = (relativePath) => {
|
|
48994
49036
|
const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
48995
49037
|
if (!normalized) return;
|
|
48996
|
-
addCandidate(
|
|
49038
|
+
addCandidate(path43.join(root, normalized));
|
|
48997
49039
|
if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
|
|
48998
|
-
addCandidate(
|
|
49040
|
+
addCandidate(path43.join(root, "facts", normalized));
|
|
48999
49041
|
}
|
|
49000
49042
|
};
|
|
49001
|
-
if (
|
|
49043
|
+
if (path43.isAbsolute(resultPath)) {
|
|
49002
49044
|
addCandidate(resultPath);
|
|
49003
49045
|
} else {
|
|
49004
49046
|
addRelative(resultPath);
|
|
@@ -49012,7 +49054,7 @@ async function resolveQmdMemory(memoryDir, collection, resultPath) {
|
|
|
49012
49054
|
try {
|
|
49013
49055
|
const stats = await lstat8(candidate);
|
|
49014
49056
|
if (!stats.isFile() || stats.isSymbolicLink()) continue;
|
|
49015
|
-
const parsed = parseFrontmatter2(await
|
|
49057
|
+
const parsed = parseFrontmatter2(await readFile27(candidate, "utf8"));
|
|
49016
49058
|
if (!parsed.id || parsed.id.trim().length === 0) {
|
|
49017
49059
|
throw new Error("QMD result has no canonical frontmatter id");
|
|
49018
49060
|
}
|