@remnic/bench 9.6.32 → 9.6.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +223 -16
- package/dist/index.js +1312 -72
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -8998,8 +8998,8 @@ function parseRetryAfterMs(value) {
|
|
|
8998
8998
|
if (Number.isNaN(asNumber2)) {
|
|
8999
8999
|
const dateMs = Date.parse(value);
|
|
9000
9000
|
if (Number.isFinite(dateMs)) {
|
|
9001
|
-
const
|
|
9002
|
-
return
|
|
9001
|
+
const delta2 = dateMs - Date.now();
|
|
9002
|
+
return delta2 > 0 ? Math.min(delta2, MAX_RETRY_AFTER_S * 1e3) : 0;
|
|
9003
9003
|
}
|
|
9004
9004
|
}
|
|
9005
9005
|
return void 0;
|
|
@@ -16394,11 +16394,11 @@ function tokenize(value) {
|
|
|
16394
16394
|
return normalizeText(value).replace(/[^\w\s]/g, " ").split(/\s+/).filter((token) => token.length > 0);
|
|
16395
16395
|
}
|
|
16396
16396
|
function frequencyMap(tokens) {
|
|
16397
|
-
const
|
|
16397
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
16398
16398
|
for (const token of tokens) {
|
|
16399
|
-
|
|
16399
|
+
counts2.set(token, (counts2.get(token) ?? 0) + 1);
|
|
16400
16400
|
}
|
|
16401
|
-
return
|
|
16401
|
+
return counts2;
|
|
16402
16402
|
}
|
|
16403
16403
|
function longestCommonSubsequence(left, right) {
|
|
16404
16404
|
let previous = new Array(right.length + 1).fill(0);
|
|
@@ -16806,11 +16806,11 @@ function buildAmaBenchRecallQueries(qaPairs) {
|
|
|
16806
16806
|
});
|
|
16807
16807
|
}
|
|
16808
16808
|
function questionCounts(qaPairs) {
|
|
16809
|
-
const
|
|
16809
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
16810
16810
|
for (const qa of qaPairs) {
|
|
16811
|
-
|
|
16811
|
+
counts2.set(qa.question, (counts2.get(qa.question) ?? 0) + 1);
|
|
16812
16812
|
}
|
|
16813
|
-
return
|
|
16813
|
+
return counts2;
|
|
16814
16814
|
}
|
|
16815
16815
|
function isInventoryHistoryQuestion(question) {
|
|
16816
16816
|
const normalized = question.toLowerCase();
|
|
@@ -19847,26 +19847,6 @@ import { collectTemporalLexicalCues } from "@remnic/core";
|
|
|
19847
19847
|
import { readFile as readFile15 } from "fs/promises";
|
|
19848
19848
|
import path17 from "path";
|
|
19849
19849
|
|
|
19850
|
-
// src/benchmarks/published/longmemeval/fixture.ts
|
|
19851
|
-
var LONG_MEM_EVAL_SMOKE_FIXTURE = [
|
|
19852
|
-
{
|
|
19853
|
-
question_id: 1,
|
|
19854
|
-
question_type: "single-session-user",
|
|
19855
|
-
question: "What city does the user live in?",
|
|
19856
|
-
answer: "Paris",
|
|
19857
|
-
question_date: "2025-01-01",
|
|
19858
|
-
haystack_dates: ["2024-12-01"],
|
|
19859
|
-
haystack_session_ids: ["session-1"],
|
|
19860
|
-
haystack_sessions: [
|
|
19861
|
-
[
|
|
19862
|
-
{ role: "user", content: "I moved to Paris last year." },
|
|
19863
|
-
{ role: "assistant", content: "Paris sounds great." }
|
|
19864
|
-
]
|
|
19865
|
-
],
|
|
19866
|
-
answer_session_ids: ["session-1"]
|
|
19867
|
-
}
|
|
19868
|
-
];
|
|
19869
|
-
|
|
19870
19850
|
// src/benchmarks/published/locomo/fixture.ts
|
|
19871
19851
|
var LOCOMO_SMOKE_FIXTURE = [
|
|
19872
19852
|
{
|
|
@@ -19916,6 +19896,26 @@ var LOCOMO_SMOKE_FIXTURE = [
|
|
|
19916
19896
|
}
|
|
19917
19897
|
];
|
|
19918
19898
|
|
|
19899
|
+
// src/benchmarks/published/longmemeval/fixture.ts
|
|
19900
|
+
var LONG_MEM_EVAL_SMOKE_FIXTURE = [
|
|
19901
|
+
{
|
|
19902
|
+
question_id: 1,
|
|
19903
|
+
question_type: "single-session-user",
|
|
19904
|
+
question: "What city does the user live in?",
|
|
19905
|
+
answer: "Paris",
|
|
19906
|
+
question_date: "2025-01-01",
|
|
19907
|
+
haystack_dates: ["2024-12-01"],
|
|
19908
|
+
haystack_session_ids: ["session-1"],
|
|
19909
|
+
haystack_sessions: [
|
|
19910
|
+
[
|
|
19911
|
+
{ role: "user", content: "I moved to Paris last year." },
|
|
19912
|
+
{ role: "assistant", content: "Paris sounds great." }
|
|
19913
|
+
]
|
|
19914
|
+
],
|
|
19915
|
+
answer_session_ids: ["session-1"]
|
|
19916
|
+
}
|
|
19917
|
+
];
|
|
19918
|
+
|
|
19919
19919
|
// src/benchmarks/published/dataset-loader.ts
|
|
19920
19920
|
var LONG_MEM_EVAL_DATASET_FILENAMES = Object.freeze([
|
|
19921
19921
|
"longmemeval_oracle.json",
|
|
@@ -19963,6 +19963,7 @@ async function loadDataset4(options) {
|
|
|
19963
19963
|
return {
|
|
19964
19964
|
source: "dataset",
|
|
19965
19965
|
filename,
|
|
19966
|
+
sha256: hashString(raw),
|
|
19966
19967
|
items: applyLimit4(parsed, limit),
|
|
19967
19968
|
errors
|
|
19968
19969
|
};
|
|
@@ -19978,6 +19979,7 @@ async function loadDataset4(options) {
|
|
|
19978
19979
|
}
|
|
19979
19980
|
return {
|
|
19980
19981
|
source: "smoke",
|
|
19982
|
+
sha256: hashCanonicalJson(options.smokeFixture),
|
|
19981
19983
|
items: applyLimit4([...options.smokeFixture], limit),
|
|
19982
19984
|
errors
|
|
19983
19985
|
};
|
|
@@ -21263,11 +21265,12 @@ var locomoDefinition = {
|
|
|
21263
21265
|
}
|
|
21264
21266
|
};
|
|
21265
21267
|
async function runLoCoMoBenchmark(options) {
|
|
21266
|
-
const
|
|
21268
|
+
const loaded = await loadLoCoMoDataset(
|
|
21267
21269
|
options.mode,
|
|
21268
21270
|
options.datasetDir,
|
|
21269
21271
|
options.limit
|
|
21270
21272
|
);
|
|
21273
|
+
const conversations = loaded.items;
|
|
21271
21274
|
const trialLimit = resolveTrialLimit(options.benchmarkOptions?.trialLimit);
|
|
21272
21275
|
const multiHopRecallComposition = resolveLoCoMoBooleanOption(
|
|
21273
21276
|
options.benchmarkOptions?.multiHopRecallComposition,
|
|
@@ -21276,7 +21279,7 @@ async function runLoCoMoBenchmark(options) {
|
|
|
21276
21279
|
);
|
|
21277
21280
|
const plans = applyTrialLimit(
|
|
21278
21281
|
conversations.map(
|
|
21279
|
-
(conversation) =>
|
|
21282
|
+
(conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition)
|
|
21280
21283
|
),
|
|
21281
21284
|
trialLimit
|
|
21282
21285
|
);
|
|
@@ -21347,7 +21350,7 @@ function applyTrialLimit(plans, trialLimit) {
|
|
|
21347
21350
|
}
|
|
21348
21351
|
return limitedPlans;
|
|
21349
21352
|
}
|
|
21350
|
-
function
|
|
21353
|
+
function buildLoCoMoPlan(conversation, multiHopRecallComposition) {
|
|
21351
21354
|
const sessions = extractSessions(conversation.conversation);
|
|
21352
21355
|
const speakerA = typeof conversation.conversation.speaker_a === "string" ? conversation.conversation.speaker_a : "Speaker A";
|
|
21353
21356
|
const ingestSessions = [];
|
|
@@ -21383,9 +21386,9 @@ function buildTrial(conversationId, qa, questionIndex, sessionIds, multiHopRecal
|
|
|
21383
21386
|
expected: qa.answer,
|
|
21384
21387
|
recallSessionIds: sessionIds,
|
|
21385
21388
|
answerFormat: "short-with-specifics",
|
|
21386
|
-
recallTextTransform: ({ question, recalledText }) =>
|
|
21389
|
+
recallTextTransform: ({ question, recalledText }) => transformLoCoMoRecallText({
|
|
21387
21390
|
question,
|
|
21388
|
-
recalledText
|
|
21391
|
+
recalledText,
|
|
21389
21392
|
multiHopRecallComposition
|
|
21390
21393
|
}),
|
|
21391
21394
|
answerFallback: ({ question, recalledText }) => answerLoCoMoFromRecall(question, recalledText),
|
|
@@ -21586,10 +21589,20 @@ function sanitizeLoCoMoRecallText(args) {
|
|
|
21586
21589
|
(id) => queryVisibleIds.has(id) ? id : ""
|
|
21587
21590
|
);
|
|
21588
21591
|
}
|
|
21589
|
-
function
|
|
21590
|
-
const
|
|
21591
|
-
|
|
21592
|
-
|
|
21592
|
+
function transformLoCoMoRecallText(args) {
|
|
21593
|
+
const sanitized = sanitizeLoCoMoRecallText(args);
|
|
21594
|
+
return prioritizeLoCoMoRecallTextWithTrace({
|
|
21595
|
+
...args,
|
|
21596
|
+
recalledText: sanitized
|
|
21597
|
+
}).text;
|
|
21598
|
+
}
|
|
21599
|
+
function prioritizeLoCoMoRecallTextWithTrace(args) {
|
|
21600
|
+
const inputLines = args.recalledText.replaceAll("\r\n", "\n").replaceAll("\r", "\n").split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
21601
|
+
const lines = dedupePreserveOrder(inputLines);
|
|
21602
|
+
const inputOrdinalByLine = /* @__PURE__ */ new Map();
|
|
21603
|
+
inputLines.forEach((line, index) => {
|
|
21604
|
+
if (!inputOrdinalByLine.has(line)) inputOrdinalByLine.set(line, index);
|
|
21605
|
+
});
|
|
21593
21606
|
const questionTokens = expandLoCoMoQuestionTokens(
|
|
21594
21607
|
tokenizeForLoCoMo(args.question)
|
|
21595
21608
|
);
|
|
@@ -21622,25 +21635,90 @@ function prioritizeLoCoMoRecallText(args) {
|
|
|
21622
21635
|
)
|
|
21623
21636
|
];
|
|
21624
21637
|
if (direct.length === 0 && linkedHops.length === 0) {
|
|
21625
|
-
|
|
21638
|
+
const { text: text2 } = truncateLoCoMoContext(
|
|
21626
21639
|
args.recalledText,
|
|
21627
21640
|
LOCOMO_FALLBACK_CONTEXT_MAX_CHARS
|
|
21628
21641
|
);
|
|
21642
|
+
return {
|
|
21643
|
+
text: text2,
|
|
21644
|
+
receipt: {
|
|
21645
|
+
schemaVersion: 1,
|
|
21646
|
+
mode: "fallback",
|
|
21647
|
+
multiHopRecallComposition: args.multiHopRecallComposition,
|
|
21648
|
+
input: digestLoCoMoContent(args.recalledText),
|
|
21649
|
+
output: digestLoCoMoContent(text2),
|
|
21650
|
+
selectedLines: []
|
|
21651
|
+
}
|
|
21652
|
+
};
|
|
21629
21653
|
}
|
|
21630
|
-
const sections = [
|
|
21631
|
-
|
|
21632
|
-
|
|
21633
|
-
|
|
21654
|
+
const sections = ["## LoCoMo Question-Focused Evidence"];
|
|
21655
|
+
const selectedRanges = [];
|
|
21656
|
+
const appendSelectedLine = (input, stage, hop) => {
|
|
21657
|
+
const output = truncateLoCoMoLine(input);
|
|
21658
|
+
const inputOrdinal = inputOrdinalByLine.get(input);
|
|
21659
|
+
if (inputOrdinal === void 0) {
|
|
21660
|
+
throw new Error("LoCoMo composition selected a line outside its normalized input.");
|
|
21661
|
+
}
|
|
21662
|
+
const outputStart = sections.join("\n").length + 1;
|
|
21663
|
+
sections.push(output);
|
|
21664
|
+
selectedRanges.push({
|
|
21665
|
+
input,
|
|
21666
|
+
output,
|
|
21667
|
+
inputOrdinal,
|
|
21668
|
+
stage,
|
|
21669
|
+
...hop === void 0 ? {} : { hop },
|
|
21670
|
+
outputStart,
|
|
21671
|
+
outputEnd: outputStart + output.length
|
|
21672
|
+
});
|
|
21673
|
+
};
|
|
21674
|
+
for (const entry of direct) appendSelectedLine(entry.line, "direct");
|
|
21634
21675
|
for (const hop of linkedHops) {
|
|
21635
|
-
sections.push(
|
|
21636
|
-
|
|
21637
|
-
...hop.lines.map(truncateLoCoMoLine)
|
|
21638
|
-
);
|
|
21676
|
+
sections.push(`## LoCoMo Linked Evidence (hop ${hop.hop})`);
|
|
21677
|
+
for (const line of hop.lines) appendSelectedLine(line, "linked", hop.hop);
|
|
21639
21678
|
}
|
|
21640
|
-
|
|
21679
|
+
const truncation = truncateLoCoMoContext(
|
|
21641
21680
|
sections.join("\n"),
|
|
21642
21681
|
LOCOMO_FOCUSED_CONTEXT_MAX_CHARS
|
|
21643
21682
|
);
|
|
21683
|
+
const { text, safePrefixEnd } = truncation;
|
|
21684
|
+
const selectedLines = selectedRanges.map((entry) => {
|
|
21685
|
+
const visibleStart = Math.min(entry.outputStart, safePrefixEnd);
|
|
21686
|
+
const visibleEnd = Math.min(entry.outputEnd, safePrefixEnd);
|
|
21687
|
+
const visible = visibleEnd - visibleStart === entry.output.length;
|
|
21688
|
+
return buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd);
|
|
21689
|
+
});
|
|
21690
|
+
return {
|
|
21691
|
+
text,
|
|
21692
|
+
receipt: {
|
|
21693
|
+
schemaVersion: 1,
|
|
21694
|
+
mode: "focused",
|
|
21695
|
+
multiHopRecallComposition: args.multiHopRecallComposition,
|
|
21696
|
+
input: digestLoCoMoContent(args.recalledText),
|
|
21697
|
+
output: digestLoCoMoContent(text),
|
|
21698
|
+
selectedLines
|
|
21699
|
+
}
|
|
21700
|
+
};
|
|
21701
|
+
}
|
|
21702
|
+
function buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd) {
|
|
21703
|
+
return {
|
|
21704
|
+
inputOrdinal: entry.inputOrdinal,
|
|
21705
|
+
input: digestLoCoMoContent(entry.input),
|
|
21706
|
+
output: digestLoCoMoContent(entry.output),
|
|
21707
|
+
stage: entry.stage,
|
|
21708
|
+
...entry.hop === void 0 ? {} : { hop: entry.hop },
|
|
21709
|
+
visible,
|
|
21710
|
+
outputStart: entry.outputStart,
|
|
21711
|
+
outputEnd: entry.outputEnd,
|
|
21712
|
+
visibleStart,
|
|
21713
|
+
visibleEnd
|
|
21714
|
+
};
|
|
21715
|
+
}
|
|
21716
|
+
function digestLoCoMoContent(value) {
|
|
21717
|
+
return {
|
|
21718
|
+
sha256: hashString(value),
|
|
21719
|
+
charCount: value.length,
|
|
21720
|
+
lineCount: value.length === 0 ? 0 : value.split("\n").length
|
|
21721
|
+
};
|
|
21644
21722
|
}
|
|
21645
21723
|
function composeLoCoMoLinkedEvidence(args) {
|
|
21646
21724
|
if (args.direct.length === 0 || args.remainingLineBudget <= 0) {
|
|
@@ -21898,13 +21976,16 @@ function truncateLoCoMoLine(line) {
|
|
|
21898
21976
|
}
|
|
21899
21977
|
function truncateLoCoMoContext(text, maxChars) {
|
|
21900
21978
|
if (text.length <= maxChars) {
|
|
21901
|
-
return text;
|
|
21979
|
+
return { text, safePrefixEnd: text.length };
|
|
21902
21980
|
}
|
|
21903
21981
|
const truncated = text.slice(0, maxChars);
|
|
21904
21982
|
const lastNewline = truncated.lastIndexOf("\n");
|
|
21905
21983
|
const safePrefix = lastNewline > 0 ? truncated.slice(0, lastNewline) : truncated;
|
|
21906
|
-
return
|
|
21907
|
-
|
|
21984
|
+
return {
|
|
21985
|
+
text: `${safePrefix}
|
|
21986
|
+
[LoCoMo context truncated to ${maxChars} characters]`,
|
|
21987
|
+
safePrefixEnd: safePrefix.length
|
|
21988
|
+
};
|
|
21908
21989
|
}
|
|
21909
21990
|
function countHiddenEvidenceIdsInRecall(evidence, question, recalledText) {
|
|
21910
21991
|
const queryVisibleIds = collectDialogueIds(question);
|
|
@@ -21925,7 +22006,7 @@ function collectDialogueIds(text) {
|
|
|
21925
22006
|
function escapeRegExp2(value) {
|
|
21926
22007
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
21927
22008
|
}
|
|
21928
|
-
async function
|
|
22009
|
+
async function loadLoCoMoDataset(mode, datasetDir, limit) {
|
|
21929
22010
|
const loaded = await loadLoCoMo10({
|
|
21930
22011
|
mode,
|
|
21931
22012
|
datasetDir,
|
|
@@ -21957,7 +22038,15 @@ async function loadDataset6(mode, datasetDir, limit) {
|
|
|
21957
22038
|
"[remnic-bench] LoCoMo falling back to smoke fixture: " + loaded.errors.join(" | ")
|
|
21958
22039
|
);
|
|
21959
22040
|
}
|
|
21960
|
-
|
|
22041
|
+
if (!loaded.sha256) {
|
|
22042
|
+
throw new Error("LoCoMo dataset loader did not provide a content hash.");
|
|
22043
|
+
}
|
|
22044
|
+
return {
|
|
22045
|
+
source: loaded.source,
|
|
22046
|
+
...loaded.filename === void 0 ? {} : { filename: loaded.filename },
|
|
22047
|
+
sha256: loaded.sha256,
|
|
22048
|
+
items: loaded.items
|
|
22049
|
+
};
|
|
21961
22050
|
}
|
|
21962
22051
|
function parseDataset2(raw, filename) {
|
|
21963
22052
|
const parsed = JSON.parse(raw);
|
|
@@ -22241,7 +22330,7 @@ async function loadBeamDatasetPreview(options) {
|
|
|
22241
22330
|
}
|
|
22242
22331
|
let dataset;
|
|
22243
22332
|
try {
|
|
22244
|
-
dataset = await
|
|
22333
|
+
dataset = await loadDataset6(
|
|
22245
22334
|
options.mode === "quick" ? "quick" : "full",
|
|
22246
22335
|
options.datasetDir,
|
|
22247
22336
|
options.limit
|
|
@@ -22280,7 +22369,7 @@ async function loadBeamDatasetPreview(options) {
|
|
|
22280
22369
|
};
|
|
22281
22370
|
}
|
|
22282
22371
|
async function runBeamBenchmark(options) {
|
|
22283
|
-
const dataset = await
|
|
22372
|
+
const dataset = await loadDataset6(options.mode, options.datasetDir, options.limit);
|
|
22284
22373
|
const tasks = [];
|
|
22285
22374
|
const taskFilter = normalizeBeamTaskFilter(
|
|
22286
22375
|
options.benchmarkOptions?.taskFilter
|
|
@@ -22467,7 +22556,7 @@ async function runBeamBenchmark(options) {
|
|
|
22467
22556
|
}
|
|
22468
22557
|
};
|
|
22469
22558
|
}
|
|
22470
|
-
async function
|
|
22559
|
+
async function loadDataset6(mode, datasetDir, limit) {
|
|
22471
22560
|
const normalizedLimit = normalizeLimit5(limit);
|
|
22472
22561
|
const ensureDatasetEntries = (entryCount) => {
|
|
22473
22562
|
if (entryCount === 0) {
|
|
@@ -23653,7 +23742,7 @@ var personaMemDefinition = {
|
|
|
23653
23742
|
}
|
|
23654
23743
|
};
|
|
23655
23744
|
async function runPersonaMemBenchmark(options) {
|
|
23656
|
-
const samples = await
|
|
23745
|
+
const samples = await loadDataset7(options.mode, options.datasetDir, options.limit);
|
|
23657
23746
|
const tasks = [];
|
|
23658
23747
|
const totalTasks = samples.length;
|
|
23659
23748
|
for (let sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
|
|
@@ -23833,7 +23922,7 @@ async function runPersonaMemBenchmark(options) {
|
|
|
23833
23922
|
}
|
|
23834
23923
|
};
|
|
23835
23924
|
}
|
|
23836
|
-
async function
|
|
23925
|
+
async function loadDataset7(mode, datasetDir, limit) {
|
|
23837
23926
|
const normalizedLimit = normalizeLimit6(limit);
|
|
23838
23927
|
const ensureDatasetSamples = (samples) => {
|
|
23839
23928
|
if (samples.length === 0) {
|
|
@@ -24441,7 +24530,7 @@ var memBenchDefinition = {
|
|
|
24441
24530
|
}
|
|
24442
24531
|
};
|
|
24443
24532
|
async function runMemBenchBenchmark(options) {
|
|
24444
|
-
const dataset = await
|
|
24533
|
+
const dataset = await loadDataset8(options.mode, options.datasetDir, options.limit);
|
|
24445
24534
|
const tasks = [];
|
|
24446
24535
|
const totalTasks = dataset.length;
|
|
24447
24536
|
for (const testCase of dataset) {
|
|
@@ -24615,7 +24704,7 @@ async function runMemBenchBenchmark(options) {
|
|
|
24615
24704
|
}
|
|
24616
24705
|
};
|
|
24617
24706
|
}
|
|
24618
|
-
async function
|
|
24707
|
+
async function loadDataset8(mode, datasetDir, limit) {
|
|
24619
24708
|
const normalizedLimit = normalizeLimit7(limit);
|
|
24620
24709
|
const ensureDatasetCases = (cases) => {
|
|
24621
24710
|
if (cases.length === 0) {
|
|
@@ -25646,7 +25735,7 @@ var memoryAgentBenchDefinition = {
|
|
|
25646
25735
|
}
|
|
25647
25736
|
};
|
|
25648
25737
|
async function runMemoryAgentBenchBenchmark(options) {
|
|
25649
|
-
const rawDataset = await
|
|
25738
|
+
const rawDataset = await loadDataset9(options.mode, options.datasetDir, options.limit);
|
|
25650
25739
|
const trialLimit = resolveTrialLimit2(options.benchmarkOptions?.trialLimit);
|
|
25651
25740
|
const benchmarkOptions = trialLimit === void 0 ? options.benchmarkOptions : { ...options.benchmarkOptions ?? {}, trialLimit };
|
|
25652
25741
|
const dataset = applyTrialLimit2(rawDataset, trialLimit);
|
|
@@ -26365,11 +26454,11 @@ function officialF1(prediction, groundTruth) {
|
|
|
26365
26454
|
return 2 * precision * recall / (precision + recall);
|
|
26366
26455
|
}
|
|
26367
26456
|
function countTokens(tokens) {
|
|
26368
|
-
const
|
|
26457
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
26369
26458
|
for (const token of tokens) {
|
|
26370
|
-
|
|
26459
|
+
counts2.set(token, (counts2.get(token) ?? 0) + 1);
|
|
26371
26460
|
}
|
|
26372
|
-
return
|
|
26461
|
+
return counts2;
|
|
26373
26462
|
}
|
|
26374
26463
|
function extractRecommendationMovies(output, movieCandidates, aliasCounts) {
|
|
26375
26464
|
let recommendationText = output;
|
|
@@ -26435,14 +26524,14 @@ function stripTrailingRecommendationPunctuation(value) {
|
|
|
26435
26524
|
return value.replace(/^["'`]+/g, "").replace(/["'`.!?;:]+$/g, "").trim();
|
|
26436
26525
|
}
|
|
26437
26526
|
function countMovieAliases(movieCandidates) {
|
|
26438
|
-
const
|
|
26527
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
26439
26528
|
for (const movie of movieCandidates) {
|
|
26440
26529
|
for (const alias of movieAliases(movie)) {
|
|
26441
26530
|
const normalizedAlias = alias.toLowerCase();
|
|
26442
|
-
|
|
26531
|
+
counts2.set(normalizedAlias, (counts2.get(normalizedAlias) ?? 0) + 1);
|
|
26443
26532
|
}
|
|
26444
26533
|
}
|
|
26445
|
-
return
|
|
26534
|
+
return counts2;
|
|
26446
26535
|
}
|
|
26447
26536
|
function movieAliases(movie) {
|
|
26448
26537
|
const aliases = [movie];
|
|
@@ -26619,7 +26708,7 @@ function decodeUrlComponentSafely(value) {
|
|
|
26619
26708
|
return value;
|
|
26620
26709
|
}
|
|
26621
26710
|
}
|
|
26622
|
-
async function
|
|
26711
|
+
async function loadDataset9(mode, datasetDir, limit) {
|
|
26623
26712
|
const normalizedLimit = normalizeLimit8(limit);
|
|
26624
26713
|
const ensureDatasetItems = (items) => {
|
|
26625
26714
|
if (items.length === 0) {
|
|
@@ -35522,10 +35611,10 @@ function uptakeLatency(log, corrections, cap) {
|
|
|
35522
35611
|
let resolved = cap;
|
|
35523
35612
|
let found = false;
|
|
35524
35613
|
for (const entry of post) {
|
|
35525
|
-
const
|
|
35526
|
-
if (
|
|
35614
|
+
const delta2 = entry.turnIndex - correction.turnIndex;
|
|
35615
|
+
if (delta2 > cap) break;
|
|
35527
35616
|
if (probePassesForCorrection(entry, correction)) {
|
|
35528
|
-
resolved =
|
|
35617
|
+
resolved = delta2;
|
|
35529
35618
|
found = true;
|
|
35530
35619
|
break;
|
|
35531
35620
|
}
|
|
@@ -38369,11 +38458,11 @@ function compareResults(baseline, candidate, threshold = 0.05, lowerIsBetter = /
|
|
|
38369
38458
|
candidate,
|
|
38370
38459
|
metricName
|
|
38371
38460
|
);
|
|
38372
|
-
const
|
|
38461
|
+
const delta2 = aggregate.mean - baselineAggregate.mean;
|
|
38373
38462
|
const metricDelta = {
|
|
38374
38463
|
baseline: baselineAggregate.mean,
|
|
38375
38464
|
candidate: aggregate.mean,
|
|
38376
|
-
delta,
|
|
38465
|
+
delta: delta2,
|
|
38377
38466
|
percentChange: percentChange(aggregate.mean, baselineAggregate.mean),
|
|
38378
38467
|
effectSize: {
|
|
38379
38468
|
cohensD: 0,
|
|
@@ -39194,6 +39283,1146 @@ function formatSignedScore2(value) {
|
|
|
39194
39283
|
return `${value >= 0 ? "+" : ""}${formatScore2(value)}`;
|
|
39195
39284
|
}
|
|
39196
39285
|
|
|
39286
|
+
// src/stats/locomo-retrieval-trace-delta.ts
|
|
39287
|
+
var LOCOMO_RETRIEVAL_TRACE_DELTA_SCHEMA_VERSION = 1;
|
|
39288
|
+
var CATEGORIES = ["single_hop", "multi_hop", "temporal", "open_domain", "adversarial"];
|
|
39289
|
+
var MECHANISMS = [
|
|
39290
|
+
"real-core-visible-lcm-displacement",
|
|
39291
|
+
"lcm-selection-change",
|
|
39292
|
+
"composition-filter-displacement",
|
|
39293
|
+
"composition-digest-change",
|
|
39294
|
+
"budget-truncation-change",
|
|
39295
|
+
"mixed",
|
|
39296
|
+
"no-structural-delta",
|
|
39297
|
+
"insufficient-exact-lineage"
|
|
39298
|
+
];
|
|
39299
|
+
function diagnoseLoCoMoRetrievalTraceDelta(baseline, real) {
|
|
39300
|
+
assertReceipt(baseline, "baseline");
|
|
39301
|
+
assertReceipt(real, "real");
|
|
39302
|
+
assertComparable(baseline, real);
|
|
39303
|
+
const tasks = baseline.tasks.map(
|
|
39304
|
+
(baselineTask, index) => compareTask(baselineTask, real.tasks[index])
|
|
39305
|
+
);
|
|
39306
|
+
const overall = summarize(tasks);
|
|
39307
|
+
const categories = CATEGORIES.filter((category) => tasks.some((task) => task.category === category)).map(
|
|
39308
|
+
(category) => ({ category, ...summarize(tasks.filter((task) => task.category === category)) })
|
|
39309
|
+
);
|
|
39310
|
+
const multiHop = tasks.filter((task) => task.category === "multi_hop");
|
|
39311
|
+
const candidates = MECHANISMS.filter(
|
|
39312
|
+
(mechanism) => mechanism !== "no-structural-delta" && mechanism !== "insufficient-exact-lineage"
|
|
39313
|
+
).map((mechanism) => ({ mechanism, count: multiHop.filter((task) => task.mechanism === mechanism).length }));
|
|
39314
|
+
candidates.sort((left, right) => right.count - left.count || left.mechanism.localeCompare(right.mechanism));
|
|
39315
|
+
const dominant = candidates[0];
|
|
39316
|
+
const supported = dominant !== void 0 && dominant.count >= 2 && dominant.count * 2 > multiHop.length;
|
|
39317
|
+
const withoutHash = {
|
|
39318
|
+
schemaVersion: LOCOMO_RETRIEVAL_TRACE_DELTA_SCHEMA_VERSION,
|
|
39319
|
+
benchmarkId: "locomo",
|
|
39320
|
+
analysisKind: "paired-retrieval-structural-delta",
|
|
39321
|
+
sensitivity: {
|
|
39322
|
+
classification: "restricted",
|
|
39323
|
+
contentEncoding: "sha256+length",
|
|
39324
|
+
containsGold: false,
|
|
39325
|
+
containsRawContent: false,
|
|
39326
|
+
containsRawIdentifiers: false
|
|
39327
|
+
},
|
|
39328
|
+
comparison: {
|
|
39329
|
+
baselineArtifactHash: baseline.artifactHash,
|
|
39330
|
+
realArtifactHash: real.artifactHash,
|
|
39331
|
+
retrievalConfigHashesDiffer: true,
|
|
39332
|
+
taskOrderSha256: hashCanonicalJson(baseline.tasks.map((task) => digestIdentifier(task.taskId)))
|
|
39333
|
+
},
|
|
39334
|
+
overall,
|
|
39335
|
+
categories,
|
|
39336
|
+
dominantMultiHopMechanism: {
|
|
39337
|
+
status: supported ? "supported" : "not-supported",
|
|
39338
|
+
...supported && dominant ? { mechanism: dominant.mechanism } : {},
|
|
39339
|
+
count: dominant?.count ?? 0,
|
|
39340
|
+
taskCount: multiHop.length,
|
|
39341
|
+
rule: "strict-majority-and-at-least-two"
|
|
39342
|
+
},
|
|
39343
|
+
tasks,
|
|
39344
|
+
evidenceBoundary: {
|
|
39345
|
+
attribution: "observed-structural-mechanism-only",
|
|
39346
|
+
causalClaim: false,
|
|
39347
|
+
exactLineageRequired: true,
|
|
39348
|
+
explanation: "Labels summarize paired, content-free structural differences. The budget-truncation label denotes a fixed-budget tail-geometry transition with a stable recorded composition outcome; it does not prove identical prefix content. No label proves that a retrieval mechanism caused an answer or score change."
|
|
39349
|
+
}
|
|
39350
|
+
};
|
|
39351
|
+
return { ...withoutHash, artifactHash: hashCanonicalJson(withoutHash) };
|
|
39352
|
+
}
|
|
39353
|
+
function serializeLoCoMoRetrievalTraceDelta(report) {
|
|
39354
|
+
return `${canonicalJsonStringify(report, 2)}
|
|
39355
|
+
`;
|
|
39356
|
+
}
|
|
39357
|
+
function compareTask(baseline, real) {
|
|
39358
|
+
const category = categoryOf(baseline.taskId);
|
|
39359
|
+
const dimensions = {
|
|
39360
|
+
sectionVisibleChars: delta(signatures(baseline, "sectionVisibleChars"), signatures(real, "sectionVisibleChars")),
|
|
39361
|
+
selections: delta(signatures(baseline, "selections"), signatures(real, "selections")),
|
|
39362
|
+
archiveRows: delta(signatures(baseline, "archiveRows"), signatures(real, "archiveRows")),
|
|
39363
|
+
lcmCandidates: delta(signatures(baseline, "lcmCandidates"), signatures(real, "lcmCandidates")),
|
|
39364
|
+
coreResults: delta(signatures(baseline, "coreResults"), signatures(real, "coreResults")),
|
|
39365
|
+
coreFilters: delta(signatures(baseline, "coreFilters"), signatures(real, "coreFilters")),
|
|
39366
|
+
coreBudget: delta(signatures(baseline, "coreBudget"), signatures(real, "coreBudget")),
|
|
39367
|
+
recallBudget: delta(signatures(baseline, "recallBudget"), signatures(real, "recallBudget")),
|
|
39368
|
+
compositionPolicy: delta(signatures(baseline, "compositionPolicy"), signatures(real, "compositionPolicy")),
|
|
39369
|
+
compositionDigests: delta(signatures(baseline, "compositionDigests"), signatures(real, "compositionDigests"))
|
|
39370
|
+
};
|
|
39371
|
+
const hasCompleteExactLineage = (task) => {
|
|
39372
|
+
return task.sessions.every((session) => {
|
|
39373
|
+
const renderedSections = session.trace.sections.filter(
|
|
39374
|
+
(section) => section.source === "lcm-summary" || section.source === "raw-row"
|
|
39375
|
+
);
|
|
39376
|
+
const lcmSelections = session.trace.selections.filter(
|
|
39377
|
+
(selection) => selection.kind === "lcm-summary" || selection.kind === "raw-row"
|
|
39378
|
+
);
|
|
39379
|
+
const selectionMatchesSection = (selection) => renderedSections.some((section) => section.id === selection.sectionId && section.source === selection.kind);
|
|
39380
|
+
const renderedLineageComplete = renderedSections.every(
|
|
39381
|
+
(section) => lcmSelections.some(
|
|
39382
|
+
(selection) => selection.sectionId === section.id && selection.kind === section.source && hasExactSelectionLineage(selection)
|
|
39383
|
+
)
|
|
39384
|
+
);
|
|
39385
|
+
const candidateLineageComplete = session.trace.lcmCandidates.every(hasExactCandidateLineage);
|
|
39386
|
+
const hasCarrier = renderedSections.length > 0 && renderedLineageComplete || renderedSections.length === 0 && lcmSelections.length === 0 && session.trace.lcmCandidates.length > 0 && candidateLineageComplete;
|
|
39387
|
+
const hasLcmEvidence = renderedSections.length > 0 || lcmSelections.length > 0 || session.trace.lcmCandidates.length > 0;
|
|
39388
|
+
return (!hasLcmEvidence || hasCarrier) && lcmSelections.every(selectionMatchesSection) && lcmSelections.every(hasExactSelectionLineage) && (renderedSections.length > 0 || candidateLineageComplete) && (session.trace.coreCapture?.results.every(
|
|
39389
|
+
(result) => isSha256Hex(result.memoryIdRef.sha256) && result.memoryIdRef.length > 0
|
|
39390
|
+
) ?? true);
|
|
39391
|
+
});
|
|
39392
|
+
};
|
|
39393
|
+
const exact = hasCompleteExactLineage(baseline) && hasCompleteExactLineage(real);
|
|
39394
|
+
const lcmSelectionsChanged = delta(
|
|
39395
|
+
exactSelectionSignatures(baseline, "lcm", "emitted"),
|
|
39396
|
+
exactSelectionSignatures(real, "lcm", "emitted")
|
|
39397
|
+
).changed;
|
|
39398
|
+
const lcmArchiveRowsChanged = delta(
|
|
39399
|
+
exactArchiveRowSignatures(baseline, "lcm"),
|
|
39400
|
+
exactArchiveRowSignatures(real, "lcm")
|
|
39401
|
+
).changed;
|
|
39402
|
+
const auxiliarySelectionsChanged = delta(
|
|
39403
|
+
exactSelectionSignatures(baseline, "auxiliary", "emitted"),
|
|
39404
|
+
exactSelectionSignatures(real, "auxiliary", "emitted")
|
|
39405
|
+
).changed;
|
|
39406
|
+
const auxiliaryArchiveRowsChanged = delta(
|
|
39407
|
+
exactArchiveRowSignatures(baseline, "auxiliary"),
|
|
39408
|
+
exactArchiveRowSignatures(real, "auxiliary")
|
|
39409
|
+
).changed;
|
|
39410
|
+
const core = dimensions.coreResults.changed || dimensions.coreFilters.changed || dimensions.coreBudget.changed;
|
|
39411
|
+
const baselineVisible = visibleCharsByGroup(baseline);
|
|
39412
|
+
const realVisible = visibleCharsByGroup(real);
|
|
39413
|
+
const coreSections = sectionGroupChanged(baseline, real, "core", "emitted");
|
|
39414
|
+
const lcmSections = sectionGroupChanged(baseline, real, "lcm", "emitted");
|
|
39415
|
+
const otherSections = sectionGroupChanged(baseline, real, "other", "emitted");
|
|
39416
|
+
const lcm = lcmSections || lcmSelectionsChanged || lcmArchiveRowsChanged || dimensions.lcmCandidates.changed;
|
|
39417
|
+
const auxiliary = auxiliarySelectionsChanged || auxiliaryArchiveRowsChanged;
|
|
39418
|
+
const other = otherSections || auxiliary;
|
|
39419
|
+
const composedSelectionsChanged = delta(
|
|
39420
|
+
structuralSelectionSignatures(baseline, "all", "composed", false),
|
|
39421
|
+
structuralSelectionSignatures(real, "all", "composed", false)
|
|
39422
|
+
).changed;
|
|
39423
|
+
const allCandidatesChanged = delta(candidateSignatures(baseline, false), candidateSignatures(real, false)).changed;
|
|
39424
|
+
const compositionPolicy = dimensions.compositionPolicy.changed;
|
|
39425
|
+
const compositionDigests = dimensions.compositionDigests.changed;
|
|
39426
|
+
const compositionOutcomeChanged = compositionOutcomeSignature(baseline) !== compositionOutcomeSignature(real);
|
|
39427
|
+
const budget = dimensions.recallBudget.changed;
|
|
39428
|
+
const coreVisibleLcmDisplacement = core && lcm && realVisible.core > baselineVisible.core && realVisible.lcm < baselineVisible.lcm && !other && !budget && !compositionPolicy;
|
|
39429
|
+
const budgetTailGeometryTransition = budget && hasFixedBudgetTailGeometryTransition(baseline, real) && !core && !composedSelectionsChanged && !allCandidatesChanged && !compositionPolicy && !compositionOutcomeChanged;
|
|
39430
|
+
let mechanism;
|
|
39431
|
+
if (!exact) mechanism = "insufficient-exact-lineage";
|
|
39432
|
+
else if (coreVisibleLcmDisplacement) mechanism = "real-core-visible-lcm-displacement";
|
|
39433
|
+
else if (lcm && !core && !coreSections && !other && !budget && !compositionPolicy)
|
|
39434
|
+
mechanism = "lcm-selection-change";
|
|
39435
|
+
else if (budgetTailGeometryTransition) mechanism = "budget-truncation-change";
|
|
39436
|
+
else if (compositionPolicy && !core && !coreSections && !lcm && !other && !budget)
|
|
39437
|
+
mechanism = "composition-filter-displacement";
|
|
39438
|
+
else if (compositionDigests && !compositionPolicy && !core && !coreSections && !lcm && !other && !budget)
|
|
39439
|
+
mechanism = "composition-digest-change";
|
|
39440
|
+
else if (!core && !coreSections && !lcm && !other && !budget && !compositionPolicy && !compositionDigests)
|
|
39441
|
+
mechanism = "no-structural-delta";
|
|
39442
|
+
else mechanism = "mixed";
|
|
39443
|
+
return { taskRef: digestIdentifier(baseline.taskId), category, mechanism, dimensions };
|
|
39444
|
+
}
|
|
39445
|
+
function visibleCharsByGroup(task) {
|
|
39446
|
+
const output = { core: 0, lcm: 0, other: 0 };
|
|
39447
|
+
for (const session of task.sessions) {
|
|
39448
|
+
for (const section of session.trace.sections) {
|
|
39449
|
+
if (section.source === "core") {
|
|
39450
|
+
output.core += section.visibleChars;
|
|
39451
|
+
} else if (section.source === "lcm-summary" || section.source === "raw-row") {
|
|
39452
|
+
output.lcm += section.visibleChars;
|
|
39453
|
+
} else {
|
|
39454
|
+
output.other += section.visibleChars;
|
|
39455
|
+
}
|
|
39456
|
+
}
|
|
39457
|
+
}
|
|
39458
|
+
return output;
|
|
39459
|
+
}
|
|
39460
|
+
function sectionGroupChanged(baseline, real, group, projection) {
|
|
39461
|
+
const signaturesFor = (task) => {
|
|
39462
|
+
const output = [];
|
|
39463
|
+
task.sessions.forEach((session, sessionOrdinal) => {
|
|
39464
|
+
session.trace.sections.forEach((section, sectionOrdinal) => {
|
|
39465
|
+
if (sectionGroup(section.source) === group) {
|
|
39466
|
+
output.push(structuralSectionSignature(section, sessionOrdinal, sectionOrdinal, projection));
|
|
39467
|
+
}
|
|
39468
|
+
});
|
|
39469
|
+
});
|
|
39470
|
+
return output;
|
|
39471
|
+
};
|
|
39472
|
+
return delta(signaturesFor(baseline), signaturesFor(real)).changed;
|
|
39473
|
+
}
|
|
39474
|
+
function sectionGroup(source) {
|
|
39475
|
+
if (source === "core") return "core";
|
|
39476
|
+
if (source === "lcm-summary" || source === "raw-row") return "lcm";
|
|
39477
|
+
return "other";
|
|
39478
|
+
}
|
|
39479
|
+
function structuralSectionSignature(section, sessionOrdinal, sectionOrdinal, projection) {
|
|
39480
|
+
return hashCanonicalJson({
|
|
39481
|
+
sessionOrdinal,
|
|
39482
|
+
sectionOrdinal,
|
|
39483
|
+
sectionIdRef: digestIdentifier(section.id),
|
|
39484
|
+
source: section.source,
|
|
39485
|
+
separatorStart: section.separatorStart,
|
|
39486
|
+
contentStart: section.contentStart,
|
|
39487
|
+
contentEnd: section.contentEnd,
|
|
39488
|
+
composedStart: section.composedStart,
|
|
39489
|
+
composedEnd: section.composedEnd,
|
|
39490
|
+
...projection === "emitted" ? {
|
|
39491
|
+
visibleStart: section.visibleStart,
|
|
39492
|
+
visibleEnd: section.visibleEnd,
|
|
39493
|
+
visibleChars: section.visibleChars
|
|
39494
|
+
} : {}
|
|
39495
|
+
});
|
|
39496
|
+
}
|
|
39497
|
+
function selectionInScope(selection, scope) {
|
|
39498
|
+
const isLcm = selection.kind === "lcm-summary" || selection.kind === "raw-row";
|
|
39499
|
+
return scope === "all" || (scope === "lcm" ? isLcm : !isLcm);
|
|
39500
|
+
}
|
|
39501
|
+
function exactSelectionSignatures(task, scope, projection) {
|
|
39502
|
+
return structuralSelectionSignatures(task, scope, projection, true);
|
|
39503
|
+
}
|
|
39504
|
+
function structuralSelectionSignatures(task, scope, projection, exactOnly) {
|
|
39505
|
+
const output = [];
|
|
39506
|
+
task.sessions.forEach((session, sessionOrdinal) => {
|
|
39507
|
+
for (const value of session.trace.selections) {
|
|
39508
|
+
if (!selectionInScope(value, scope) || exactOnly && !hasExactSelectionLineage(value)) continue;
|
|
39509
|
+
output.push(
|
|
39510
|
+
hashCanonicalJson({
|
|
39511
|
+
sessionOrdinal,
|
|
39512
|
+
sectionIdRef: digestIdentifier(value.sectionId),
|
|
39513
|
+
kind: value.kind,
|
|
39514
|
+
lineageStatus: value.lineageStatus,
|
|
39515
|
+
turnIndex: value.turnIndex,
|
|
39516
|
+
role: value.role,
|
|
39517
|
+
score: value.score,
|
|
39518
|
+
summary: value.summary,
|
|
39519
|
+
archiveRowIds: value.archiveRowIds,
|
|
39520
|
+
composedStart: value.composedStart,
|
|
39521
|
+
composedEnd: value.composedEnd,
|
|
39522
|
+
...projection === "emitted" ? {
|
|
39523
|
+
visibleStart: value.visibleStart,
|
|
39524
|
+
visibleEnd: value.visibleEnd
|
|
39525
|
+
} : {}
|
|
39526
|
+
})
|
|
39527
|
+
);
|
|
39528
|
+
}
|
|
39529
|
+
});
|
|
39530
|
+
return output;
|
|
39531
|
+
}
|
|
39532
|
+
function exactArchiveRowSignatures(task, scope) {
|
|
39533
|
+
const output = [];
|
|
39534
|
+
task.sessions.forEach((session, sessionOrdinal) => {
|
|
39535
|
+
for (const value of session.trace.selections) {
|
|
39536
|
+
if (!selectionInScope(value, scope) || !hasExactSelectionLineage(value)) continue;
|
|
39537
|
+
for (const archiveRowId of value.archiveRowIds ?? []) {
|
|
39538
|
+
output.push(hashCanonicalJson({ sessionOrdinal, archiveRowId }));
|
|
39539
|
+
}
|
|
39540
|
+
}
|
|
39541
|
+
});
|
|
39542
|
+
return output;
|
|
39543
|
+
}
|
|
39544
|
+
function hasExactSelectionLineage(selection) {
|
|
39545
|
+
if (selection.lineageStatus !== "exact") return false;
|
|
39546
|
+
if (selection.kind === "lcm-summary") {
|
|
39547
|
+
return selection.summary !== void 0 && Number.isSafeInteger(selection.summary.depth) && selection.summary.depth >= 0 && Number.isSafeInteger(selection.summary.msgStart) && selection.summary.msgStart >= 0 && Number.isSafeInteger(selection.summary.msgEnd) && selection.summary.msgEnd >= selection.summary.msgStart;
|
|
39548
|
+
}
|
|
39549
|
+
return Array.isArray(selection.archiveRowIds) && selection.archiveRowIds.length > 0 && selection.archiveRowIds.every((id) => Number.isSafeInteger(id) && id > 0);
|
|
39550
|
+
}
|
|
39551
|
+
function hasExactCandidateLineage(candidate) {
|
|
39552
|
+
return candidate.lineageStatus === "exact" && Number.isSafeInteger(candidate.archiveRowId) && candidate.archiveRowId > 0;
|
|
39553
|
+
}
|
|
39554
|
+
function candidateSignatures(task, exactOnly) {
|
|
39555
|
+
const output = [];
|
|
39556
|
+
task.sessions.forEach((session, sessionOrdinal) => {
|
|
39557
|
+
for (const value of session.trace.lcmCandidates) {
|
|
39558
|
+
if (exactOnly && !hasExactCandidateLineage(value)) continue;
|
|
39559
|
+
output.push(
|
|
39560
|
+
hashCanonicalJson({
|
|
39561
|
+
sessionOrdinal,
|
|
39562
|
+
rank: value.rank,
|
|
39563
|
+
archiveRowId: value.archiveRowId,
|
|
39564
|
+
turnIndex: value.turnIndex,
|
|
39565
|
+
role: value.role,
|
|
39566
|
+
score: value.score,
|
|
39567
|
+
lineageStatus: value.lineageStatus
|
|
39568
|
+
})
|
|
39569
|
+
);
|
|
39570
|
+
}
|
|
39571
|
+
});
|
|
39572
|
+
return output;
|
|
39573
|
+
}
|
|
39574
|
+
function signatures(task, dimension) {
|
|
39575
|
+
if (dimension === "selections") return exactSelectionSignatures(task, "all", "emitted");
|
|
39576
|
+
if (dimension === "archiveRows") return exactArchiveRowSignatures(task, "all");
|
|
39577
|
+
if (dimension === "lcmCandidates") return candidateSignatures(task, true);
|
|
39578
|
+
if (dimension === "recallBudget") {
|
|
39579
|
+
return [
|
|
39580
|
+
hashCanonicalJson({
|
|
39581
|
+
recallBudgetChars: task.recallBudgetChars,
|
|
39582
|
+
budgets: task.sessions.map((session) => ({
|
|
39583
|
+
requestedChars: session.trace.budget.requestedChars,
|
|
39584
|
+
truncated: session.trace.budget.truncated
|
|
39585
|
+
}))
|
|
39586
|
+
})
|
|
39587
|
+
];
|
|
39588
|
+
}
|
|
39589
|
+
if (dimension === "compositionPolicy") {
|
|
39590
|
+
return [
|
|
39591
|
+
hashCanonicalJson({
|
|
39592
|
+
mode: task.composition.mode,
|
|
39593
|
+
multiHopRecallComposition: task.composition.multiHopRecallComposition,
|
|
39594
|
+
selectedLines: task.composition.selectedLines.map((line) => ({
|
|
39595
|
+
inputOrdinal: line.inputOrdinal,
|
|
39596
|
+
stage: line.stage,
|
|
39597
|
+
hop: line.hop,
|
|
39598
|
+
visible: line.visible,
|
|
39599
|
+
outputStart: line.outputStart,
|
|
39600
|
+
outputEnd: line.outputEnd,
|
|
39601
|
+
visibleStart: line.visibleStart,
|
|
39602
|
+
visibleEnd: line.visibleEnd
|
|
39603
|
+
}))
|
|
39604
|
+
})
|
|
39605
|
+
];
|
|
39606
|
+
}
|
|
39607
|
+
if (dimension === "compositionDigests") {
|
|
39608
|
+
return [
|
|
39609
|
+
hashCanonicalJson({
|
|
39610
|
+
input: task.composition.input,
|
|
39611
|
+
output: task.composition.output,
|
|
39612
|
+
selectedLines: task.composition.selectedLines.map((line) => ({ input: line.input, output: line.output }))
|
|
39613
|
+
})
|
|
39614
|
+
];
|
|
39615
|
+
}
|
|
39616
|
+
const output = [];
|
|
39617
|
+
task.sessions.forEach((session, sessionOrdinal) => {
|
|
39618
|
+
const trace2 = session.trace;
|
|
39619
|
+
if (dimension === "sectionVisibleChars") {
|
|
39620
|
+
trace2.sections.forEach((value, sectionOrdinal) => {
|
|
39621
|
+
output.push(structuralSectionSignature(value, sessionOrdinal, sectionOrdinal, "emitted"));
|
|
39622
|
+
});
|
|
39623
|
+
} else if (dimension === "coreResults") {
|
|
39624
|
+
for (const [resultOrdinal, value] of (trace2.coreCapture?.results ?? []).entries())
|
|
39625
|
+
output.push(
|
|
39626
|
+
hashCanonicalJson({
|
|
39627
|
+
sessionOrdinal,
|
|
39628
|
+
resultOrdinal,
|
|
39629
|
+
memoryIdRef: value.memoryIdRef,
|
|
39630
|
+
servedBy: value.servedBy,
|
|
39631
|
+
scoreDecomposition: value.scoreDecomposition,
|
|
39632
|
+
admittedBy: value.admittedBy,
|
|
39633
|
+
rejectedBy: value.rejectedBy,
|
|
39634
|
+
disclosure: value.disclosure,
|
|
39635
|
+
estimatedTokens: value.estimatedTokens
|
|
39636
|
+
})
|
|
39637
|
+
);
|
|
39638
|
+
} else if (dimension === "coreFilters") {
|
|
39639
|
+
for (const value of trace2.coreCapture?.filters ?? [])
|
|
39640
|
+
output.push(
|
|
39641
|
+
hashCanonicalJson({
|
|
39642
|
+
sessionOrdinal,
|
|
39643
|
+
name: value.name,
|
|
39644
|
+
considered: value.considered,
|
|
39645
|
+
admitted: value.admitted
|
|
39646
|
+
})
|
|
39647
|
+
);
|
|
39648
|
+
} else if (dimension === "coreBudget") {
|
|
39649
|
+
if (trace2.coreCapture) output.push(hashCanonicalJson({ sessionOrdinal, ...trace2.coreCapture.budget }));
|
|
39650
|
+
}
|
|
39651
|
+
});
|
|
39652
|
+
return output;
|
|
39653
|
+
}
|
|
39654
|
+
function compositionOutcomeSignature(task) {
|
|
39655
|
+
return hashCanonicalJson({
|
|
39656
|
+
output: task.composition.output,
|
|
39657
|
+
selectedLines: task.composition.selectedLines.map((line) => ({ input: line.input, output: line.output }))
|
|
39658
|
+
});
|
|
39659
|
+
}
|
|
39660
|
+
function hasFixedBudgetTailGeometryTransition(baseline, real) {
|
|
39661
|
+
let sawTransition = false;
|
|
39662
|
+
for (let index = 0; index < baseline.sessions.length; index += 1) {
|
|
39663
|
+
const baselineSession = baseline.sessions[index];
|
|
39664
|
+
const realSession = real.sessions[index];
|
|
39665
|
+
const baselineTrace = baselineSession?.trace;
|
|
39666
|
+
const realTrace = realSession?.trace;
|
|
39667
|
+
if (!baselineSession || !realSession || !baselineTrace || !realTrace || baselineTrace.budget.requestedChars !== realTrace.budget.requestedChars) {
|
|
39668
|
+
return false;
|
|
39669
|
+
}
|
|
39670
|
+
if (baselineTrace.budget.truncated === realTrace.budget.truncated) {
|
|
39671
|
+
if (baselineTrace.budget.composedChars !== realTrace.budget.composedChars || baselineTrace.budget.returnedChars !== realTrace.budget.returnedChars || sectionGroupChanged(
|
|
39672
|
+
{ ...baseline, sessions: [baselineSession] },
|
|
39673
|
+
{ ...real, sessions: [realSession] },
|
|
39674
|
+
"core",
|
|
39675
|
+
"composed"
|
|
39676
|
+
) || sectionGroupChanged(
|
|
39677
|
+
{ ...baseline, sessions: [baselineSession] },
|
|
39678
|
+
{ ...real, sessions: [realSession] },
|
|
39679
|
+
"lcm",
|
|
39680
|
+
"composed"
|
|
39681
|
+
) || sectionGroupChanged(
|
|
39682
|
+
{ ...baseline, sessions: [baselineSession] },
|
|
39683
|
+
{ ...real, sessions: [realSession] },
|
|
39684
|
+
"other",
|
|
39685
|
+
"composed"
|
|
39686
|
+
)) {
|
|
39687
|
+
return false;
|
|
39688
|
+
}
|
|
39689
|
+
continue;
|
|
39690
|
+
}
|
|
39691
|
+
const shorter = baselineTrace.budget.truncated ? realTrace : baselineTrace;
|
|
39692
|
+
const longer = baselineTrace.budget.truncated ? baselineTrace : realTrace;
|
|
39693
|
+
const requested = shorter.budget.requestedChars;
|
|
39694
|
+
if (shorter.budget.truncated || !longer.budget.truncated || shorter.budget.composedChars > requested || longer.budget.composedChars <= requested || shorter.budget.returnedChars !== shorter.budget.composedChars || longer.budget.returnedChars !== requested || shorter.sections.length === 0 || shorter.sections.length !== longer.sections.length) {
|
|
39695
|
+
return false;
|
|
39696
|
+
}
|
|
39697
|
+
const finalIndex = shorter.sections.length - 1;
|
|
39698
|
+
for (let sectionIndex = 0; sectionIndex < finalIndex; sectionIndex += 1) {
|
|
39699
|
+
const shortSection = shorter.sections[sectionIndex];
|
|
39700
|
+
const longSection = longer.sections[sectionIndex];
|
|
39701
|
+
if (!shortSection || !longSection) return false;
|
|
39702
|
+
if (structuralSectionSignature(shortSection, 0, sectionIndex, "composed") !== structuralSectionSignature(longSection, 0, sectionIndex, "composed")) {
|
|
39703
|
+
return false;
|
|
39704
|
+
}
|
|
39705
|
+
}
|
|
39706
|
+
const shortFinal = shorter.sections[finalIndex];
|
|
39707
|
+
const longFinal = longer.sections[finalIndex];
|
|
39708
|
+
if (!shortFinal || !longFinal) return false;
|
|
39709
|
+
const extension = longFinal.composedEnd - shortFinal.composedEnd;
|
|
39710
|
+
if (shortFinal.id !== longFinal.id || shortFinal.source !== longFinal.source || shortFinal.separatorStart !== longFinal.separatorStart || shortFinal.contentStart !== longFinal.contentStart || shortFinal.composedStart !== longFinal.composedStart || shortFinal.contentEnd !== shortFinal.composedEnd || longFinal.contentEnd !== longFinal.composedEnd || shortFinal.composedEnd !== shorter.budget.composedChars || longFinal.composedEnd !== longer.budget.composedChars || extension <= 0 || longFinal.contentEnd - shortFinal.contentEnd !== extension) {
|
|
39711
|
+
return false;
|
|
39712
|
+
}
|
|
39713
|
+
sawTransition = true;
|
|
39714
|
+
}
|
|
39715
|
+
return sawTransition;
|
|
39716
|
+
}
|
|
39717
|
+
function delta(baseline, real) {
|
|
39718
|
+
const realCounts = counts(real);
|
|
39719
|
+
let sharedCount = 0;
|
|
39720
|
+
for (const entry of baseline) {
|
|
39721
|
+
const available = realCounts.get(entry) ?? 0;
|
|
39722
|
+
if (available > 0) {
|
|
39723
|
+
sharedCount += 1;
|
|
39724
|
+
realCounts.set(entry, available - 1);
|
|
39725
|
+
}
|
|
39726
|
+
}
|
|
39727
|
+
return {
|
|
39728
|
+
baselineCount: baseline.length,
|
|
39729
|
+
realCount: real.length,
|
|
39730
|
+
sharedCount,
|
|
39731
|
+
baselineOnlyCount: baseline.length - sharedCount,
|
|
39732
|
+
realOnlyCount: real.length - sharedCount,
|
|
39733
|
+
changed: sharedCount !== baseline.length || sharedCount !== real.length
|
|
39734
|
+
};
|
|
39735
|
+
}
|
|
39736
|
+
function counts(values) {
|
|
39737
|
+
const output = /* @__PURE__ */ new Map();
|
|
39738
|
+
for (const value of values) output.set(value, (output.get(value) ?? 0) + 1);
|
|
39739
|
+
return output;
|
|
39740
|
+
}
|
|
39741
|
+
function summarize(tasks) {
|
|
39742
|
+
const mechanisms = Object.fromEntries(MECHANISMS.map((mechanism) => [mechanism, 0]));
|
|
39743
|
+
for (const task of tasks) mechanisms[task.mechanism] += 1;
|
|
39744
|
+
return { taskCount: tasks.length, mechanisms };
|
|
39745
|
+
}
|
|
39746
|
+
function digestIdentifier(value) {
|
|
39747
|
+
return { sha256: hashString(value), length: value.length };
|
|
39748
|
+
}
|
|
39749
|
+
function categoryOf(taskId) {
|
|
39750
|
+
const category = CATEGORIES.find((candidate) => taskId.endsWith(`-${candidate}`));
|
|
39751
|
+
if (!category) throw new Error("LoCoMo retrieval trace task id has an unsupported category.");
|
|
39752
|
+
return category;
|
|
39753
|
+
}
|
|
39754
|
+
function assertReceipt(receipt, label) {
|
|
39755
|
+
if (!receipt || typeof receipt !== "object") throw new Error(`${label} receipt must be an object.`);
|
|
39756
|
+
assertFiniteJson(receipt, `${label} receipt`);
|
|
39757
|
+
assertExactKeys(
|
|
39758
|
+
receipt,
|
|
39759
|
+
["schemaVersion", "benchmarkId", "captureKind", "artifactHash", "sensitivity", "provenance", "selection", "tasks"],
|
|
39760
|
+
`${label} receipt`
|
|
39761
|
+
);
|
|
39762
|
+
assertExactKeys(
|
|
39763
|
+
receipt.sensitivity,
|
|
39764
|
+
["classification", "contentEncoding", "containsGold", "containsRawContent"],
|
|
39765
|
+
`${label} receipt.sensitivity`
|
|
39766
|
+
);
|
|
39767
|
+
assertExactKeys(
|
|
39768
|
+
receipt.provenance,
|
|
39769
|
+
[
|
|
39770
|
+
"gitSha",
|
|
39771
|
+
"remnicVersion",
|
|
39772
|
+
"runtimeProfile",
|
|
39773
|
+
"adapterMode",
|
|
39774
|
+
"replayExtractionMode",
|
|
39775
|
+
"providerFree",
|
|
39776
|
+
"dataset",
|
|
39777
|
+
"retrievalConfigSha256",
|
|
39778
|
+
"recallBudget"
|
|
39779
|
+
],
|
|
39780
|
+
`${label} receipt.provenance`
|
|
39781
|
+
);
|
|
39782
|
+
assertExactKeys(receipt.provenance.dataset, ["id", "sha256"], `${label} receipt.provenance.dataset`);
|
|
39783
|
+
assertExactKeys(
|
|
39784
|
+
receipt.provenance.recallBudget,
|
|
39785
|
+
["algorithm", "version"],
|
|
39786
|
+
`${label} receipt.provenance.recallBudget`
|
|
39787
|
+
);
|
|
39788
|
+
assertExactKeys(
|
|
39789
|
+
receipt.selection,
|
|
39790
|
+
["algorithm", "version", "seed", "candidateCount", "selectedCount", "selectedTaskIds", "selectedTaskIdsSha256"],
|
|
39791
|
+
`${label} receipt.selection`
|
|
39792
|
+
);
|
|
39793
|
+
const { artifactHash, ...withoutHash } = receipt;
|
|
39794
|
+
if (!isSha256Hex(artifactHash) || hashCanonicalJson(withoutHash) !== artifactHash) {
|
|
39795
|
+
throw new Error(`${label} retrieval trace artifact hash verification failed.`);
|
|
39796
|
+
}
|
|
39797
|
+
if (receipt.schemaVersion !== 1 || receipt.benchmarkId !== "locomo" || receipt.captureKind !== "retrieval-only" || receipt.sensitivity.classification !== "restricted" || receipt.sensitivity.contentEncoding !== "sha256+length" || receipt.sensitivity.containsGold !== false || receipt.sensitivity.containsRawContent !== false || receipt.provenance.providerFree !== true || receipt.provenance.adapterMode !== "direct" || receipt.provenance.replayExtractionMode !== "skip" || receipt.provenance.dataset.id !== "locomo-10" || receipt.provenance.recallBudget.algorithm !== "benchmarkRecallBudgetForSessionCount" || receipt.provenance.recallBudget.version !== 1 || typeof receipt.provenance.gitSha !== "string" || receipt.provenance.gitSha.length === 0 || typeof receipt.provenance.remnicVersion !== "string" || receipt.provenance.remnicVersion.length === 0 || receipt.selection.version !== 1 || receipt.selection.algorithm !== "explicit-task-ids" && receipt.selection.algorithm !== "sha256-seeded-sample" || receipt.selection.algorithm === "explicit-task-ids" && receipt.selection.seed !== void 0 || receipt.selection.algorithm === "sha256-seeded-sample" && !isNonNegativeSafeInteger(receipt.selection.seed) || !Array.isArray(receipt.selection.selectedTaskIds) || receipt.selection.selectedTaskIds.some((taskId) => typeof taskId !== "string" || taskId.length === 0) || !Array.isArray(receipt.tasks) || !Number.isSafeInteger(receipt.selection.candidateCount) || !Number.isSafeInteger(receipt.selection.selectedCount) || receipt.selection.selectedCount <= 0 || receipt.selection.candidateCount < receipt.selection.selectedCount || receipt.selection.selectedCount !== receipt.tasks.length || receipt.selection.selectedTaskIds.length !== receipt.tasks.length || new Set(receipt.selection.selectedTaskIds).size !== receipt.selection.selectedTaskIds.length || new Set(receipt.tasks.map((task) => task.taskId)).size !== receipt.tasks.length || receipt.selection.selectedTaskIdsSha256 !== hashCanonicalJson(receipt.selection.selectedTaskIds) || receipt.selection.selectedTaskIds.some((taskId, index) => taskId !== receipt.tasks[index]?.taskId) || receipt.tasks.length === 0) {
|
|
39798
|
+
throw new Error(`${label} retrieval trace receipt violates the restricted provider-free contract.`);
|
|
39799
|
+
}
|
|
39800
|
+
if (!isSha256Hex(receipt.provenance.dataset.sha256) || !isSha256Hex(receipt.provenance.retrievalConfigSha256)) {
|
|
39801
|
+
throw new Error(`${label} retrieval trace receipt contains invalid provenance hashes.`);
|
|
39802
|
+
}
|
|
39803
|
+
for (const task of receipt.tasks) {
|
|
39804
|
+
assertExactKeys(
|
|
39805
|
+
task,
|
|
39806
|
+
["taskId", "question", "recallBudgetChars", "sessions", "composition"],
|
|
39807
|
+
`${label} retrieval trace task`
|
|
39808
|
+
);
|
|
39809
|
+
if (typeof task.taskId !== "string" || task.taskId.length === 0 || !Array.isArray(task.sessions) || task.sessions.length === 0 || !isDigest(task.question) || !Number.isSafeInteger(task.recallBudgetChars) || task.recallBudgetChars < 0) {
|
|
39810
|
+
throw new Error(`${label} retrieval trace task structure is invalid.`);
|
|
39811
|
+
}
|
|
39812
|
+
categoryOf(task.taskId);
|
|
39813
|
+
assertComposition(task.composition, label);
|
|
39814
|
+
for (const session of task.sessions) {
|
|
39815
|
+
assertExactKeys(session, ["session", "trace"], `${label} retrieval trace session`);
|
|
39816
|
+
const trace2 = session.trace;
|
|
39817
|
+
const expectedComposedChars = Array.isArray(trace2.sections) ? trace2.sections.reduce((maximum, section) => Math.max(maximum, section.composedEnd), 0) : -1;
|
|
39818
|
+
assertExactKeys(
|
|
39819
|
+
trace2,
|
|
39820
|
+
["schemaVersion", "sensitivity", "sections", "selections", "lcmCandidates", "coreCapture", "budget"],
|
|
39821
|
+
`${label} retrieval structural trace`
|
|
39822
|
+
);
|
|
39823
|
+
assertExactKeys(
|
|
39824
|
+
trace2.sensitivity,
|
|
39825
|
+
["classification", "contentEncoding", "containsGold"],
|
|
39826
|
+
`${label} retrieval trace sensitivity`
|
|
39827
|
+
);
|
|
39828
|
+
assertExactKeys(
|
|
39829
|
+
trace2.budget,
|
|
39830
|
+
["requestedChars", "composedChars", "returnedChars", "truncated"],
|
|
39831
|
+
`${label} retrieval trace budget`
|
|
39832
|
+
);
|
|
39833
|
+
if (!isDigest(session.session) || !trace2 || trace2.schemaVersion !== 1 || trace2.sensitivity.classification !== "restricted" || trace2.sensitivity.contentEncoding !== "sha256+length" || trace2.sensitivity.containsGold !== false || !Array.isArray(trace2.sections) || !Array.isArray(trace2.selections) || !Array.isArray(trace2.lcmCandidates) || new Set(trace2.sections.map((section) => section.id)).size !== trace2.sections.length || !isTraceBudget(trace2.budget) || trace2.budget.requestedChars !== task.recallBudgetChars || trace2.budget.composedChars !== expectedComposedChars || trace2.budget.returnedChars !== Math.min(trace2.budget.requestedChars, trace2.budget.composedChars) || trace2.budget.returnedChars > trace2.budget.composedChars || trace2.budget.truncated !== trace2.budget.returnedChars < trace2.budget.composedChars) {
|
|
39834
|
+
throw new Error(`${label} retrieval trace session structure is invalid.`);
|
|
39835
|
+
}
|
|
39836
|
+
let previousContentEnd = 0;
|
|
39837
|
+
for (const [sectionIndex, section] of trace2.sections.entries()) {
|
|
39838
|
+
assertExactKeys(
|
|
39839
|
+
section,
|
|
39840
|
+
[
|
|
39841
|
+
"id",
|
|
39842
|
+
"source",
|
|
39843
|
+
"separatorStart",
|
|
39844
|
+
"contentStart",
|
|
39845
|
+
"contentEnd",
|
|
39846
|
+
"composedStart",
|
|
39847
|
+
"composedEnd",
|
|
39848
|
+
"visibleStart",
|
|
39849
|
+
"visibleEnd",
|
|
39850
|
+
"visibleChars"
|
|
39851
|
+
],
|
|
39852
|
+
`${label} retrieval trace section`
|
|
39853
|
+
);
|
|
39854
|
+
const expectedVisibleStart = Math.min(section.composedStart, trace2.budget.returnedChars);
|
|
39855
|
+
const expectedVisibleEnd = Math.max(
|
|
39856
|
+
expectedVisibleStart,
|
|
39857
|
+
Math.min(section.composedEnd, trace2.budget.returnedChars)
|
|
39858
|
+
);
|
|
39859
|
+
const expectedSeparatorStart = sectionIndex === 0 ? 0 : previousContentEnd;
|
|
39860
|
+
const expectedContentStart = expectedSeparatorStart + (sectionIndex === 0 ? 0 : 2);
|
|
39861
|
+
if (![
|
|
39862
|
+
"derived",
|
|
39863
|
+
"explicit-cue",
|
|
39864
|
+
"trajectory-analysis",
|
|
39865
|
+
"core",
|
|
39866
|
+
"evidence-pack",
|
|
39867
|
+
"lcm-summary",
|
|
39868
|
+
"raw-row"
|
|
39869
|
+
].includes(section.source) || typeof section.id !== "string" || section.id.length === 0 || !isTraceRange(section) || ![section.separatorStart, section.contentStart, section.contentEnd].every(isNonNegativeSafeInteger) || section.separatorStart !== expectedSeparatorStart || section.contentStart !== expectedContentStart || section.contentEnd < section.contentStart || section.composedStart !== section.separatorStart || section.composedEnd !== section.contentEnd || !Number.isSafeInteger(section.visibleChars) || section.visibleChars < 0 || section.visibleStart !== expectedVisibleStart || section.visibleEnd !== expectedVisibleEnd || section.visibleChars !== expectedVisibleEnd - expectedVisibleStart) {
|
|
39870
|
+
throw new Error(`${label} retrieval trace section structure is invalid.`);
|
|
39871
|
+
}
|
|
39872
|
+
previousContentEnd = section.contentEnd;
|
|
39873
|
+
}
|
|
39874
|
+
for (const selection of trace2.selections) {
|
|
39875
|
+
assertExactKeys(
|
|
39876
|
+
selection,
|
|
39877
|
+
[
|
|
39878
|
+
"sectionId",
|
|
39879
|
+
"kind",
|
|
39880
|
+
"lineageStatus",
|
|
39881
|
+
"archiveRowIds",
|
|
39882
|
+
"turnIndex",
|
|
39883
|
+
"role",
|
|
39884
|
+
"score",
|
|
39885
|
+
"summary",
|
|
39886
|
+
"composedStart",
|
|
39887
|
+
"composedEnd",
|
|
39888
|
+
"visibleStart",
|
|
39889
|
+
"visibleEnd"
|
|
39890
|
+
],
|
|
39891
|
+
`${label} retrieval trace selection`
|
|
39892
|
+
);
|
|
39893
|
+
if (selection.summary !== void 0) {
|
|
39894
|
+
assertExactKeys(
|
|
39895
|
+
selection.summary,
|
|
39896
|
+
["depth", "msgStart", "msgEnd"],
|
|
39897
|
+
`${label} retrieval trace selection summary`
|
|
39898
|
+
);
|
|
39899
|
+
}
|
|
39900
|
+
const selectedSection = trace2.sections.find((section) => section.id === selection.sectionId);
|
|
39901
|
+
const expectedVisibleStart = Math.min(selection.composedStart, trace2.budget.returnedChars);
|
|
39902
|
+
const expectedVisibleEnd = Math.max(
|
|
39903
|
+
expectedVisibleStart,
|
|
39904
|
+
Math.min(selection.composedEnd, trace2.budget.returnedChars)
|
|
39905
|
+
);
|
|
39906
|
+
if (!["evidence-block", "trajectory-line", "lcm-summary", "raw-row"].includes(selection.kind) || typeof selection.sectionId !== "string" || selection.sectionId.length === 0 || selectedSection === void 0 || selection.kind === "lcm-summary" && selectedSection.source !== "lcm-summary" || selection.kind === "raw-row" && selectedSection.source !== "raw-row" || selection.kind === "evidence-block" && selectedSection.source !== "explicit-cue" && selectedSection.source !== "evidence-pack" || selection.kind === "trajectory-line" && selectedSection.source !== "trajectory-analysis" || !isTraceRange(selection) || selection.composedStart < selectedSection.contentStart || selection.composedEnd > selectedSection.contentEnd || selection.visibleStart !== expectedVisibleStart || selection.visibleEnd !== expectedVisibleEnd || selection.lineageStatus !== "exact" && selection.lineageStatus !== "unavailable" || selection.turnIndex !== void 0 && !isNonNegativeSafeInteger(selection.turnIndex) || selection.role !== void 0 && typeof selection.role !== "string" || selection.score !== void 0 && !Number.isFinite(selection.score) || selection.summary !== void 0 && ![selection.summary.depth, selection.summary.msgStart, selection.summary.msgEnd].every(
|
|
39907
|
+
isNonNegativeSafeInteger
|
|
39908
|
+
) || selection.archiveRowIds !== void 0 && (!Array.isArray(selection.archiveRowIds) || selection.archiveRowIds.some((id) => !Number.isSafeInteger(id) || id <= 0))) {
|
|
39909
|
+
throw new Error(`${label} retrieval trace selection structure is invalid.`);
|
|
39910
|
+
}
|
|
39911
|
+
}
|
|
39912
|
+
for (const candidate of trace2.lcmCandidates) {
|
|
39913
|
+
assertExactKeys(
|
|
39914
|
+
candidate,
|
|
39915
|
+
["rank", "archiveRowId", "turnIndex", "role", "score", "lineageStatus"],
|
|
39916
|
+
`${label} retrieval trace LCM candidate`
|
|
39917
|
+
);
|
|
39918
|
+
if (candidate.lineageStatus !== "exact" && candidate.lineageStatus !== "unavailable" || !Number.isSafeInteger(candidate.rank) || candidate.rank < 0 || !isNonNegativeSafeInteger(candidate.turnIndex) || typeof candidate.role !== "string" || candidate.score !== void 0 && !Number.isFinite(candidate.score) || candidate.lineageStatus === "exact" && candidate.archiveRowId === void 0 || candidate.lineageStatus === "unavailable" && candidate.archiveRowId !== void 0 || candidate.archiveRowId !== void 0 && (!Number.isSafeInteger(candidate.archiveRowId) || candidate.archiveRowId <= 0)) {
|
|
39919
|
+
throw new Error(`${label} retrieval trace LCM candidate structure is invalid.`);
|
|
39920
|
+
}
|
|
39921
|
+
}
|
|
39922
|
+
if (trace2.coreCapture) {
|
|
39923
|
+
assertExactKeys(trace2.coreCapture, ["budget", "filters", "results"], `${label} retrieval trace core capture`);
|
|
39924
|
+
assertExactKeys(trace2.coreCapture.budget, ["chars", "used"], `${label} retrieval trace core budget`);
|
|
39925
|
+
for (const filter of trace2.coreCapture.filters) {
|
|
39926
|
+
assertExactKeys(filter, ["name", "considered", "admitted"], `${label} retrieval trace core filter`);
|
|
39927
|
+
if (typeof filter.name !== "string" || !isNonNegativeSafeInteger(filter.considered) || !isNonNegativeSafeInteger(filter.admitted) || filter.admitted > filter.considered) {
|
|
39928
|
+
throw new Error(`${label} retrieval trace core filter structure is invalid.`);
|
|
39929
|
+
}
|
|
39930
|
+
}
|
|
39931
|
+
for (const result of trace2.coreCapture.results) {
|
|
39932
|
+
assertExactKeys(
|
|
39933
|
+
result,
|
|
39934
|
+
[
|
|
39935
|
+
"memoryIdRef",
|
|
39936
|
+
"servedBy",
|
|
39937
|
+
"scoreDecomposition",
|
|
39938
|
+
"admittedBy",
|
|
39939
|
+
"rejectedBy",
|
|
39940
|
+
"disclosure",
|
|
39941
|
+
"estimatedTokens"
|
|
39942
|
+
],
|
|
39943
|
+
`${label} retrieval trace core result`
|
|
39944
|
+
);
|
|
39945
|
+
assertExactKeys(result.memoryIdRef, ["sha256", "length"], `${label} retrieval trace core memory reference`);
|
|
39946
|
+
assertExactKeys(
|
|
39947
|
+
result.scoreDecomposition,
|
|
39948
|
+
["vector", "bm25", "importance", "mmrPenalty", "tierPrior", "reinforcementBoost", "final"],
|
|
39949
|
+
`${label} retrieval trace score decomposition`
|
|
39950
|
+
);
|
|
39951
|
+
if (typeof result.servedBy !== "string" || !Number.isFinite(result.scoreDecomposition.final) || !Object.values(result.scoreDecomposition).every((score) => score === void 0 || Number.isFinite(score)) || !Array.isArray(result.admittedBy) || result.admittedBy.some((reason) => typeof reason !== "string") || result.rejectedBy !== void 0 && typeof result.rejectedBy !== "string" || result.disclosure !== void 0 && !["chunk", "section", "raw"].includes(result.disclosure) || result.estimatedTokens !== void 0 && !isNonNegativeSafeInteger(result.estimatedTokens)) {
|
|
39952
|
+
throw new Error(`${label} retrieval trace core result structure is invalid.`);
|
|
39953
|
+
}
|
|
39954
|
+
}
|
|
39955
|
+
if (!isCountBudget(trace2.coreCapture.budget) || !Array.isArray(trace2.coreCapture.filters) || !Array.isArray(trace2.coreCapture.results) || trace2.coreCapture.results.some(
|
|
39956
|
+
(result) => !isSha256Hex(result.memoryIdRef?.sha256) || !Number.isSafeInteger(result.memoryIdRef?.length) || result.memoryIdRef.length <= 0
|
|
39957
|
+
)) {
|
|
39958
|
+
throw new Error(`${label} retrieval trace core capture structure is invalid.`);
|
|
39959
|
+
}
|
|
39960
|
+
}
|
|
39961
|
+
}
|
|
39962
|
+
}
|
|
39963
|
+
}
|
|
39964
|
+
function assertComposition(composition, label) {
|
|
39965
|
+
assertExactKeys(
|
|
39966
|
+
composition,
|
|
39967
|
+
["schemaVersion", "mode", "multiHopRecallComposition", "input", "output", "selectedLines"],
|
|
39968
|
+
`${label} retrieval trace composition`
|
|
39969
|
+
);
|
|
39970
|
+
if (!composition || composition.schemaVersion !== 1 || composition.mode !== "focused" && composition.mode !== "fallback" || typeof composition.multiHopRecallComposition !== "boolean" || !isDigest(composition.input) || !isDigest(composition.output) || !Array.isArray(composition.selectedLines)) {
|
|
39971
|
+
throw new Error(`${label} retrieval trace composition structure is invalid.`);
|
|
39972
|
+
}
|
|
39973
|
+
let previousOutputEnd = -1;
|
|
39974
|
+
for (const line of composition.selectedLines) {
|
|
39975
|
+
assertExactKeys(
|
|
39976
|
+
line,
|
|
39977
|
+
[
|
|
39978
|
+
"inputOrdinal",
|
|
39979
|
+
"input",
|
|
39980
|
+
"output",
|
|
39981
|
+
"stage",
|
|
39982
|
+
"hop",
|
|
39983
|
+
"visible",
|
|
39984
|
+
"outputStart",
|
|
39985
|
+
"outputEnd",
|
|
39986
|
+
"visibleStart",
|
|
39987
|
+
"visibleEnd"
|
|
39988
|
+
],
|
|
39989
|
+
`${label} retrieval trace composition line`
|
|
39990
|
+
);
|
|
39991
|
+
if (!isNonNegativeSafeInteger(line.inputOrdinal) || !isDigest(line.input) || !isDigest(line.output) || line.stage !== "direct" && line.stage !== "linked" || line.hop !== void 0 && !isNonNegativeSafeInteger(line.hop) || typeof line.visible !== "boolean" || ![line.outputStart, line.outputEnd, line.visibleStart, line.visibleEnd].every(isNonNegativeSafeInteger) || line.outputStart < previousOutputEnd || line.outputEnd < line.outputStart || line.outputEnd - line.outputStart !== line.output.charCount || line.visibleEnd < line.visibleStart || line.visibleStart > line.outputStart || line.visibleEnd > line.outputEnd || line.visibleEnd > composition.output.charCount || line.visibleStart < line.outputStart && line.visibleStart !== line.visibleEnd || line.visible !== (line.visibleStart === line.outputStart && line.visibleEnd === line.outputEnd)) {
|
|
39992
|
+
throw new Error(`${label} retrieval trace composition line structure is invalid.`);
|
|
39993
|
+
}
|
|
39994
|
+
previousOutputEnd = line.outputEnd;
|
|
39995
|
+
}
|
|
39996
|
+
}
|
|
39997
|
+
function assertExactKeys(value, allowed, label) {
|
|
39998
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
39999
|
+
throw new Error(`${label} must be an object.`);
|
|
40000
|
+
}
|
|
40001
|
+
const allowedKeys = new Set(allowed);
|
|
40002
|
+
if (Object.keys(value).some((key) => !allowedKeys.has(key))) {
|
|
40003
|
+
throw new Error(`${label} contains an unsupported field.`);
|
|
40004
|
+
}
|
|
40005
|
+
}
|
|
40006
|
+
function isNonNegativeSafeInteger(value) {
|
|
40007
|
+
return Number.isSafeInteger(value) && value >= 0;
|
|
40008
|
+
}
|
|
40009
|
+
function isTraceRange(value) {
|
|
40010
|
+
return [value.composedStart, value.composedEnd, value.visibleStart, value.visibleEnd].every(isNonNegativeSafeInteger) && value.composedEnd >= value.composedStart && value.visibleEnd >= value.visibleStart;
|
|
40011
|
+
}
|
|
40012
|
+
function isTraceBudget(value) {
|
|
40013
|
+
if (!value || typeof value !== "object") return false;
|
|
40014
|
+
const budget = value;
|
|
40015
|
+
return [budget.requestedChars, budget.composedChars, budget.returnedChars].every(
|
|
40016
|
+
(entry) => Number.isSafeInteger(entry) && entry >= 0
|
|
40017
|
+
) && typeof budget.truncated === "boolean";
|
|
40018
|
+
}
|
|
40019
|
+
function isCountBudget(value) {
|
|
40020
|
+
if (!value || typeof value !== "object") return false;
|
|
40021
|
+
const budget = value;
|
|
40022
|
+
return [budget.chars, budget.used].every((entry) => Number.isSafeInteger(entry) && entry >= 0);
|
|
40023
|
+
}
|
|
40024
|
+
function assertFiniteJson(value, label) {
|
|
40025
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return;
|
|
40026
|
+
if (typeof value === "number") {
|
|
40027
|
+
if (!Number.isFinite(value)) throw new Error(`${label} contains a non-finite number.`);
|
|
40028
|
+
return;
|
|
40029
|
+
}
|
|
40030
|
+
if (Array.isArray(value)) {
|
|
40031
|
+
value.forEach((entry, index) => assertFiniteJson(entry, `${label}[${index}]`));
|
|
40032
|
+
return;
|
|
40033
|
+
}
|
|
40034
|
+
if (!value || typeof value !== "object") throw new Error(`${label} is not canonical JSON.`);
|
|
40035
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
40036
|
+
if (entry === void 0) throw new Error(`${label}.${key} is undefined.`);
|
|
40037
|
+
assertFiniteJson(entry, `${label}.${key}`);
|
|
40038
|
+
}
|
|
40039
|
+
}
|
|
40040
|
+
function assertComparable(baseline, real) {
|
|
40041
|
+
if (baseline.provenance.runtimeProfile !== "baseline" || real.provenance.runtimeProfile !== "real") {
|
|
40042
|
+
throw new Error("Paired retrieval traces require baseline and real runtime profiles in that order.");
|
|
40043
|
+
}
|
|
40044
|
+
const matching = [
|
|
40045
|
+
[baseline.schemaVersion, real.schemaVersion],
|
|
40046
|
+
[baseline.benchmarkId, real.benchmarkId],
|
|
40047
|
+
[baseline.provenance.gitSha, real.provenance.gitSha],
|
|
40048
|
+
[baseline.provenance.remnicVersion, real.provenance.remnicVersion],
|
|
40049
|
+
[baseline.provenance.dataset.sha256, real.provenance.dataset.sha256],
|
|
40050
|
+
[baseline.provenance.recallBudget.version, real.provenance.recallBudget.version],
|
|
40051
|
+
[baseline.selection.selectedTaskIdsSha256, real.selection.selectedTaskIdsSha256],
|
|
40052
|
+
[canonicalJsonStringify(baseline.selection), canonicalJsonStringify(real.selection)],
|
|
40053
|
+
[baseline.tasks.length, real.tasks.length]
|
|
40054
|
+
];
|
|
40055
|
+
if (matching.some(([left, right]) => left !== right))
|
|
40056
|
+
throw new Error("Paired retrieval trace provenance does not match.");
|
|
40057
|
+
if (baseline.provenance.retrievalConfigSha256 === real.provenance.retrievalConfigSha256) {
|
|
40058
|
+
throw new Error("Paired retrieval traces must use different baseline and real retrieval configuration hashes.");
|
|
40059
|
+
}
|
|
40060
|
+
baseline.tasks.forEach((left, index) => {
|
|
40061
|
+
const right = real.tasks[index];
|
|
40062
|
+
if (!right || left.taskId !== right.taskId || canonicalJsonStringify(left.question) !== canonicalJsonStringify(right.question) || left.recallBudgetChars !== right.recallBudgetChars || left.sessions.length !== right.sessions.length || left.composition.multiHopRecallComposition !== right.composition.multiHopRecallComposition) {
|
|
40063
|
+
throw new Error(`Paired retrieval trace task mismatch at index ${index}.`);
|
|
40064
|
+
}
|
|
40065
|
+
left.sessions.forEach((session, sessionIndex) => {
|
|
40066
|
+
const other = right.sessions[sessionIndex];
|
|
40067
|
+
if (!other || canonicalJsonStringify(session.session) !== canonicalJsonStringify(other.session)) {
|
|
40068
|
+
throw new Error(`Paired retrieval trace session mismatch at task ${index}, session ${sessionIndex}.`);
|
|
40069
|
+
}
|
|
40070
|
+
if (session.trace.budget.requestedChars !== other.trace.budget.requestedChars) {
|
|
40071
|
+
throw new Error(`Paired retrieval trace budget mismatch at task ${index}, session ${sessionIndex}.`);
|
|
40072
|
+
}
|
|
40073
|
+
});
|
|
40074
|
+
});
|
|
40075
|
+
}
|
|
40076
|
+
function isDigest(value) {
|
|
40077
|
+
if (!value || typeof value !== "object") return false;
|
|
40078
|
+
if (Object.keys(value).some((key) => !["sha256", "charCount", "lineCount"].includes(key))) return false;
|
|
40079
|
+
const digest = value;
|
|
40080
|
+
return isSha256Hex(digest.sha256) && Number.isSafeInteger(digest.charCount) && digest.charCount >= 0 && Number.isSafeInteger(digest.lineCount) && digest.lineCount >= 0;
|
|
40081
|
+
}
|
|
40082
|
+
|
|
40083
|
+
// src/benchmarks/published/locomo/retrieval-trace-runner.ts
|
|
40084
|
+
var LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION = 1;
|
|
40085
|
+
var LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION = 1;
|
|
40086
|
+
var LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION = 1;
|
|
40087
|
+
async function preflightLoCoMoRetrievalTraceCapture(options) {
|
|
40088
|
+
assertCaptureOptions(options);
|
|
40089
|
+
assertProviderFreeRetrievalConfig(options.retrievalConfig);
|
|
40090
|
+
const loaded = await loadLoCoMoDataset("full", options.datasetDir);
|
|
40091
|
+
const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
|
|
40092
|
+
const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
|
|
40093
|
+
const selectable = plans.flatMap(
|
|
40094
|
+
(plan, planIndex) => plan.trials.map((trial) => ({ taskId: trial.taskId, planIndex }))
|
|
40095
|
+
);
|
|
40096
|
+
selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
|
|
40097
|
+
}
|
|
40098
|
+
function buildProviderFreeLoCoMoRetrievalConfig(retrievalConfig) {
|
|
40099
|
+
const sanitized = sanitizeProviderFreeRetrievalConfig(retrievalConfig);
|
|
40100
|
+
return assertProviderFreeRetrievalConfig({
|
|
40101
|
+
...sanitized,
|
|
40102
|
+
localLlmEnabled: false,
|
|
40103
|
+
localLlmFastEnabled: false,
|
|
40104
|
+
recallPlannerEnabled: false,
|
|
40105
|
+
embeddingFallbackEnabled: false,
|
|
40106
|
+
hostEmbeddingProviderEnabled: false,
|
|
40107
|
+
openaiApiKey: false,
|
|
40108
|
+
modelSource: "plugin"
|
|
40109
|
+
});
|
|
40110
|
+
}
|
|
40111
|
+
async function captureLoCoMoRetrievalTrace(options) {
|
|
40112
|
+
assertCaptureOptions(options);
|
|
40113
|
+
const retrievalConfig = assertProviderFreeRetrievalConfig(options.retrievalConfig);
|
|
40114
|
+
const recallWithTrace = options.system.recallWithTrace?.bind(options.system);
|
|
40115
|
+
if (!recallWithTrace) {
|
|
40116
|
+
throw new Error("LoCoMo retrieval trace capture requires system.recallWithTrace().");
|
|
40117
|
+
}
|
|
40118
|
+
const loaded = await loadLoCoMoDataset("full", options.datasetDir);
|
|
40119
|
+
const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
|
|
40120
|
+
const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
|
|
40121
|
+
const selectable = plans.flatMap(
|
|
40122
|
+
(plan, planIndex) => plan.trials.map(
|
|
40123
|
+
(trial) => ({
|
|
40124
|
+
taskId: trial.taskId,
|
|
40125
|
+
question: trial.question,
|
|
40126
|
+
recallSessionIds: [...trial.recallSessionIds],
|
|
40127
|
+
planIndex
|
|
40128
|
+
})
|
|
40129
|
+
)
|
|
40130
|
+
);
|
|
40131
|
+
const selection = selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
|
|
40132
|
+
const selectedIds = new Set(selection.selectedTaskIds);
|
|
40133
|
+
const tasks = [];
|
|
40134
|
+
for (let planIndex = 0; planIndex < plans.length; planIndex += 1) {
|
|
40135
|
+
const selected = selectable.filter((task) => task.planIndex === planIndex && selectedIds.has(task.taskId));
|
|
40136
|
+
if (selected.length === 0) continue;
|
|
40137
|
+
const plan = plans[planIndex];
|
|
40138
|
+
if (!plan) throw new Error(`Missing LoCoMo plan at index ${planIndex}.`);
|
|
40139
|
+
await options.system.reset();
|
|
40140
|
+
for (const session of plan.ingestSessions) {
|
|
40141
|
+
if (session.messages.length > 0) {
|
|
40142
|
+
await options.system.store(session.sessionId, session.messages);
|
|
40143
|
+
}
|
|
40144
|
+
}
|
|
40145
|
+
await options.system.drain?.();
|
|
40146
|
+
for (const selectedTask of selected) {
|
|
40147
|
+
const recallBudgetChars = benchmarkRecallBudgetForSessionCount(selectedTask.recallSessionIds.length);
|
|
40148
|
+
const recalled = await Promise.all(
|
|
40149
|
+
selectedTask.recallSessionIds.map(async (sessionId) => {
|
|
40150
|
+
const result = await recallWithTrace(sessionId, selectedTask.question, recallBudgetChars);
|
|
40151
|
+
return {
|
|
40152
|
+
text: result.text,
|
|
40153
|
+
receipt: {
|
|
40154
|
+
session: digestContent(sessionId),
|
|
40155
|
+
trace: sanitizeStructuralTrace(result.trace)
|
|
40156
|
+
}
|
|
40157
|
+
};
|
|
40158
|
+
})
|
|
40159
|
+
);
|
|
40160
|
+
const rawRecalledText = recalled.map((entry) => entry.text).filter(Boolean).join("\n\n");
|
|
40161
|
+
const sanitized = sanitizeLoCoMoRecallText({
|
|
40162
|
+
question: selectedTask.question,
|
|
40163
|
+
recalledText: rawRecalledText
|
|
40164
|
+
});
|
|
40165
|
+
const composition = prioritizeLoCoMoRecallTextWithTrace({
|
|
40166
|
+
question: selectedTask.question,
|
|
40167
|
+
recalledText: sanitized,
|
|
40168
|
+
multiHopRecallComposition
|
|
40169
|
+
});
|
|
40170
|
+
tasks.push({
|
|
40171
|
+
taskId: selectedTask.taskId,
|
|
40172
|
+
question: digestContent(selectedTask.question),
|
|
40173
|
+
recallBudgetChars,
|
|
40174
|
+
sessions: recalled.map((entry) => entry.receipt),
|
|
40175
|
+
composition: composition.receipt
|
|
40176
|
+
});
|
|
40177
|
+
}
|
|
40178
|
+
}
|
|
40179
|
+
const withoutHash = {
|
|
40180
|
+
schemaVersion: LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
|
|
40181
|
+
benchmarkId: "locomo",
|
|
40182
|
+
captureKind: "retrieval-only",
|
|
40183
|
+
sensitivity: {
|
|
40184
|
+
classification: "restricted",
|
|
40185
|
+
contentEncoding: "sha256+length",
|
|
40186
|
+
containsGold: false,
|
|
40187
|
+
containsRawContent: false
|
|
40188
|
+
},
|
|
40189
|
+
provenance: {
|
|
40190
|
+
gitSha: options.gitSha,
|
|
40191
|
+
remnicVersion: options.remnicVersion,
|
|
40192
|
+
runtimeProfile: options.runtimeProfile,
|
|
40193
|
+
adapterMode: "direct",
|
|
40194
|
+
replayExtractionMode: "skip",
|
|
40195
|
+
providerFree: true,
|
|
40196
|
+
dataset: { id: "locomo-10", sha256: loaded.sha256 },
|
|
40197
|
+
retrievalConfigSha256: hashCanonicalJson(retrievalConfig),
|
|
40198
|
+
recallBudget: {
|
|
40199
|
+
algorithm: "benchmarkRecallBudgetForSessionCount",
|
|
40200
|
+
version: LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION
|
|
40201
|
+
}
|
|
40202
|
+
},
|
|
40203
|
+
selection,
|
|
40204
|
+
tasks
|
|
40205
|
+
};
|
|
40206
|
+
return {
|
|
40207
|
+
...withoutHash,
|
|
40208
|
+
artifactHash: hashCanonicalJson(withoutHash)
|
|
40209
|
+
};
|
|
40210
|
+
}
|
|
40211
|
+
function selectLoCoMoRetrievalTraceTasks(tasks, selector) {
|
|
40212
|
+
const allIds = tasks.map((task) => task.taskId);
|
|
40213
|
+
if (new Set(allIds).size !== allIds.length) {
|
|
40214
|
+
throw new Error("LoCoMo retrieval trace task ids must be unique.");
|
|
40215
|
+
}
|
|
40216
|
+
let selected;
|
|
40217
|
+
let algorithm;
|
|
40218
|
+
let seed;
|
|
40219
|
+
const hasTaskIds = "taskIds" in selector && selector.taskIds !== void 0;
|
|
40220
|
+
const hasSampleSize = "sampleSize" in selector && selector.sampleSize !== void 0;
|
|
40221
|
+
if (Number(hasTaskIds) + Number(hasSampleSize) !== 1) {
|
|
40222
|
+
throw new Error("Choose exactly one LoCoMo retrieval trace selector.");
|
|
40223
|
+
}
|
|
40224
|
+
if (hasTaskIds && "seed" in selector && selector.seed !== void 0) {
|
|
40225
|
+
throw new Error("LoCoMo retrieval trace seed is valid only for seeded sampling.");
|
|
40226
|
+
}
|
|
40227
|
+
if (hasTaskIds) {
|
|
40228
|
+
algorithm = "explicit-task-ids";
|
|
40229
|
+
const requestedTaskIds = selector.taskIds;
|
|
40230
|
+
if (!requestedTaskIds) throw new Error("LoCoMo explicit task ids are required.");
|
|
40231
|
+
const requested = [...requestedTaskIds];
|
|
40232
|
+
if (requested.length === 0) {
|
|
40233
|
+
throw new Error("LoCoMo retrieval trace explicit task selection cannot be empty.");
|
|
40234
|
+
}
|
|
40235
|
+
if (new Set(requested).size !== requested.length) {
|
|
40236
|
+
throw new Error("LoCoMo retrieval trace explicit task ids must not contain duplicates.");
|
|
40237
|
+
}
|
|
40238
|
+
const available = new Set(allIds);
|
|
40239
|
+
const unknown = requested.filter((taskId) => !available.has(taskId));
|
|
40240
|
+
if (unknown.length > 0) {
|
|
40241
|
+
throw new Error(`Unknown LoCoMo retrieval trace task id: ${unknown[0]}`);
|
|
40242
|
+
}
|
|
40243
|
+
const requestedSet = new Set(requested);
|
|
40244
|
+
selected = allIds.filter((taskId) => requestedSet.has(taskId));
|
|
40245
|
+
} else {
|
|
40246
|
+
algorithm = "sha256-seeded-sample";
|
|
40247
|
+
const sampleSize = selector.sampleSize;
|
|
40248
|
+
seed = selector.seed;
|
|
40249
|
+
if (sampleSize === void 0 || seed === void 0) {
|
|
40250
|
+
throw new Error("LoCoMo seeded sampling requires sampleSize and seed.");
|
|
40251
|
+
}
|
|
40252
|
+
if (!Number.isSafeInteger(sampleSize) || sampleSize <= 0 || sampleSize > allIds.length) {
|
|
40253
|
+
throw new Error(`LoCoMo retrieval trace sampleSize must be an integer from 1 to ${allIds.length}.`);
|
|
40254
|
+
}
|
|
40255
|
+
if (!Number.isSafeInteger(seed) || seed < 0) {
|
|
40256
|
+
throw new Error("LoCoMo retrieval trace seed must be a non-negative safe integer.");
|
|
40257
|
+
}
|
|
40258
|
+
const sampled = [...allIds].sort((left, right) => {
|
|
40259
|
+
const leftHash = hashString(`${seed}\0${left}`);
|
|
40260
|
+
const rightHash = hashString(`${seed}\0${right}`);
|
|
40261
|
+
return leftHash.localeCompare(rightHash) || left.localeCompare(right);
|
|
40262
|
+
}).slice(0, sampleSize);
|
|
40263
|
+
const sampledSet = new Set(sampled);
|
|
40264
|
+
selected = allIds.filter((taskId) => sampledSet.has(taskId));
|
|
40265
|
+
}
|
|
40266
|
+
return {
|
|
40267
|
+
algorithm,
|
|
40268
|
+
version: LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
|
|
40269
|
+
...seed === void 0 ? {} : { seed },
|
|
40270
|
+
candidateCount: allIds.length,
|
|
40271
|
+
selectedCount: selected.length,
|
|
40272
|
+
selectedTaskIds: selected,
|
|
40273
|
+
selectedTaskIdsSha256: hashCanonicalJson(selected)
|
|
40274
|
+
};
|
|
40275
|
+
}
|
|
40276
|
+
function serializeLoCoMoRetrievalTraceReceipt(receipt) {
|
|
40277
|
+
return `${canonicalJsonStringify(receipt, 2)}
|
|
40278
|
+
`;
|
|
40279
|
+
}
|
|
40280
|
+
function sanitizeStructuralTrace(trace2) {
|
|
40281
|
+
return {
|
|
40282
|
+
schemaVersion: trace2.schemaVersion,
|
|
40283
|
+
sensitivity: { ...trace2.sensitivity },
|
|
40284
|
+
sections: trace2.sections.map((section) => ({ ...section })),
|
|
40285
|
+
selections: trace2.selections.map(({ summary, ...selection }) => ({
|
|
40286
|
+
...selection,
|
|
40287
|
+
...selection.archiveRowIds === void 0 ? {} : { archiveRowIds: [...selection.archiveRowIds] },
|
|
40288
|
+
...summary === void 0 ? {} : { summary: { depth: summary.depth, msgStart: summary.msgStart, msgEnd: summary.msgEnd } }
|
|
40289
|
+
})),
|
|
40290
|
+
lcmCandidates: trace2.lcmCandidates.map((candidate) => ({ ...candidate })),
|
|
40291
|
+
...trace2.coreCapture === void 0 ? {} : {
|
|
40292
|
+
coreCapture: {
|
|
40293
|
+
budget: { ...trace2.coreCapture.budget },
|
|
40294
|
+
filters: trace2.coreCapture.filters.map((filter) => ({ ...filter })),
|
|
40295
|
+
results: trace2.coreCapture.results.map((result) => {
|
|
40296
|
+
assertMemoryIdRef(result.memoryIdRef);
|
|
40297
|
+
const score = result.scoreDecomposition;
|
|
40298
|
+
return {
|
|
40299
|
+
memoryIdRef: {
|
|
40300
|
+
sha256: result.memoryIdRef.sha256,
|
|
40301
|
+
length: result.memoryIdRef.length
|
|
40302
|
+
},
|
|
40303
|
+
servedBy: result.servedBy,
|
|
40304
|
+
scoreDecomposition: {
|
|
40305
|
+
...score.vector === void 0 ? {} : { vector: score.vector },
|
|
40306
|
+
...score.bm25 === void 0 ? {} : { bm25: score.bm25 },
|
|
40307
|
+
...score.importance === void 0 ? {} : { importance: score.importance },
|
|
40308
|
+
...score.mmrPenalty === void 0 ? {} : { mmrPenalty: score.mmrPenalty },
|
|
40309
|
+
...score.tierPrior === void 0 ? {} : { tierPrior: score.tierPrior },
|
|
40310
|
+
...score.reinforcementBoost === void 0 ? {} : { reinforcementBoost: score.reinforcementBoost },
|
|
40311
|
+
final: score.final
|
|
40312
|
+
},
|
|
40313
|
+
admittedBy: [...result.admittedBy],
|
|
40314
|
+
...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
|
|
40315
|
+
...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
|
|
40316
|
+
...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
|
|
40317
|
+
};
|
|
40318
|
+
})
|
|
40319
|
+
}
|
|
40320
|
+
},
|
|
40321
|
+
budget: { ...trace2.budget }
|
|
40322
|
+
};
|
|
40323
|
+
}
|
|
40324
|
+
function digestContent(value) {
|
|
40325
|
+
return {
|
|
40326
|
+
sha256: hashString(value),
|
|
40327
|
+
charCount: value.length,
|
|
40328
|
+
lineCount: value.length === 0 ? 0 : value.split("\n").length
|
|
40329
|
+
};
|
|
40330
|
+
}
|
|
40331
|
+
function assertCaptureOptions(options) {
|
|
40332
|
+
if (!options.datasetDir.trim()) {
|
|
40333
|
+
throw new Error("LoCoMo retrieval trace capture requires datasetDir.");
|
|
40334
|
+
}
|
|
40335
|
+
if (options.runtimeProfile !== "baseline" && options.runtimeProfile !== "real") {
|
|
40336
|
+
throw new Error('LoCoMo retrieval trace runtimeProfile must be "baseline" or "real".');
|
|
40337
|
+
}
|
|
40338
|
+
if (!options.gitSha.trim() || !options.remnicVersion.trim() || options.gitSha === "unknown" || options.remnicVersion === "unknown") {
|
|
40339
|
+
throw new Error("LoCoMo retrieval trace provenance requires gitSha and remnicVersion.");
|
|
40340
|
+
}
|
|
40341
|
+
if (options.providerFreeConfirmed !== true) {
|
|
40342
|
+
throw new Error("LoCoMo retrieval trace capture requires explicit provider-free confirmation.");
|
|
40343
|
+
}
|
|
40344
|
+
}
|
|
40345
|
+
function assertProviderFreeRetrievalConfig(value) {
|
|
40346
|
+
const config = assertJsonConfig(value);
|
|
40347
|
+
for (const key of [
|
|
40348
|
+
"localLlmEnabled",
|
|
40349
|
+
"localLlmFastEnabled",
|
|
40350
|
+
"recallPlannerEnabled",
|
|
40351
|
+
"embeddingFallbackEnabled",
|
|
40352
|
+
"hostEmbeddingProviderEnabled",
|
|
40353
|
+
"openaiApiKey"
|
|
40354
|
+
]) {
|
|
40355
|
+
if (config[key] !== false) {
|
|
40356
|
+
throw new Error(`retrievalConfig.${key} must be false for provider-free capture.`);
|
|
40357
|
+
}
|
|
40358
|
+
}
|
|
40359
|
+
if (config.modelSource !== "plugin") {
|
|
40360
|
+
throw new Error('retrievalConfig.modelSource must be "plugin" for provider-free capture.');
|
|
40361
|
+
}
|
|
40362
|
+
return config;
|
|
40363
|
+
}
|
|
40364
|
+
function assertMemoryIdRef(value) {
|
|
40365
|
+
if (!value || typeof value !== "object" || !/^[0-9a-f]{64}$/u.test(value.sha256) || !Number.isSafeInteger(value.length) || value.length <= 0) {
|
|
40366
|
+
throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
|
|
40367
|
+
}
|
|
40368
|
+
}
|
|
40369
|
+
function assertJsonConfig(value, path40 = "retrievalConfig") {
|
|
40370
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
40371
|
+
if (typeof value === "number") {
|
|
40372
|
+
if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
|
|
40373
|
+
return value;
|
|
40374
|
+
}
|
|
40375
|
+
if (Array.isArray(value)) {
|
|
40376
|
+
return value.map((entry, index) => assertJsonConfig(entry, `${path40}[${index}]`));
|
|
40377
|
+
}
|
|
40378
|
+
if (!value || typeof value !== "object") {
|
|
40379
|
+
throw new Error(`${path40} must be JSON-serializable and provider-free.`);
|
|
40380
|
+
}
|
|
40381
|
+
const output = {};
|
|
40382
|
+
for (const key of Object.keys(value).sort()) {
|
|
40383
|
+
const child = value[key];
|
|
40384
|
+
if (key === "openaiApiKey") {
|
|
40385
|
+
if (child !== false) {
|
|
40386
|
+
throw new Error(`${path40}.${key} must be exactly false for provider-free capture.`);
|
|
40387
|
+
}
|
|
40388
|
+
output[key] = false;
|
|
40389
|
+
continue;
|
|
40390
|
+
}
|
|
40391
|
+
if (isSecretKey(key)) {
|
|
40392
|
+
throw new Error(`${path40}.${key} contains secret-bearing configuration.`);
|
|
40393
|
+
}
|
|
40394
|
+
if (child === void 0) continue;
|
|
40395
|
+
if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
|
|
40396
|
+
throw new Error(`${path40}.${key} is provider-capable configuration.`);
|
|
40397
|
+
}
|
|
40398
|
+
output[key] = assertJsonConfig(child, `${path40}.${key}`);
|
|
40399
|
+
}
|
|
40400
|
+
return output;
|
|
40401
|
+
}
|
|
40402
|
+
function sanitizeProviderFreeRetrievalConfig(value, path40 = "retrievalConfig") {
|
|
40403
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
40404
|
+
if (typeof value === "number") {
|
|
40405
|
+
if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
|
|
40406
|
+
return value;
|
|
40407
|
+
}
|
|
40408
|
+
if (Array.isArray(value)) {
|
|
40409
|
+
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path40}[${index}]`));
|
|
40410
|
+
}
|
|
40411
|
+
if (!value || typeof value !== "object") {
|
|
40412
|
+
throw new Error(`${path40} must be JSON-serializable.`);
|
|
40413
|
+
}
|
|
40414
|
+
const output = {};
|
|
40415
|
+
for (const key of Object.keys(value).sort()) {
|
|
40416
|
+
const child = value[key];
|
|
40417
|
+
if (child === void 0) continue;
|
|
40418
|
+
if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
|
|
40419
|
+
continue;
|
|
40420
|
+
}
|
|
40421
|
+
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path40}.${key}`);
|
|
40422
|
+
}
|
|
40423
|
+
return output;
|
|
40424
|
+
}
|
|
40425
|
+
|
|
39197
40426
|
// src/integrity/sealed-qrels.ts
|
|
39198
40427
|
import { readFile as readFile20 } from "fs/promises";
|
|
39199
40428
|
function isSealedQrelsArtifact(value) {
|
|
@@ -43064,6 +44293,10 @@ export {
|
|
|
43064
44293
|
LOCOMO_FULL_TASK_COUNT,
|
|
43065
44294
|
LOCOMO_RECALL_DIFF_LINE_LIMIT,
|
|
43066
44295
|
LOCOMO_RECALL_EXCERPT_CHARS,
|
|
44296
|
+
LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION,
|
|
44297
|
+
LOCOMO_RETRIEVAL_TRACE_DELTA_SCHEMA_VERSION,
|
|
44298
|
+
LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
|
|
44299
|
+
LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
|
|
43067
44300
|
LONG_MEM_EVAL_DATASET_FILENAMES,
|
|
43068
44301
|
LettaMemCorrectAdapter,
|
|
43069
44302
|
LocalLabPreflightError,
|
|
@@ -43120,10 +44353,12 @@ export {
|
|
|
43120
44353
|
buildCodexCreditReceipt,
|
|
43121
44354
|
buildJudgePayload,
|
|
43122
44355
|
buildOracleTrajectoryRecall,
|
|
44356
|
+
buildProviderFreeLoCoMoRetrievalConfig,
|
|
43123
44357
|
buildSchemaTierFixture,
|
|
43124
44358
|
buildSchemaTierSmokeFixture,
|
|
43125
44359
|
calendarFixture,
|
|
43126
44360
|
canonicalJsonStringify,
|
|
44361
|
+
captureLoCoMoRetrievalTrace,
|
|
43127
44362
|
captureMachineFingerprint,
|
|
43128
44363
|
chatFixture,
|
|
43129
44364
|
checkCodingGraphRegression,
|
|
@@ -43174,6 +44409,7 @@ export {
|
|
|
43174
44409
|
defaultBenchmarkBaselineDir,
|
|
43175
44410
|
defaultBenchmarkPublishPath,
|
|
43176
44411
|
deleteBenchmarkResults,
|
|
44412
|
+
diagnoseLoCoMoRetrievalTraceDelta,
|
|
43177
44413
|
diagnoseLoComoProfileDelta,
|
|
43178
44414
|
diagnoseLoComoRecallDelta,
|
|
43179
44415
|
discoverAllProviders,
|
|
@@ -43192,6 +44428,7 @@ export {
|
|
|
43192
44428
|
getAblationCell,
|
|
43193
44429
|
getBenchmark,
|
|
43194
44430
|
getBenchmarkLowerIsBetter,
|
|
44431
|
+
getGitSha,
|
|
43195
44432
|
getMemoryEvalDimension,
|
|
43196
44433
|
getProviderBackedJudgePromptIdentity,
|
|
43197
44434
|
getRemnicVersion,
|
|
@@ -43245,6 +44482,7 @@ export {
|
|
|
43245
44482
|
parseSealedQrels,
|
|
43246
44483
|
pickStableQualifiedName,
|
|
43247
44484
|
precisionAtK,
|
|
44485
|
+
preflightLoCoMoRetrievalTraceCapture,
|
|
43248
44486
|
preflightLocalLabRole,
|
|
43249
44487
|
projectFolderFixture,
|
|
43250
44488
|
recallAtK,
|
|
@@ -43300,6 +44538,8 @@ export {
|
|
|
43300
44538
|
selectFixtureVariant,
|
|
43301
44539
|
serializeBenchmarkArtifact,
|
|
43302
44540
|
serializeJsonl,
|
|
44541
|
+
serializeLoCoMoRetrievalTraceDelta,
|
|
44542
|
+
serializeLoCoMoRetrievalTraceReceipt,
|
|
43303
44543
|
serializeSealedQrels,
|
|
43304
44544
|
shuffleTasks,
|
|
43305
44545
|
timed,
|