@remnic/bench 9.6.31 → 9.6.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +254 -16
- package/dist/index.js +948 -130
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -597,7 +597,7 @@ var LettaMemCorrectAdapter = class {
|
|
|
597
597
|
var REQUIRED_FRONTMATTER_FIELDS = ["title", "type", "state", "created", "see-also"];
|
|
598
598
|
|
|
599
599
|
// src/adapters/remnic-adapter.ts
|
|
600
|
-
import { createHash } from "crypto";
|
|
600
|
+
import { createHash as createHash2 } from "crypto";
|
|
601
601
|
import { execFile } from "child_process";
|
|
602
602
|
import {
|
|
603
603
|
chmod,
|
|
@@ -629,6 +629,226 @@ import {
|
|
|
629
629
|
parseEntityFile,
|
|
630
630
|
serializeEntityFile
|
|
631
631
|
} from "@remnic/core";
|
|
632
|
+
import {
|
|
633
|
+
lcmEvidenceIdentity
|
|
634
|
+
} from "@remnic/core/lcm";
|
|
635
|
+
|
|
636
|
+
// src/adapters/remnic-recall-trace.ts
|
|
637
|
+
import { createHash } from "crypto";
|
|
638
|
+
import {
|
|
639
|
+
lcmArchiveRowId
|
|
640
|
+
} from "@remnic/core/lcm";
|
|
641
|
+
function visibleRange(start, end, returnedChars) {
|
|
642
|
+
const visibleStart = Math.min(start, returnedChars);
|
|
643
|
+
return {
|
|
644
|
+
visibleStart,
|
|
645
|
+
visibleEnd: Math.max(visibleStart, Math.min(end, returnedChars))
|
|
646
|
+
};
|
|
647
|
+
}
|
|
648
|
+
function projectBenchCoreCapture(snapshot) {
|
|
649
|
+
return {
|
|
650
|
+
snapshotId: snapshot.snapshotId,
|
|
651
|
+
capturedAt: snapshot.capturedAt,
|
|
652
|
+
...snapshot.traceId === void 0 ? {} : { traceId: snapshot.traceId },
|
|
653
|
+
budget: { chars: snapshot.budget.chars, used: snapshot.budget.used },
|
|
654
|
+
filters: snapshot.filters.map(({ name, considered, admitted }) => ({
|
|
655
|
+
name,
|
|
656
|
+
considered,
|
|
657
|
+
admitted
|
|
658
|
+
})),
|
|
659
|
+
results: snapshot.results.map((result) => {
|
|
660
|
+
const scores = result.scoreDecomposition;
|
|
661
|
+
return {
|
|
662
|
+
memoryIdRef: {
|
|
663
|
+
sha256: createHash("sha256").update(result.memoryId, "utf8").digest("hex"),
|
|
664
|
+
length: Buffer.byteLength(result.memoryId, "utf8")
|
|
665
|
+
},
|
|
666
|
+
servedBy: result.servedBy,
|
|
667
|
+
scoreDecomposition: {
|
|
668
|
+
...typeof scores.vector === "number" ? { vector: scores.vector } : {},
|
|
669
|
+
...typeof scores.bm25 === "number" ? { bm25: scores.bm25 } : {},
|
|
670
|
+
...typeof scores.importance === "number" ? { importance: scores.importance } : {},
|
|
671
|
+
...typeof scores.mmrPenalty === "number" ? { mmrPenalty: scores.mmrPenalty } : {},
|
|
672
|
+
...typeof scores.tierPrior === "number" ? { tierPrior: scores.tierPrior } : {},
|
|
673
|
+
...typeof scores.reinforcementBoost === "number" ? { reinforcementBoost: scores.reinforcementBoost } : {},
|
|
674
|
+
final: scores.final
|
|
675
|
+
},
|
|
676
|
+
admittedBy: [...result.admittedBy],
|
|
677
|
+
...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
|
|
678
|
+
...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
|
|
679
|
+
...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
|
|
680
|
+
};
|
|
681
|
+
})
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
function createBenchRecallTraceRecorder(requestedChars) {
|
|
685
|
+
const sections = [];
|
|
686
|
+
const pendingSelections = [];
|
|
687
|
+
const lcmCandidates = [];
|
|
688
|
+
let composedChars = 0;
|
|
689
|
+
let coreCapture;
|
|
690
|
+
const sectionById = (sectionId) => {
|
|
691
|
+
const section = sections.find((entry) => entry.id === sectionId);
|
|
692
|
+
if (!section) throw new Error(`Unknown benchmark recall trace section: ${sectionId}`);
|
|
693
|
+
return section;
|
|
694
|
+
};
|
|
695
|
+
const appendRelativeSelection = (sectionId, kind, start, end, lineageStatus, fields = {}) => {
|
|
696
|
+
const section = sectionById(sectionId);
|
|
697
|
+
const contentLength = section.contentEnd - section.contentStart;
|
|
698
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > contentLength) {
|
|
699
|
+
throw new Error(
|
|
700
|
+
`Invalid benchmark recall trace range for ${sectionId}: ${start}..${end}.`
|
|
701
|
+
);
|
|
702
|
+
}
|
|
703
|
+
pendingSelections.push({
|
|
704
|
+
sectionId,
|
|
705
|
+
kind,
|
|
706
|
+
lineageStatus,
|
|
707
|
+
composedStart: section.contentStart + start,
|
|
708
|
+
composedEnd: section.contentStart + end,
|
|
709
|
+
...fields
|
|
710
|
+
});
|
|
711
|
+
};
|
|
712
|
+
return {
|
|
713
|
+
appendSection(id, source, renderedLength) {
|
|
714
|
+
if (!Number.isSafeInteger(renderedLength) || renderedLength < 0) {
|
|
715
|
+
throw new Error("Benchmark recall trace section length must be a non-negative integer.");
|
|
716
|
+
}
|
|
717
|
+
if (sections.some((section) => section.id === id)) {
|
|
718
|
+
throw new Error(`Duplicate benchmark recall trace section: ${id}`);
|
|
719
|
+
}
|
|
720
|
+
const separatorStart = composedChars;
|
|
721
|
+
const contentStart = composedChars + (sections.length === 0 ? 0 : 2);
|
|
722
|
+
const contentEnd = contentStart + renderedLength;
|
|
723
|
+
sections.push({
|
|
724
|
+
id,
|
|
725
|
+
source,
|
|
726
|
+
separatorStart,
|
|
727
|
+
contentStart,
|
|
728
|
+
contentEnd,
|
|
729
|
+
composedStart: separatorStart,
|
|
730
|
+
composedEnd: contentEnd,
|
|
731
|
+
visibleStart: 0,
|
|
732
|
+
visibleEnd: 0,
|
|
733
|
+
visibleChars: 0
|
|
734
|
+
});
|
|
735
|
+
composedChars = contentEnd;
|
|
736
|
+
},
|
|
737
|
+
recordEvidenceSelections(sectionId, receipts) {
|
|
738
|
+
for (const receipt of receipts) {
|
|
739
|
+
appendRelativeSelection(
|
|
740
|
+
sectionId,
|
|
741
|
+
"evidence-block",
|
|
742
|
+
receipt.blockStart,
|
|
743
|
+
receipt.blockEnd,
|
|
744
|
+
receipt.item.archiveRowId === void 0 ? "unavailable" : "exact",
|
|
745
|
+
{
|
|
746
|
+
...receipt.item.archiveRowId === void 0 ? {} : { archiveRowIds: [receipt.item.archiveRowId] },
|
|
747
|
+
...receipt.item.turnIndex === void 0 ? {} : { turnIndex: receipt.item.turnIndex },
|
|
748
|
+
...receipt.item.role === void 0 ? {} : { role: receipt.item.role },
|
|
749
|
+
...receipt.item.score === void 0 ? {} : { score: receipt.item.score }
|
|
750
|
+
}
|
|
751
|
+
);
|
|
752
|
+
}
|
|
753
|
+
},
|
|
754
|
+
recordTrajectorySelections(sectionId, receipts) {
|
|
755
|
+
for (const receipt of receipts) {
|
|
756
|
+
appendRelativeSelection(
|
|
757
|
+
sectionId,
|
|
758
|
+
"trajectory-line",
|
|
759
|
+
receipt.lineStart,
|
|
760
|
+
receipt.lineEnd,
|
|
761
|
+
receipt.lineageStatus,
|
|
762
|
+
{
|
|
763
|
+
archiveRowIds: [
|
|
764
|
+
...receipt.actionArchiveRowIds,
|
|
765
|
+
...receipt.observationArchiveRowIds
|
|
766
|
+
]
|
|
767
|
+
}
|
|
768
|
+
);
|
|
769
|
+
}
|
|
770
|
+
},
|
|
771
|
+
recordSummarySelections(sectionId, receipts) {
|
|
772
|
+
for (const receipt of receipts) {
|
|
773
|
+
appendRelativeSelection(
|
|
774
|
+
sectionId,
|
|
775
|
+
"lcm-summary",
|
|
776
|
+
receipt.entryStart,
|
|
777
|
+
receipt.entryEnd,
|
|
778
|
+
"exact",
|
|
779
|
+
{
|
|
780
|
+
summary: {
|
|
781
|
+
id: receipt.id,
|
|
782
|
+
depth: receipt.depth,
|
|
783
|
+
msgStart: receipt.msgStart,
|
|
784
|
+
msgEnd: receipt.msgEnd
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
},
|
|
790
|
+
recordRawRow(sectionId, range, row) {
|
|
791
|
+
const archiveRowId = lcmArchiveRowId(row);
|
|
792
|
+
appendRelativeSelection(
|
|
793
|
+
sectionId,
|
|
794
|
+
"raw-row",
|
|
795
|
+
range.start,
|
|
796
|
+
range.end,
|
|
797
|
+
archiveRowId === void 0 ? "unavailable" : "exact",
|
|
798
|
+
{
|
|
799
|
+
...archiveRowId === void 0 ? {} : { archiveRowIds: [archiveRowId] },
|
|
800
|
+
turnIndex: row.turn_index,
|
|
801
|
+
role: row.role
|
|
802
|
+
}
|
|
803
|
+
);
|
|
804
|
+
},
|
|
805
|
+
recordLcmCandidate(candidate) {
|
|
806
|
+
lcmCandidates.push({ ...candidate });
|
|
807
|
+
},
|
|
808
|
+
recordCoreCapture(snapshot) {
|
|
809
|
+
coreCapture = snapshot ? projectBenchCoreCapture(snapshot) : void 0;
|
|
810
|
+
},
|
|
811
|
+
finalize(returnedChars) {
|
|
812
|
+
const normalizedReturnedChars = Math.max(0, Math.min(returnedChars, composedChars));
|
|
813
|
+
return {
|
|
814
|
+
schemaVersion: 1,
|
|
815
|
+
sensitivity: {
|
|
816
|
+
classification: "restricted",
|
|
817
|
+
contentEncoding: "sha256+length",
|
|
818
|
+
containsGold: false
|
|
819
|
+
},
|
|
820
|
+
sections: sections.map((section) => {
|
|
821
|
+
const visible = visibleRange(
|
|
822
|
+
section.composedStart,
|
|
823
|
+
section.composedEnd,
|
|
824
|
+
normalizedReturnedChars
|
|
825
|
+
);
|
|
826
|
+
return {
|
|
827
|
+
...section,
|
|
828
|
+
...visible,
|
|
829
|
+
visibleChars: visible.visibleEnd - visible.visibleStart
|
|
830
|
+
};
|
|
831
|
+
}),
|
|
832
|
+
selections: pendingSelections.map((selection) => ({
|
|
833
|
+
...selection,
|
|
834
|
+
...visibleRange(
|
|
835
|
+
selection.composedStart,
|
|
836
|
+
selection.composedEnd,
|
|
837
|
+
normalizedReturnedChars
|
|
838
|
+
)
|
|
839
|
+
})),
|
|
840
|
+
lcmCandidates: lcmCandidates.map((candidate) => ({ ...candidate })),
|
|
841
|
+
...coreCapture === void 0 ? {} : { coreCapture },
|
|
842
|
+
budget: {
|
|
843
|
+
requestedChars,
|
|
844
|
+
composedChars,
|
|
845
|
+
returnedChars: normalizedReturnedChars,
|
|
846
|
+
truncated: normalizedReturnedChars < composedChars
|
|
847
|
+
}
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
};
|
|
851
|
+
}
|
|
632
852
|
|
|
633
853
|
// src/recall-budget.ts
|
|
634
854
|
var DEFAULT_BENCH_RECALL_BUDGET_CHARS = 24e3;
|
|
@@ -1264,7 +1484,7 @@ function benchCoreMemoryTier(memory) {
|
|
|
1264
1484
|
return memory.path.includes(`${path.sep}cold${path.sep}`) ? "cold" : "hot";
|
|
1265
1485
|
}
|
|
1266
1486
|
function benchCoreMemorySource(sessionId) {
|
|
1267
|
-
return `bench-replay-${
|
|
1487
|
+
return `bench-replay-${createHash2("sha256").update(sessionId).digest("hex").slice(0, 16)}`;
|
|
1268
1488
|
}
|
|
1269
1489
|
function resolveSessionScopedCorrectionDecision(plan, ownedIds, phase) {
|
|
1270
1490
|
const actionTargets = [];
|
|
@@ -2009,7 +2229,8 @@ function createAdapterFactory(mode) {
|
|
|
2009
2229
|
}
|
|
2010
2230
|
return correctionAccess.service;
|
|
2011
2231
|
};
|
|
2012
|
-
|
|
2232
|
+
const composeRecall = /* @__PURE__ */ Symbol("benchRecallComposer");
|
|
2233
|
+
const adapter = {
|
|
2013
2234
|
async store(sessionId, messages, control) {
|
|
2014
2235
|
throwIfBenchPhaseAborted(control, "store");
|
|
2015
2236
|
sessionId = normalizeBenchSessionId(sessionId);
|
|
@@ -2128,7 +2349,7 @@ function createAdapterFactory(mode) {
|
|
|
2128
2349
|
throw error;
|
|
2129
2350
|
}
|
|
2130
2351
|
},
|
|
2131
|
-
async
|
|
2352
|
+
async [composeRecall](sessionId, query, budgetChars, recallOptions = {}, control, traceRecorder) {
|
|
2132
2353
|
throwIfBenchPhaseAborted(control, "recall");
|
|
2133
2354
|
const waitForRecall = (promise) => withBenchPhaseAbort(promise, control, "recall");
|
|
2134
2355
|
sessionId = normalizeBenchSessionId(sessionId);
|
|
@@ -2150,6 +2371,10 @@ function createAdapterFactory(mode) {
|
|
|
2150
2371
|
);
|
|
2151
2372
|
}
|
|
2152
2373
|
const sections = [];
|
|
2374
|
+
const appendSection = (id, source, rendered) => {
|
|
2375
|
+
traceRecorder?.appendSection(id, source, rendered.length);
|
|
2376
|
+
sections.push(rendered);
|
|
2377
|
+
};
|
|
2153
2378
|
let usedChars = 0;
|
|
2154
2379
|
const explicitReferences = historicalRecall ? [] : collectExplicitTurnReferences(query);
|
|
2155
2380
|
const hasExplicitReferences = explicitReferences.length > 0;
|
|
@@ -2175,7 +2400,7 @@ function createAdapterFactory(mode) {
|
|
|
2175
2400
|
}));
|
|
2176
2401
|
if (temporalIntervalEvidence) {
|
|
2177
2402
|
hasTemporalIntervalEvidence = true;
|
|
2178
|
-
|
|
2403
|
+
appendSection("temporal-interval", "derived", temporalIntervalEvidence);
|
|
2179
2404
|
usedChars += temporalIntervalEvidence.length;
|
|
2180
2405
|
}
|
|
2181
2406
|
}
|
|
@@ -2187,7 +2412,7 @@ function createAdapterFactory(mode) {
|
|
|
2187
2412
|
}));
|
|
2188
2413
|
if (dependencyVersionEvidence) {
|
|
2189
2414
|
hasDependencyVersionEvidence = true;
|
|
2190
|
-
|
|
2415
|
+
appendSection("dependency-version", "derived", dependencyVersionEvidence);
|
|
2191
2416
|
usedChars += dependencyVersionEvidence.length;
|
|
2192
2417
|
}
|
|
2193
2418
|
}
|
|
@@ -2199,7 +2424,7 @@ function createAdapterFactory(mode) {
|
|
|
2199
2424
|
maxChars: Math.min(3e3, Math.floor(budget * 0.25))
|
|
2200
2425
|
}));
|
|
2201
2426
|
if (latestQuantitativeEvidence) {
|
|
2202
|
-
|
|
2427
|
+
appendSection("latest-quantitative", "derived", latestQuantitativeEvidence);
|
|
2203
2428
|
usedChars += latestQuantitativeEvidence.length;
|
|
2204
2429
|
}
|
|
2205
2430
|
}
|
|
@@ -2211,10 +2436,11 @@ function createAdapterFactory(mode) {
|
|
|
2211
2436
|
}));
|
|
2212
2437
|
if (userImplementationTargetEvidence) {
|
|
2213
2438
|
hasUserImplementationTargetEvidence = true;
|
|
2214
|
-
|
|
2439
|
+
appendSection("implementation-targets", "derived", userImplementationTargetEvidence);
|
|
2215
2440
|
usedChars += userImplementationTargetEvidence.length;
|
|
2216
2441
|
}
|
|
2217
2442
|
}
|
|
2443
|
+
const explicitCueSelections = [];
|
|
2218
2444
|
const exactReferenceEvidence = historicalRecall || hasDependencyVersionEvidence ? "" : await waitForRecall(buildExplicitCueRecallSection({
|
|
2219
2445
|
engine,
|
|
2220
2446
|
sessionId,
|
|
@@ -2223,12 +2449,17 @@ function createAdapterFactory(mode) {
|
|
|
2223
2449
|
maxItemChars: CORE_EXPLICIT_CUE_MAX_ITEM_CHARS,
|
|
2224
2450
|
maxReferences: CORE_EXPLICIT_CUE_MAX_REFERENCES,
|
|
2225
2451
|
includeBenchmarkAnchorCues: sessionId.startsWith("beam-"),
|
|
2226
|
-
includeStructuredPlanCues: sessionId.startsWith("arena-")
|
|
2452
|
+
includeStructuredPlanCues: sessionId.startsWith("arena-"),
|
|
2453
|
+
...traceRecorder ? { onEvidenceSelected: (receipt) => {
|
|
2454
|
+
explicitCueSelections.push(receipt);
|
|
2455
|
+
} } : {}
|
|
2227
2456
|
}));
|
|
2228
2457
|
if (exactReferenceEvidence) {
|
|
2229
|
-
|
|
2458
|
+
appendSection("explicit-cue", "explicit-cue", exactReferenceEvidence);
|
|
2459
|
+
traceRecorder?.recordEvidenceSelections("explicit-cue", explicitCueSelections);
|
|
2230
2460
|
usedChars += exactReferenceEvidence.length;
|
|
2231
2461
|
}
|
|
2462
|
+
const trajectorySelections = [];
|
|
2232
2463
|
const trajectoryAnalysisEvidence = !historicalRecall && sessionId.startsWith("ama-") ? await waitForRecall(buildTrajectoryAnalysisRecallSection({
|
|
2233
2464
|
engine,
|
|
2234
2465
|
sessionId,
|
|
@@ -2236,10 +2467,17 @@ function createAdapterFactory(mode) {
|
|
|
2236
2467
|
maxChars: Math.min(
|
|
2237
2468
|
CORE_TRAJECTORY_ANALYSIS_MAX_CHARS,
|
|
2238
2469
|
Math.max(0, Math.floor((budget - usedChars) * 0.55))
|
|
2239
|
-
)
|
|
2470
|
+
),
|
|
2471
|
+
...traceRecorder ? { onLineSelected: (receipt) => {
|
|
2472
|
+
trajectorySelections.push(receipt);
|
|
2473
|
+
} } : {}
|
|
2240
2474
|
})) : "";
|
|
2241
2475
|
if (trajectoryAnalysisEvidence) {
|
|
2242
|
-
|
|
2476
|
+
appendSection("trajectory-analysis", "trajectory-analysis", trajectoryAnalysisEvidence);
|
|
2477
|
+
traceRecorder?.recordTrajectorySelections(
|
|
2478
|
+
"trajectory-analysis",
|
|
2479
|
+
trajectorySelections
|
|
2480
|
+
);
|
|
2243
2481
|
usedChars += trajectoryAnalysisEvidence.length;
|
|
2244
2482
|
}
|
|
2245
2483
|
if (includeCoreRecall && !requireDirectPersonalHistoryEvidence && !requireDirectTemporalEvidence && !hasTemporalIntervalEvidence && !hasDependencyVersionEvidence && !hasUserImplementationTargetEvidence) {
|
|
@@ -2252,17 +2490,27 @@ function createAdapterFactory(mode) {
|
|
|
2252
2490
|
)
|
|
2253
2491
|
)
|
|
2254
2492
|
);
|
|
2255
|
-
const
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2493
|
+
const coreOptions = {
|
|
2494
|
+
budgetCharsOverride: coreBudget,
|
|
2495
|
+
mode: "full",
|
|
2496
|
+
...control?.signal ? { abortSignal: control.signal } : {},
|
|
2497
|
+
...recallAsOf ? { asOf: recallAsOf } : {}
|
|
2498
|
+
};
|
|
2499
|
+
const coreRecall = traceRecorder ? await withBenchPhaseAbort(
|
|
2500
|
+
state.orchestrator.recallWithXrayCapture(query, sessionId, coreOptions),
|
|
2501
|
+
control,
|
|
2502
|
+
"recall",
|
|
2503
|
+
{ waitForCompletionOnAbort: true }
|
|
2504
|
+
).then((capture) => {
|
|
2505
|
+
traceRecorder.recordCoreCapture(capture.snapshot);
|
|
2506
|
+
return capture.result;
|
|
2507
|
+
}) : await waitForRecall(
|
|
2508
|
+
state.orchestrator.recall(query, sessionId, coreOptions)
|
|
2261
2509
|
);
|
|
2262
2510
|
if (coreRecall.trim().length > 0) {
|
|
2263
2511
|
const section = `## Remnic recall pipeline
|
|
2264
2512
|
${coreRecall.trim()}`;
|
|
2265
|
-
|
|
2513
|
+
appendSection("core", "core", section);
|
|
2266
2514
|
usedChars += section.length;
|
|
2267
2515
|
}
|
|
2268
2516
|
}
|
|
@@ -2273,7 +2521,7 @@ ${coreRecall.trim()}`;
|
|
|
2273
2521
|
"## Remnic historical recall",
|
|
2274
2522
|
`No historically valid Remnic memories matched this query as of ${recallAsOf}.`
|
|
2275
2523
|
].join("\n");
|
|
2276
|
-
|
|
2524
|
+
appendSection("historical-empty", "derived", section);
|
|
2277
2525
|
usedChars += section.length;
|
|
2278
2526
|
}
|
|
2279
2527
|
}
|
|
@@ -2289,6 +2537,30 @@ ${coreRecall.trim()}`;
|
|
|
2289
2537
|
sessionId
|
|
2290
2538
|
)
|
|
2291
2539
|
);
|
|
2540
|
+
const exactSearchHits = /* @__PURE__ */ new Map();
|
|
2541
|
+
const uniqueLegacySearchHits = /* @__PURE__ */ new Map();
|
|
2542
|
+
const ambiguousLegacySearchHitIds = /* @__PURE__ */ new Set();
|
|
2543
|
+
searchResults.forEach((result, rank) => {
|
|
2544
|
+
const identity = lcmEvidenceIdentity(result, result.session_id);
|
|
2545
|
+
if (identity.archiveRowId !== void 0 && !exactSearchHits.has(identity.id)) {
|
|
2546
|
+
exactSearchHits.set(identity.id, result);
|
|
2547
|
+
} else if (identity.archiveRowId === void 0) {
|
|
2548
|
+
if (uniqueLegacySearchHits.has(identity.id)) {
|
|
2549
|
+
uniqueLegacySearchHits.delete(identity.id);
|
|
2550
|
+
ambiguousLegacySearchHitIds.add(identity.id);
|
|
2551
|
+
} else if (!ambiguousLegacySearchHitIds.has(identity.id)) {
|
|
2552
|
+
uniqueLegacySearchHits.set(identity.id, result);
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
traceRecorder?.recordLcmCandidate({
|
|
2556
|
+
rank: rank + 1,
|
|
2557
|
+
...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
|
|
2558
|
+
turnIndex: result.turn_index,
|
|
2559
|
+
role: result.role,
|
|
2560
|
+
...typeof result.score === "number" ? { score: result.score } : {},
|
|
2561
|
+
lineageStatus: identity.archiveRowId === void 0 ? "unavailable" : "exact"
|
|
2562
|
+
});
|
|
2563
|
+
});
|
|
2292
2564
|
if (searchResults.length > 0) {
|
|
2293
2565
|
const evidenceItems = [];
|
|
2294
2566
|
const directTemporalEvidenceItems = [];
|
|
@@ -2307,8 +2579,20 @@ ${coreRecall.trim()}`;
|
|
|
2307
2579
|
includeCoreRecall ? 1600 : 600
|
|
2308
2580
|
)
|
|
2309
2581
|
);
|
|
2582
|
+
const expandedIdentityCounts = /* @__PURE__ */ new Map();
|
|
2583
|
+
for (const message of expanded) {
|
|
2584
|
+
const expandedIdentity = lcmEvidenceIdentity(
|
|
2585
|
+
message,
|
|
2586
|
+
result.session_id
|
|
2587
|
+
);
|
|
2588
|
+
expandedIdentityCounts.set(
|
|
2589
|
+
expandedIdentity.id,
|
|
2590
|
+
(expandedIdentityCounts.get(expandedIdentity.id) ?? 0) + 1
|
|
2591
|
+
);
|
|
2592
|
+
}
|
|
2310
2593
|
if (expanded.length === 0) {
|
|
2311
|
-
const
|
|
2594
|
+
const identity = lcmEvidenceIdentity(result, result.session_id);
|
|
2595
|
+
const { id } = identity;
|
|
2312
2596
|
if (!directTemporalTurnIds.has(id) && shouldIncludeDirectTemporalEvidence(
|
|
2313
2597
|
result.content,
|
|
2314
2598
|
query,
|
|
@@ -2317,6 +2601,7 @@ ${coreRecall.trim()}`;
|
|
|
2317
2601
|
directTemporalTurnIds.add(id);
|
|
2318
2602
|
directTemporalEvidenceItems.push({
|
|
2319
2603
|
id,
|
|
2604
|
+
...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
|
|
2320
2605
|
sessionId: result.session_id,
|
|
2321
2606
|
turnIndex: result.turn_index,
|
|
2322
2607
|
role: result.role,
|
|
@@ -2336,6 +2621,7 @@ ${coreRecall.trim()}`;
|
|
|
2336
2621
|
seenTurns.add(id);
|
|
2337
2622
|
evidenceItems.push({
|
|
2338
2623
|
id,
|
|
2624
|
+
...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
|
|
2339
2625
|
sessionId: result.session_id,
|
|
2340
2626
|
turnIndex: result.turn_index,
|
|
2341
2627
|
role: result.role,
|
|
@@ -2346,7 +2632,10 @@ ${coreRecall.trim()}`;
|
|
|
2346
2632
|
continue;
|
|
2347
2633
|
}
|
|
2348
2634
|
for (const message of expanded) {
|
|
2349
|
-
const
|
|
2635
|
+
const identity = lcmEvidenceIdentity(message, result.session_id);
|
|
2636
|
+
const { id } = identity;
|
|
2637
|
+
const attributableSearchHit = identity.archiveRowId === void 0 ? expandedIdentityCounts.get(identity.id) === 1 ? uniqueLegacySearchHits.get(identity.id) : void 0 : exactSearchHits.get(identity.id);
|
|
2638
|
+
const attributableScore = typeof attributableSearchHit?.score === "number" ? attributableSearchHit.score : void 0;
|
|
2350
2639
|
if (seenTurns.has(id)) continue;
|
|
2351
2640
|
if (!directTemporalTurnIds.has(id) && shouldIncludeDirectTemporalEvidence(
|
|
2352
2641
|
message.content,
|
|
@@ -2356,11 +2645,12 @@ ${coreRecall.trim()}`;
|
|
|
2356
2645
|
directTemporalTurnIds.add(id);
|
|
2357
2646
|
directTemporalEvidenceItems.push({
|
|
2358
2647
|
id,
|
|
2648
|
+
...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
|
|
2359
2649
|
sessionId: result.session_id,
|
|
2360
2650
|
turnIndex: message.turn_index,
|
|
2361
2651
|
role: message.role,
|
|
2362
2652
|
content: message.content,
|
|
2363
|
-
...
|
|
2653
|
+
...attributableScore === void 0 ? {} : { score: attributableScore }
|
|
2364
2654
|
});
|
|
2365
2655
|
}
|
|
2366
2656
|
if (!shouldIncludeFocusedSearchEvidence(
|
|
@@ -2377,18 +2667,23 @@ ${coreRecall.trim()}`;
|
|
|
2377
2667
|
seenTurns.add(id);
|
|
2378
2668
|
evidenceItems.push({
|
|
2379
2669
|
id,
|
|
2670
|
+
...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
|
|
2380
2671
|
sessionId: result.session_id,
|
|
2381
2672
|
turnIndex: message.turn_index,
|
|
2382
2673
|
role: message.role,
|
|
2383
2674
|
content: message.content,
|
|
2384
|
-
...
|
|
2675
|
+
...attributableScore === void 0 ? {} : { score: attributableScore }
|
|
2385
2676
|
});
|
|
2386
2677
|
}
|
|
2387
2678
|
}
|
|
2679
|
+
const directTemporalSelections = [];
|
|
2388
2680
|
const directTemporalEvidence = buildEvidencePack(directTemporalEvidenceItems, {
|
|
2389
2681
|
title: "Direct temporal evidence",
|
|
2390
2682
|
maxChars: Math.min(searchBudget, 3e3),
|
|
2391
|
-
maxItemChars: 900
|
|
2683
|
+
maxItemChars: 900,
|
|
2684
|
+
...traceRecorder ? { onSelection: (receipt) => {
|
|
2685
|
+
directTemporalSelections.push(receipt);
|
|
2686
|
+
} } : {}
|
|
2392
2687
|
});
|
|
2393
2688
|
let remainingSearchBudget = searchBudget;
|
|
2394
2689
|
if (directTemporalEvidence) {
|
|
@@ -2396,7 +2691,11 @@ ${coreRecall.trim()}`;
|
|
|
2396
2691
|
directTemporalEvidence,
|
|
2397
2692
|
"These direct temporal statements match the question wording. Prefer them over indirect schedule-update context unless the question asks for the latest or current value."
|
|
2398
2693
|
].join("\n\n");
|
|
2399
|
-
|
|
2694
|
+
appendSection("direct-temporal", "evidence-pack", section);
|
|
2695
|
+
traceRecorder?.recordEvidenceSelections(
|
|
2696
|
+
"direct-temporal",
|
|
2697
|
+
directTemporalSelections
|
|
2698
|
+
);
|
|
2400
2699
|
usedChars += section.length;
|
|
2401
2700
|
remainingSearchBudget = 0;
|
|
2402
2701
|
}
|
|
@@ -2405,19 +2704,27 @@ ${coreRecall.trim()}`;
|
|
|
2405
2704
|
evidenceItems
|
|
2406
2705
|
);
|
|
2407
2706
|
if (contradictionGuidance) {
|
|
2408
|
-
|
|
2707
|
+
appendSection("contradiction-guidance", "derived", contradictionGuidance);
|
|
2409
2708
|
usedChars += contradictionGuidance.length;
|
|
2410
2709
|
}
|
|
2710
|
+
const searchSelections = [];
|
|
2411
2711
|
const searchEvidence = buildEvidencePack(
|
|
2412
2712
|
directTemporalEvidence ? evidenceItems.filter((item) => !directTemporalTurnIds.has(item.id)) : evidenceItems,
|
|
2413
2713
|
{
|
|
2414
2714
|
title: "Search evidence",
|
|
2415
2715
|
maxChars: remainingSearchBudget,
|
|
2416
|
-
maxItemChars: 900
|
|
2716
|
+
maxItemChars: 900,
|
|
2717
|
+
...traceRecorder ? { onSelection: (receipt) => {
|
|
2718
|
+
searchSelections.push(receipt);
|
|
2719
|
+
} } : {}
|
|
2417
2720
|
}
|
|
2418
2721
|
);
|
|
2419
2722
|
if (searchEvidence) {
|
|
2420
|
-
|
|
2723
|
+
appendSection("search-evidence", "evidence-pack", searchEvidence);
|
|
2724
|
+
traceRecorder?.recordEvidenceSelections(
|
|
2725
|
+
"search-evidence",
|
|
2726
|
+
searchSelections
|
|
2727
|
+
);
|
|
2421
2728
|
usedChars += searchEvidence.length;
|
|
2422
2729
|
}
|
|
2423
2730
|
}
|
|
@@ -2429,17 +2736,26 @@ ${coreRecall.trim()}`;
|
|
|
2429
2736
|
"## Remnic recall sufficiency",
|
|
2430
2737
|
"No direct evidence found for the requested personal background or previous development projects in this session."
|
|
2431
2738
|
].join("\n");
|
|
2432
|
-
|
|
2739
|
+
appendSection("personal-history-empty", "derived", section);
|
|
2433
2740
|
usedChars += section.length;
|
|
2434
2741
|
}
|
|
2435
2742
|
}
|
|
2436
2743
|
if (!suppressBroadSummary) {
|
|
2437
2744
|
const summaryBudget = Math.max(0, budget - usedChars - 4);
|
|
2438
|
-
const
|
|
2745
|
+
const summaryCapture = traceRecorder && engine.assembleRecallWithTrace ? await waitForRecall(
|
|
2746
|
+
engine.assembleRecallWithTrace(sessionId, summaryBudget)
|
|
2747
|
+
) : void 0;
|
|
2748
|
+
const recallText = summaryCapture?.text ?? await waitForRecall(
|
|
2439
2749
|
engine.assembleRecall(sessionId, summaryBudget)
|
|
2440
2750
|
);
|
|
2441
2751
|
if (recallText) {
|
|
2442
|
-
|
|
2752
|
+
appendSection("lcm-summary", "lcm-summary", recallText);
|
|
2753
|
+
if (summaryCapture) {
|
|
2754
|
+
traceRecorder?.recordSummarySelections(
|
|
2755
|
+
"lcm-summary",
|
|
2756
|
+
summaryCapture.selectedSummaries
|
|
2757
|
+
);
|
|
2758
|
+
}
|
|
2443
2759
|
}
|
|
2444
2760
|
}
|
|
2445
2761
|
if (!historicalRecall && sections.length === 0) {
|
|
@@ -2455,16 +2771,50 @@ ${coreRecall.trim()}`;
|
|
|
2455
2771
|
)
|
|
2456
2772
|
);
|
|
2457
2773
|
if (expanded.length > 0) {
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2774
|
+
const prefix = "## Raw messages\n";
|
|
2775
|
+
const rows = expanded.map(
|
|
2776
|
+
(message) => `[${message.role}]: ${message.content}`
|
|
2461
2777
|
);
|
|
2778
|
+
const rawSection = `${prefix}${rows.join("\n")}`;
|
|
2779
|
+
appendSection("raw-messages", "raw-row", rawSection);
|
|
2780
|
+
let rowStart = prefix.length;
|
|
2781
|
+
expanded.forEach((message, index) => {
|
|
2782
|
+
const rowEnd = rowStart + rows[index].length;
|
|
2783
|
+
traceRecorder?.recordRawRow(
|
|
2784
|
+
"raw-messages",
|
|
2785
|
+
{ start: rowStart, end: rowEnd },
|
|
2786
|
+
message
|
|
2787
|
+
);
|
|
2788
|
+
rowStart = rowEnd + 1;
|
|
2789
|
+
});
|
|
2462
2790
|
}
|
|
2463
2791
|
}
|
|
2464
2792
|
}
|
|
2465
2793
|
const joined = sections.join("\n\n");
|
|
2466
2794
|
return joined.length > budget ? joined.slice(0, budget) : joined;
|
|
2467
2795
|
},
|
|
2796
|
+
recall(sessionId, query, budgetChars, recallOptions = {}, control) {
|
|
2797
|
+
return adapter[composeRecall](
|
|
2798
|
+
sessionId,
|
|
2799
|
+
query,
|
|
2800
|
+
budgetChars,
|
|
2801
|
+
recallOptions,
|
|
2802
|
+
control
|
|
2803
|
+
);
|
|
2804
|
+
},
|
|
2805
|
+
async recallWithTrace(sessionId, query, budgetChars, recallOptions = {}, control) {
|
|
2806
|
+
const budget = budgetChars ?? DEFAULT_BENCH_RECALL_BUDGET_CHARS;
|
|
2807
|
+
const traceRecorder = createBenchRecallTraceRecorder(Math.max(0, budget));
|
|
2808
|
+
const text = await adapter[composeRecall](
|
|
2809
|
+
sessionId,
|
|
2810
|
+
query,
|
|
2811
|
+
budgetChars,
|
|
2812
|
+
recallOptions,
|
|
2813
|
+
control,
|
|
2814
|
+
traceRecorder
|
|
2815
|
+
);
|
|
2816
|
+
return { text, trace: traceRecorder.finalize(text.length) };
|
|
2817
|
+
},
|
|
2468
2818
|
async assessRecallSupport(request, control) {
|
|
2469
2819
|
throwIfBenchPhaseAborted(control, "assessRecallSupport");
|
|
2470
2820
|
return assessRemnicRecallSupport(request, answerSupportMinCoverage);
|
|
@@ -2699,6 +3049,7 @@ ${expanded.map((message) => `[${message.role}]: ${message.content}`).join("\n")}
|
|
|
2699
3049
|
responder: options.responder,
|
|
2700
3050
|
judge: options.judge
|
|
2701
3051
|
};
|
|
3052
|
+
return adapter;
|
|
2702
3053
|
};
|
|
2703
3054
|
}
|
|
2704
3055
|
var createLightweightAdapter = createAdapterFactory("lightweight");
|
|
@@ -3434,7 +3785,7 @@ function extractStructuredTrajectoryCueNumber(content) {
|
|
|
3434
3785
|
function nextBenchTranscriptTurnId(counters, sessionId, message) {
|
|
3435
3786
|
const index = counters.get(sessionId) ?? 0;
|
|
3436
3787
|
counters.set(sessionId, index + 1);
|
|
3437
|
-
const digest =
|
|
3788
|
+
const digest = createHash2("sha256").update(`${sessionId}
|
|
3438
3789
|
${index}
|
|
3439
3790
|
${message.role}
|
|
3440
3791
|
${message.content}`).digest("hex").slice(0, 16);
|
|
@@ -4301,6 +4652,33 @@ function createTimeoutGuardedAdapter(adapter, options) {
|
|
|
4301
4652
|
return adapter.destroy();
|
|
4302
4653
|
}
|
|
4303
4654
|
};
|
|
4655
|
+
if (adapter.recallWithTrace) {
|
|
4656
|
+
wrapped.recallWithTrace = (sessionId, query, budgetChars, recallOptions, control) => {
|
|
4657
|
+
if (phaseTimeoutMs === void 0) {
|
|
4658
|
+
return adapter.recallWithTrace(
|
|
4659
|
+
sessionId,
|
|
4660
|
+
query,
|
|
4661
|
+
budgetChars,
|
|
4662
|
+
recallOptions,
|
|
4663
|
+
control
|
|
4664
|
+
);
|
|
4665
|
+
}
|
|
4666
|
+
return run(`recallWithTrace session=${sessionId}`, async (signal) => {
|
|
4667
|
+
const merged = mergeBenchPhaseControl(signal, control);
|
|
4668
|
+
try {
|
|
4669
|
+
return await adapter.recallWithTrace(
|
|
4670
|
+
sessionId,
|
|
4671
|
+
query,
|
|
4672
|
+
budgetChars,
|
|
4673
|
+
recallOptions,
|
|
4674
|
+
merged.control
|
|
4675
|
+
);
|
|
4676
|
+
} finally {
|
|
4677
|
+
merged.cleanup();
|
|
4678
|
+
}
|
|
4679
|
+
});
|
|
4680
|
+
};
|
|
4681
|
+
}
|
|
4304
4682
|
if (adapter.drain) {
|
|
4305
4683
|
wrapped.drain = (control) => drainTimeoutMs === void 0 ? adapter.drain(control) : run(
|
|
4306
4684
|
"drain",
|
|
@@ -5101,7 +5479,7 @@ function listMemoryEvalBenchmarkIds() {
|
|
|
5101
5479
|
import {
|
|
5102
5480
|
createCipheriv,
|
|
5103
5481
|
createDecipheriv,
|
|
5104
|
-
createHash as
|
|
5482
|
+
createHash as createHash3,
|
|
5105
5483
|
randomBytes,
|
|
5106
5484
|
timingSafeEqual
|
|
5107
5485
|
} from "crypto";
|
|
@@ -5112,10 +5490,10 @@ var AES_TAG_LENGTH = 16;
|
|
|
5112
5490
|
var INTEGRITY_HASH_ALGORITHM = "sha256";
|
|
5113
5491
|
var INTEGRITY_CIPHER_ALGORITHM = "aes-256-gcm";
|
|
5114
5492
|
function hashString(value) {
|
|
5115
|
-
return
|
|
5493
|
+
return createHash3(INTEGRITY_HASH_ALGORITHM).update(value, "utf8").digest("hex");
|
|
5116
5494
|
}
|
|
5117
5495
|
function hashBytes(value) {
|
|
5118
|
-
return
|
|
5496
|
+
return createHash3(INTEGRITY_HASH_ALGORITHM).update(value).digest("hex");
|
|
5119
5497
|
}
|
|
5120
5498
|
function canonicalJsonStringify(value, space) {
|
|
5121
5499
|
return JSON.stringify(value, canonicalReplacer, space);
|
|
@@ -5497,14 +5875,14 @@ var BENCHMARK_RESULT_SCHEMA = {
|
|
|
5497
5875
|
|
|
5498
5876
|
// src/repro-manifest.ts
|
|
5499
5877
|
import { execFileSync } from "child_process";
|
|
5500
|
-
import { createHash as
|
|
5878
|
+
import { createHash as createHash5 } from "crypto";
|
|
5501
5879
|
import { createReadStream } from "fs";
|
|
5502
5880
|
import { lstat as lstat3, mkdir as mkdir4, readFile as readFile5, readdir as readdir4, readlink, realpath as realpath3, stat as stat2, writeFile as writeFile4 } from "fs/promises";
|
|
5503
5881
|
import os3 from "os";
|
|
5504
5882
|
import path5 from "path";
|
|
5505
5883
|
|
|
5506
5884
|
// src/providers/codex-credit-budget.ts
|
|
5507
|
-
import { createHash as
|
|
5885
|
+
import { createHash as createHash4 } from "crypto";
|
|
5508
5886
|
import { mkdir as mkdir2, open, readFile as readFile3, rename as rename2, rmdir, unlink as unlink2, writeFile as writeFile2 } from "fs/promises";
|
|
5509
5887
|
import os from "os";
|
|
5510
5888
|
import path3 from "path";
|
|
@@ -6123,7 +6501,7 @@ function isSha256(value) {
|
|
|
6123
6501
|
return typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
|
6124
6502
|
}
|
|
6125
6503
|
function sha256(value) {
|
|
6126
|
-
return
|
|
6504
|
+
return createHash4("sha256").update(value).digest("hex");
|
|
6127
6505
|
}
|
|
6128
6506
|
function normalizeZero(value) {
|
|
6129
6507
|
return Object.is(value, -0) ? 0 : value;
|
|
@@ -7270,13 +7648,13 @@ var CODEX_CREDIT_REPRO_ENV_KEYS = [
|
|
|
7270
7648
|
"REMNIC_BENCH_RUN_ID"
|
|
7271
7649
|
];
|
|
7272
7650
|
function sha256String(value) {
|
|
7273
|
-
return
|
|
7651
|
+
return createHash5("sha256").update(value).digest("hex");
|
|
7274
7652
|
}
|
|
7275
7653
|
function sha256Buffer(value) {
|
|
7276
|
-
return
|
|
7654
|
+
return createHash5("sha256").update(value).digest("hex");
|
|
7277
7655
|
}
|
|
7278
7656
|
async function sha256File(filePath) {
|
|
7279
|
-
const hash =
|
|
7657
|
+
const hash = createHash5("sha256");
|
|
7280
7658
|
await new Promise((resolve, reject) => {
|
|
7281
7659
|
const stream = createReadStream(filePath);
|
|
7282
7660
|
stream.on("data", (chunk) => hash.update(chunk));
|
|
@@ -8197,7 +8575,7 @@ async function writeBenchmarkReproManifest(resultsDir, options = {}) {
|
|
|
8197
8575
|
}
|
|
8198
8576
|
|
|
8199
8577
|
// src/published-artifact.ts
|
|
8200
|
-
import { createHash as
|
|
8578
|
+
import { createHash as createHash6 } from "crypto";
|
|
8201
8579
|
import { mkdir as mkdir5, readFile as readFile6, writeFile as writeFile5 } from "fs/promises";
|
|
8202
8580
|
import path6 from "path";
|
|
8203
8581
|
var BENCHMARK_ARTIFACT_SCHEMA_VERSION = 1;
|
|
@@ -8343,7 +8721,7 @@ function serializeBenchmarkArtifact(artifact) {
|
|
|
8343
8721
|
`;
|
|
8344
8722
|
}
|
|
8345
8723
|
function hashBenchmarkArtifact(artifact) {
|
|
8346
|
-
return
|
|
8724
|
+
return createHash6("sha256").update(serializeBenchmarkArtifact(artifact)).digest("hex");
|
|
8347
8725
|
}
|
|
8348
8726
|
async function writeBenchmarkArtifact(artifact, outputDir) {
|
|
8349
8727
|
await mkdir5(outputDir, { recursive: true });
|
|
@@ -8361,7 +8739,7 @@ async function writeBenchmarkArtifact(artifact, outputDir) {
|
|
|
8361
8739
|
return {
|
|
8362
8740
|
path: abs,
|
|
8363
8741
|
filename,
|
|
8364
|
-
sha256:
|
|
8742
|
+
sha256: createHash6("sha256").update(body).digest("hex"),
|
|
8365
8743
|
bytes: Buffer.byteLength(body, "utf8")
|
|
8366
8744
|
};
|
|
8367
8745
|
}
|
|
@@ -8475,7 +8853,7 @@ async function loadBenchmarkArtifact(filePath) {
|
|
|
8475
8853
|
const artifact = parseBenchmarkArtifact(raw);
|
|
8476
8854
|
return {
|
|
8477
8855
|
artifact,
|
|
8478
|
-
sha256:
|
|
8856
|
+
sha256: createHash6("sha256").update(raw).digest("hex"),
|
|
8479
8857
|
bytes: Buffer.byteLength(raw, "utf8")
|
|
8480
8858
|
};
|
|
8481
8859
|
}
|
|
@@ -9638,7 +10016,7 @@ function createClaudeCliProvider(config, deps) {
|
|
|
9638
10016
|
|
|
9639
10017
|
// src/providers/codex-cli.ts
|
|
9640
10018
|
import { spawn as spawn2 } from "child_process";
|
|
9641
|
-
import { createHash as
|
|
10019
|
+
import { createHash as createHash7, randomUUID } from "crypto";
|
|
9642
10020
|
import { mkdir as mkdir6, mkdtemp as mkdtemp3, readFile as readFile7, rm as rm3, writeFile as writeFile6 } from "fs/promises";
|
|
9643
10021
|
import os5 from "os";
|
|
9644
10022
|
import path8 from "path";
|
|
@@ -10506,7 +10884,7 @@ function resolveCodexCliDiagnosticsMode(config) {
|
|
|
10506
10884
|
}
|
|
10507
10885
|
function inspectCodexCompletionPrompt(prompt) {
|
|
10508
10886
|
const stats = {
|
|
10509
|
-
sha256:
|
|
10887
|
+
sha256: createHash7("sha256").update(prompt).digest("hex"),
|
|
10510
10888
|
chars: prompt.length,
|
|
10511
10889
|
lines: prompt.length === 0 ? 0 : prompt.split("\n").length
|
|
10512
10890
|
};
|
|
@@ -12642,7 +13020,7 @@ function asStringArray(value) {
|
|
|
12642
13020
|
}
|
|
12643
13021
|
|
|
12644
13022
|
// src/responders.ts
|
|
12645
|
-
import { createHash as
|
|
13023
|
+
import { createHash as createHash8 } from "crypto";
|
|
12646
13024
|
import { FallbackLlmClient } from "@remnic/core";
|
|
12647
13025
|
|
|
12648
13026
|
// src/providers/openai-responses.ts
|
|
@@ -13115,7 +13493,7 @@ function getProviderBackedJudgePromptIdentity(config) {
|
|
|
13115
13493
|
temperature: 0,
|
|
13116
13494
|
maxTokens: 16
|
|
13117
13495
|
};
|
|
13118
|
-
return `sha256:${
|
|
13496
|
+
return `sha256:${createHash8("sha256").update(JSON.stringify(contract)).digest("hex")}`;
|
|
13119
13497
|
}
|
|
13120
13498
|
var AMA_BENCH_RECOMMENDED_JUDGE_SYSTEM_PROMPT = [
|
|
13121
13499
|
"You are evaluating an AMA-Bench long-horizon memory question.",
|
|
@@ -15456,11 +15834,11 @@ async function resolveLocalLabRuntimeProfile(options) {
|
|
|
15456
15834
|
// src/benchmark.ts
|
|
15457
15835
|
import fs2 from "fs";
|
|
15458
15836
|
import path35 from "path";
|
|
15459
|
-
import { createHash as
|
|
15837
|
+
import { createHash as createHash14 } from "crypto";
|
|
15460
15838
|
import { expandTildePath as expandTildePath3 } from "@remnic/core";
|
|
15461
15839
|
|
|
15462
15840
|
// src/judges/judge-cache.ts
|
|
15463
|
-
import { createHash as
|
|
15841
|
+
import { createHash as createHash9, randomBytes as randomBytes2 } from "crypto";
|
|
15464
15842
|
import {
|
|
15465
15843
|
mkdir as mkdir9,
|
|
15466
15844
|
readFile as readFile11,
|
|
@@ -15503,8 +15881,8 @@ var JudgeCache = class {
|
|
|
15503
15881
|
}
|
|
15504
15882
|
/** Compute the sha256-hex key for a set of parts. Pure, sync, side-effect-free. */
|
|
15505
15883
|
computeKey(parts) {
|
|
15506
|
-
const fieldDigest = (value) =>
|
|
15507
|
-
return
|
|
15884
|
+
const fieldDigest = (value) => createHash9("sha256").update(value).digest();
|
|
15885
|
+
return createHash9("sha256").update(fieldDigest(parts.benchmarkId)).update(fieldDigest(parts.datasetVersion)).update(fieldDigest(parts.questionId)).update(fieldDigest(parts.answerText)).update(fieldDigest(parts.judgePromptHash)).update(fieldDigest(parts.judgeModelId)).update(fieldDigest(parts.judgeParamsHash)).digest("hex");
|
|
15508
15886
|
}
|
|
15509
15887
|
/**
|
|
15510
15888
|
* Read a previously-stored verdict. Returns `undefined` on miss, corrupted
|
|
@@ -15699,7 +16077,7 @@ function runJudgeWithCache(options) {
|
|
|
15699
16077
|
// Binary prompts are content-sensitive: two distinct prompts of
|
|
15700
16078
|
// the same character length would collide on the previous
|
|
15701
16079
|
// `binary:N` key, so key on a sha256 prefix of the prompt body.
|
|
15702
|
-
questionId: `binary:${
|
|
16080
|
+
questionId: `binary:${createHash9("sha256").update(prompt).digest("hex").slice(0, 16)}`,
|
|
15703
16081
|
answerText: prompt,
|
|
15704
16082
|
judgePromptHash: keyExtras.judgePromptHash ?? "unknown-prompt",
|
|
15705
16083
|
judgeModelId: keyExtras.judgeModelId ?? "unknown-judge",
|
|
@@ -19469,26 +19847,6 @@ import { collectTemporalLexicalCues } from "@remnic/core";
|
|
|
19469
19847
|
import { readFile as readFile15 } from "fs/promises";
|
|
19470
19848
|
import path17 from "path";
|
|
19471
19849
|
|
|
19472
|
-
// src/benchmarks/published/longmemeval/fixture.ts
|
|
19473
|
-
var LONG_MEM_EVAL_SMOKE_FIXTURE = [
|
|
19474
|
-
{
|
|
19475
|
-
question_id: 1,
|
|
19476
|
-
question_type: "single-session-user",
|
|
19477
|
-
question: "What city does the user live in?",
|
|
19478
|
-
answer: "Paris",
|
|
19479
|
-
question_date: "2025-01-01",
|
|
19480
|
-
haystack_dates: ["2024-12-01"],
|
|
19481
|
-
haystack_session_ids: ["session-1"],
|
|
19482
|
-
haystack_sessions: [
|
|
19483
|
-
[
|
|
19484
|
-
{ role: "user", content: "I moved to Paris last year." },
|
|
19485
|
-
{ role: "assistant", content: "Paris sounds great." }
|
|
19486
|
-
]
|
|
19487
|
-
],
|
|
19488
|
-
answer_session_ids: ["session-1"]
|
|
19489
|
-
}
|
|
19490
|
-
];
|
|
19491
|
-
|
|
19492
19850
|
// src/benchmarks/published/locomo/fixture.ts
|
|
19493
19851
|
var LOCOMO_SMOKE_FIXTURE = [
|
|
19494
19852
|
{
|
|
@@ -19538,6 +19896,26 @@ var LOCOMO_SMOKE_FIXTURE = [
|
|
|
19538
19896
|
}
|
|
19539
19897
|
];
|
|
19540
19898
|
|
|
19899
|
+
// src/benchmarks/published/longmemeval/fixture.ts
|
|
19900
|
+
var LONG_MEM_EVAL_SMOKE_FIXTURE = [
|
|
19901
|
+
{
|
|
19902
|
+
question_id: 1,
|
|
19903
|
+
question_type: "single-session-user",
|
|
19904
|
+
question: "What city does the user live in?",
|
|
19905
|
+
answer: "Paris",
|
|
19906
|
+
question_date: "2025-01-01",
|
|
19907
|
+
haystack_dates: ["2024-12-01"],
|
|
19908
|
+
haystack_session_ids: ["session-1"],
|
|
19909
|
+
haystack_sessions: [
|
|
19910
|
+
[
|
|
19911
|
+
{ role: "user", content: "I moved to Paris last year." },
|
|
19912
|
+
{ role: "assistant", content: "Paris sounds great." }
|
|
19913
|
+
]
|
|
19914
|
+
],
|
|
19915
|
+
answer_session_ids: ["session-1"]
|
|
19916
|
+
}
|
|
19917
|
+
];
|
|
19918
|
+
|
|
19541
19919
|
// src/benchmarks/published/dataset-loader.ts
|
|
19542
19920
|
var LONG_MEM_EVAL_DATASET_FILENAMES = Object.freeze([
|
|
19543
19921
|
"longmemeval_oracle.json",
|
|
@@ -19585,6 +19963,7 @@ async function loadDataset4(options) {
|
|
|
19585
19963
|
return {
|
|
19586
19964
|
source: "dataset",
|
|
19587
19965
|
filename,
|
|
19966
|
+
sha256: hashString(raw),
|
|
19588
19967
|
items: applyLimit4(parsed, limit),
|
|
19589
19968
|
errors
|
|
19590
19969
|
};
|
|
@@ -19600,6 +19979,7 @@ async function loadDataset4(options) {
|
|
|
19600
19979
|
}
|
|
19601
19980
|
return {
|
|
19602
19981
|
source: "smoke",
|
|
19982
|
+
sha256: hashCanonicalJson(options.smokeFixture),
|
|
19603
19983
|
items: applyLimit4([...options.smokeFixture], limit),
|
|
19604
19984
|
errors
|
|
19605
19985
|
};
|
|
@@ -20885,11 +21265,12 @@ var locomoDefinition = {
|
|
|
20885
21265
|
}
|
|
20886
21266
|
};
|
|
20887
21267
|
async function runLoCoMoBenchmark(options) {
|
|
20888
|
-
const
|
|
21268
|
+
const loaded = await loadLoCoMoDataset(
|
|
20889
21269
|
options.mode,
|
|
20890
21270
|
options.datasetDir,
|
|
20891
21271
|
options.limit
|
|
20892
21272
|
);
|
|
21273
|
+
const conversations = loaded.items;
|
|
20893
21274
|
const trialLimit = resolveTrialLimit(options.benchmarkOptions?.trialLimit);
|
|
20894
21275
|
const multiHopRecallComposition = resolveLoCoMoBooleanOption(
|
|
20895
21276
|
options.benchmarkOptions?.multiHopRecallComposition,
|
|
@@ -20898,7 +21279,7 @@ async function runLoCoMoBenchmark(options) {
|
|
|
20898
21279
|
);
|
|
20899
21280
|
const plans = applyTrialLimit(
|
|
20900
21281
|
conversations.map(
|
|
20901
|
-
(conversation) =>
|
|
21282
|
+
(conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition)
|
|
20902
21283
|
),
|
|
20903
21284
|
trialLimit
|
|
20904
21285
|
);
|
|
@@ -20969,7 +21350,7 @@ function applyTrialLimit(plans, trialLimit) {
|
|
|
20969
21350
|
}
|
|
20970
21351
|
return limitedPlans;
|
|
20971
21352
|
}
|
|
20972
|
-
function
|
|
21353
|
+
function buildLoCoMoPlan(conversation, multiHopRecallComposition) {
|
|
20973
21354
|
const sessions = extractSessions(conversation.conversation);
|
|
20974
21355
|
const speakerA = typeof conversation.conversation.speaker_a === "string" ? conversation.conversation.speaker_a : "Speaker A";
|
|
20975
21356
|
const ingestSessions = [];
|
|
@@ -21005,9 +21386,9 @@ function buildTrial(conversationId, qa, questionIndex, sessionIds, multiHopRecal
|
|
|
21005
21386
|
expected: qa.answer,
|
|
21006
21387
|
recallSessionIds: sessionIds,
|
|
21007
21388
|
answerFormat: "short-with-specifics",
|
|
21008
|
-
recallTextTransform: ({ question, recalledText }) =>
|
|
21389
|
+
recallTextTransform: ({ question, recalledText }) => transformLoCoMoRecallText({
|
|
21009
21390
|
question,
|
|
21010
|
-
recalledText
|
|
21391
|
+
recalledText,
|
|
21011
21392
|
multiHopRecallComposition
|
|
21012
21393
|
}),
|
|
21013
21394
|
answerFallback: ({ question, recalledText }) => answerLoCoMoFromRecall(question, recalledText),
|
|
@@ -21208,10 +21589,20 @@ function sanitizeLoCoMoRecallText(args) {
|
|
|
21208
21589
|
(id) => queryVisibleIds.has(id) ? id : ""
|
|
21209
21590
|
);
|
|
21210
21591
|
}
|
|
21211
|
-
function
|
|
21212
|
-
const
|
|
21213
|
-
|
|
21214
|
-
|
|
21592
|
+
function transformLoCoMoRecallText(args) {
|
|
21593
|
+
const sanitized = sanitizeLoCoMoRecallText(args);
|
|
21594
|
+
return prioritizeLoCoMoRecallTextWithTrace({
|
|
21595
|
+
...args,
|
|
21596
|
+
recalledText: sanitized
|
|
21597
|
+
}).text;
|
|
21598
|
+
}
|
|
21599
|
+
function prioritizeLoCoMoRecallTextWithTrace(args) {
|
|
21600
|
+
const inputLines = args.recalledText.replaceAll("\r\n", "\n").replaceAll("\r", "\n").split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
21601
|
+
const lines = dedupePreserveOrder(inputLines);
|
|
21602
|
+
const inputOrdinalByLine = /* @__PURE__ */ new Map();
|
|
21603
|
+
inputLines.forEach((line, index) => {
|
|
21604
|
+
if (!inputOrdinalByLine.has(line)) inputOrdinalByLine.set(line, index);
|
|
21605
|
+
});
|
|
21215
21606
|
const questionTokens = expandLoCoMoQuestionTokens(
|
|
21216
21607
|
tokenizeForLoCoMo(args.question)
|
|
21217
21608
|
);
|
|
@@ -21244,25 +21635,90 @@ function prioritizeLoCoMoRecallText(args) {
|
|
|
21244
21635
|
)
|
|
21245
21636
|
];
|
|
21246
21637
|
if (direct.length === 0 && linkedHops.length === 0) {
|
|
21247
|
-
|
|
21638
|
+
const { text: text2 } = truncateLoCoMoContext(
|
|
21248
21639
|
args.recalledText,
|
|
21249
21640
|
LOCOMO_FALLBACK_CONTEXT_MAX_CHARS
|
|
21250
21641
|
);
|
|
21642
|
+
return {
|
|
21643
|
+
text: text2,
|
|
21644
|
+
receipt: {
|
|
21645
|
+
schemaVersion: 1,
|
|
21646
|
+
mode: "fallback",
|
|
21647
|
+
multiHopRecallComposition: args.multiHopRecallComposition,
|
|
21648
|
+
input: digestLoCoMoContent(args.recalledText),
|
|
21649
|
+
output: digestLoCoMoContent(text2),
|
|
21650
|
+
selectedLines: []
|
|
21651
|
+
}
|
|
21652
|
+
};
|
|
21251
21653
|
}
|
|
21252
|
-
const sections = [
|
|
21253
|
-
|
|
21254
|
-
|
|
21255
|
-
|
|
21654
|
+
const sections = ["## LoCoMo Question-Focused Evidence"];
|
|
21655
|
+
const selectedRanges = [];
|
|
21656
|
+
const appendSelectedLine = (input, stage, hop) => {
|
|
21657
|
+
const output = truncateLoCoMoLine(input);
|
|
21658
|
+
const inputOrdinal = inputOrdinalByLine.get(input);
|
|
21659
|
+
if (inputOrdinal === void 0) {
|
|
21660
|
+
throw new Error("LoCoMo composition selected a line outside its normalized input.");
|
|
21661
|
+
}
|
|
21662
|
+
const outputStart = sections.join("\n").length + 1;
|
|
21663
|
+
sections.push(output);
|
|
21664
|
+
selectedRanges.push({
|
|
21665
|
+
input,
|
|
21666
|
+
output,
|
|
21667
|
+
inputOrdinal,
|
|
21668
|
+
stage,
|
|
21669
|
+
...hop === void 0 ? {} : { hop },
|
|
21670
|
+
outputStart,
|
|
21671
|
+
outputEnd: outputStart + output.length
|
|
21672
|
+
});
|
|
21673
|
+
};
|
|
21674
|
+
for (const entry of direct) appendSelectedLine(entry.line, "direct");
|
|
21256
21675
|
for (const hop of linkedHops) {
|
|
21257
|
-
sections.push(
|
|
21258
|
-
|
|
21259
|
-
...hop.lines.map(truncateLoCoMoLine)
|
|
21260
|
-
);
|
|
21676
|
+
sections.push(`## LoCoMo Linked Evidence (hop ${hop.hop})`);
|
|
21677
|
+
for (const line of hop.lines) appendSelectedLine(line, "linked", hop.hop);
|
|
21261
21678
|
}
|
|
21262
|
-
|
|
21679
|
+
const truncation = truncateLoCoMoContext(
|
|
21263
21680
|
sections.join("\n"),
|
|
21264
21681
|
LOCOMO_FOCUSED_CONTEXT_MAX_CHARS
|
|
21265
21682
|
);
|
|
21683
|
+
const { text, safePrefixEnd } = truncation;
|
|
21684
|
+
const selectedLines = selectedRanges.map((entry) => {
|
|
21685
|
+
const visibleStart = Math.min(entry.outputStart, safePrefixEnd);
|
|
21686
|
+
const visibleEnd = Math.min(entry.outputEnd, safePrefixEnd);
|
|
21687
|
+
const visible = visibleEnd - visibleStart === entry.output.length;
|
|
21688
|
+
return buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd);
|
|
21689
|
+
});
|
|
21690
|
+
return {
|
|
21691
|
+
text,
|
|
21692
|
+
receipt: {
|
|
21693
|
+
schemaVersion: 1,
|
|
21694
|
+
mode: "focused",
|
|
21695
|
+
multiHopRecallComposition: args.multiHopRecallComposition,
|
|
21696
|
+
input: digestLoCoMoContent(args.recalledText),
|
|
21697
|
+
output: digestLoCoMoContent(text),
|
|
21698
|
+
selectedLines
|
|
21699
|
+
}
|
|
21700
|
+
};
|
|
21701
|
+
}
|
|
21702
|
+
function buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd) {
|
|
21703
|
+
return {
|
|
21704
|
+
inputOrdinal: entry.inputOrdinal,
|
|
21705
|
+
input: digestLoCoMoContent(entry.input),
|
|
21706
|
+
output: digestLoCoMoContent(entry.output),
|
|
21707
|
+
stage: entry.stage,
|
|
21708
|
+
...entry.hop === void 0 ? {} : { hop: entry.hop },
|
|
21709
|
+
visible,
|
|
21710
|
+
outputStart: entry.outputStart,
|
|
21711
|
+
outputEnd: entry.outputEnd,
|
|
21712
|
+
visibleStart,
|
|
21713
|
+
visibleEnd
|
|
21714
|
+
};
|
|
21715
|
+
}
|
|
21716
|
+
function digestLoCoMoContent(value) {
|
|
21717
|
+
return {
|
|
21718
|
+
sha256: hashString(value),
|
|
21719
|
+
charCount: value.length,
|
|
21720
|
+
lineCount: value.length === 0 ? 0 : value.split("\n").length
|
|
21721
|
+
};
|
|
21266
21722
|
}
|
|
21267
21723
|
function composeLoCoMoLinkedEvidence(args) {
|
|
21268
21724
|
if (args.direct.length === 0 || args.remainingLineBudget <= 0) {
|
|
@@ -21520,13 +21976,16 @@ function truncateLoCoMoLine(line) {
|
|
|
21520
21976
|
}
|
|
21521
21977
|
function truncateLoCoMoContext(text, maxChars) {
|
|
21522
21978
|
if (text.length <= maxChars) {
|
|
21523
|
-
return text;
|
|
21979
|
+
return { text, safePrefixEnd: text.length };
|
|
21524
21980
|
}
|
|
21525
21981
|
const truncated = text.slice(0, maxChars);
|
|
21526
21982
|
const lastNewline = truncated.lastIndexOf("\n");
|
|
21527
21983
|
const safePrefix = lastNewline > 0 ? truncated.slice(0, lastNewline) : truncated;
|
|
21528
|
-
return
|
|
21529
|
-
|
|
21984
|
+
return {
|
|
21985
|
+
text: `${safePrefix}
|
|
21986
|
+
[LoCoMo context truncated to ${maxChars} characters]`,
|
|
21987
|
+
safePrefixEnd: safePrefix.length
|
|
21988
|
+
};
|
|
21530
21989
|
}
|
|
21531
21990
|
function countHiddenEvidenceIdsInRecall(evidence, question, recalledText) {
|
|
21532
21991
|
const queryVisibleIds = collectDialogueIds(question);
|
|
@@ -21547,7 +22006,7 @@ function collectDialogueIds(text) {
|
|
|
21547
22006
|
function escapeRegExp2(value) {
|
|
21548
22007
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
21549
22008
|
}
|
|
21550
|
-
async function
|
|
22009
|
+
async function loadLoCoMoDataset(mode, datasetDir, limit) {
|
|
21551
22010
|
const loaded = await loadLoCoMo10({
|
|
21552
22011
|
mode,
|
|
21553
22012
|
datasetDir,
|
|
@@ -21579,7 +22038,15 @@ async function loadDataset6(mode, datasetDir, limit) {
|
|
|
21579
22038
|
"[remnic-bench] LoCoMo falling back to smoke fixture: " + loaded.errors.join(" | ")
|
|
21580
22039
|
);
|
|
21581
22040
|
}
|
|
21582
|
-
|
|
22041
|
+
if (!loaded.sha256) {
|
|
22042
|
+
throw new Error("LoCoMo dataset loader did not provide a content hash.");
|
|
22043
|
+
}
|
|
22044
|
+
return {
|
|
22045
|
+
source: loaded.source,
|
|
22046
|
+
...loaded.filename === void 0 ? {} : { filename: loaded.filename },
|
|
22047
|
+
sha256: loaded.sha256,
|
|
22048
|
+
items: loaded.items
|
|
22049
|
+
};
|
|
21583
22050
|
}
|
|
21584
22051
|
function parseDataset2(raw, filename) {
|
|
21585
22052
|
const parsed = JSON.parse(raw);
|
|
@@ -21863,7 +22330,7 @@ async function loadBeamDatasetPreview(options) {
|
|
|
21863
22330
|
}
|
|
21864
22331
|
let dataset;
|
|
21865
22332
|
try {
|
|
21866
|
-
dataset = await
|
|
22333
|
+
dataset = await loadDataset6(
|
|
21867
22334
|
options.mode === "quick" ? "quick" : "full",
|
|
21868
22335
|
options.datasetDir,
|
|
21869
22336
|
options.limit
|
|
@@ -21902,7 +22369,7 @@ async function loadBeamDatasetPreview(options) {
|
|
|
21902
22369
|
};
|
|
21903
22370
|
}
|
|
21904
22371
|
async function runBeamBenchmark(options) {
|
|
21905
|
-
const dataset = await
|
|
22372
|
+
const dataset = await loadDataset6(options.mode, options.datasetDir, options.limit);
|
|
21906
22373
|
const tasks = [];
|
|
21907
22374
|
const taskFilter = normalizeBeamTaskFilter(
|
|
21908
22375
|
options.benchmarkOptions?.taskFilter
|
|
@@ -22089,7 +22556,7 @@ async function runBeamBenchmark(options) {
|
|
|
22089
22556
|
}
|
|
22090
22557
|
};
|
|
22091
22558
|
}
|
|
22092
|
-
async function
|
|
22559
|
+
async function loadDataset6(mode, datasetDir, limit) {
|
|
22093
22560
|
const normalizedLimit = normalizeLimit5(limit);
|
|
22094
22561
|
const ensureDatasetEntries = (entryCount) => {
|
|
22095
22562
|
if (entryCount === 0) {
|
|
@@ -23190,7 +23657,7 @@ var StructuredLiteralParser = class {
|
|
|
23190
23657
|
};
|
|
23191
23658
|
|
|
23192
23659
|
// src/benchmarks/published/personamem/runner.ts
|
|
23193
|
-
import { createHash as
|
|
23660
|
+
import { createHash as createHash10, randomUUID as randomUUID7 } from "crypto";
|
|
23194
23661
|
import { readFile as readFile16, realpath as realpath4 } from "fs/promises";
|
|
23195
23662
|
import path19 from "path";
|
|
23196
23663
|
|
|
@@ -23275,7 +23742,7 @@ var personaMemDefinition = {
|
|
|
23275
23742
|
}
|
|
23276
23743
|
};
|
|
23277
23744
|
async function runPersonaMemBenchmark(options) {
|
|
23278
|
-
const samples = await
|
|
23745
|
+
const samples = await loadDataset7(options.mode, options.datasetDir, options.limit);
|
|
23279
23746
|
const tasks = [];
|
|
23280
23747
|
const totalTasks = samples.length;
|
|
23281
23748
|
for (let sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
|
|
@@ -23455,7 +23922,7 @@ async function runPersonaMemBenchmark(options) {
|
|
|
23455
23922
|
}
|
|
23456
23923
|
};
|
|
23457
23924
|
}
|
|
23458
|
-
async function
|
|
23925
|
+
async function loadDataset7(mode, datasetDir, limit) {
|
|
23459
23926
|
const normalizedLimit = normalizeLimit6(limit);
|
|
23460
23927
|
const ensureDatasetSamples = (samples) => {
|
|
23461
23928
|
if (samples.length === 0) {
|
|
@@ -23795,7 +24262,7 @@ function buildMcqPrompt(sample, seed) {
|
|
|
23795
24262
|
function deterministicShuffle(values, seedMaterial) {
|
|
23796
24263
|
return values.map((value, index) => ({
|
|
23797
24264
|
value,
|
|
23798
|
-
key:
|
|
24265
|
+
key: createHash10("sha256").update(`${seedMaterial}:${index}:${value}`).digest("hex"),
|
|
23799
24266
|
index
|
|
23800
24267
|
})).sort((left, right) => {
|
|
23801
24268
|
const byKey = left.key.localeCompare(right.key);
|
|
@@ -24063,7 +24530,7 @@ var memBenchDefinition = {
|
|
|
24063
24530
|
}
|
|
24064
24531
|
};
|
|
24065
24532
|
async function runMemBenchBenchmark(options) {
|
|
24066
|
-
const dataset = await
|
|
24533
|
+
const dataset = await loadDataset8(options.mode, options.datasetDir, options.limit);
|
|
24067
24534
|
const tasks = [];
|
|
24068
24535
|
const totalTasks = dataset.length;
|
|
24069
24536
|
for (const testCase of dataset) {
|
|
@@ -24237,7 +24704,7 @@ async function runMemBenchBenchmark(options) {
|
|
|
24237
24704
|
}
|
|
24238
24705
|
};
|
|
24239
24706
|
}
|
|
24240
|
-
async function
|
|
24707
|
+
async function loadDataset8(mode, datasetDir, limit) {
|
|
24241
24708
|
const normalizedLimit = normalizeLimit7(limit);
|
|
24242
24709
|
const ensureDatasetCases = (cases) => {
|
|
24243
24710
|
if (cases.length === 0) {
|
|
@@ -25268,7 +25735,7 @@ var memoryAgentBenchDefinition = {
|
|
|
25268
25735
|
}
|
|
25269
25736
|
};
|
|
25270
25737
|
async function runMemoryAgentBenchBenchmark(options) {
|
|
25271
|
-
const rawDataset = await
|
|
25738
|
+
const rawDataset = await loadDataset9(options.mode, options.datasetDir, options.limit);
|
|
25272
25739
|
const trialLimit = resolveTrialLimit2(options.benchmarkOptions?.trialLimit);
|
|
25273
25740
|
const benchmarkOptions = trialLimit === void 0 ? options.benchmarkOptions : { ...options.benchmarkOptions ?? {}, trialLimit };
|
|
25274
25741
|
const dataset = applyTrialLimit2(rawDataset, trialLimit);
|
|
@@ -26241,7 +26708,7 @@ function decodeUrlComponentSafely(value) {
|
|
|
26241
26708
|
return value;
|
|
26242
26709
|
}
|
|
26243
26710
|
}
|
|
26244
|
-
async function
|
|
26711
|
+
async function loadDataset9(mode, datasetDir, limit) {
|
|
26245
26712
|
const normalizedLimit = normalizeLimit8(limit);
|
|
26246
26713
|
const ensureDatasetItems = (items) => {
|
|
26247
26714
|
if (items.length === 0) {
|
|
@@ -32254,7 +32721,7 @@ function pairedDeltaConfidenceInterval(candidateValues, baselineValues, options
|
|
|
32254
32721
|
}
|
|
32255
32722
|
|
|
32256
32723
|
// src/judges/sealed-rubric.ts
|
|
32257
|
-
import { createHash as
|
|
32724
|
+
import { createHash as createHash11 } from "crypto";
|
|
32258
32725
|
import { appendFileSync, mkdirSync } from "fs";
|
|
32259
32726
|
import path31 from "path";
|
|
32260
32727
|
|
|
@@ -32363,7 +32830,7 @@ function loadSealedRubric(id = DEFAULT_ASSISTANT_RUBRIC_ID, options = {}) {
|
|
|
32363
32830
|
if (typeof prompt !== "string" || prompt.length === 0) {
|
|
32364
32831
|
throw new Error(`sealed rubric not found in registry: ${id}`);
|
|
32365
32832
|
}
|
|
32366
|
-
const sha2563 =
|
|
32833
|
+
const sha2563 = createHash11("sha256").update(prompt, "utf8").digest("hex");
|
|
32367
32834
|
const version = parseVersionFromId(id);
|
|
32368
32835
|
return { id, version, prompt, sha256: sha2563 };
|
|
32369
32836
|
}
|
|
@@ -34566,7 +35033,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
|
|
|
34566
35033
|
import { randomUUID as randomUUID31 } from "crypto";
|
|
34567
35034
|
|
|
34568
35035
|
// src/benchmarks/remnic/memcorrect/generator.ts
|
|
34569
|
-
import { createHash as
|
|
35036
|
+
import { createHash as createHash12 } from "crypto";
|
|
34570
35037
|
|
|
34571
35038
|
// src/benchmarks/remnic/memcorrect/token-pools.ts
|
|
34572
35039
|
var PERSONAS = [
|
|
@@ -34890,7 +35357,7 @@ function corpusHash(corpus) {
|
|
|
34890
35357
|
uptakeLatencyCap: corpus.options.uptakeLatencyCap,
|
|
34891
35358
|
scenarios: corpus.scenarios
|
|
34892
35359
|
});
|
|
34893
|
-
return
|
|
35360
|
+
return createHash12("sha256").update(canonical).digest("hex");
|
|
34894
35361
|
}
|
|
34895
35362
|
|
|
34896
35363
|
// src/benchmarks/remnic/memcorrect/schema.ts
|
|
@@ -35845,7 +36312,7 @@ import { mkdir as mkdir17, writeFile as writeFile16 } from "fs/promises";
|
|
|
35845
36312
|
import path34 from "path";
|
|
35846
36313
|
|
|
35847
36314
|
// src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
|
|
35848
|
-
import { createHash as
|
|
36315
|
+
import { createHash as createHash13 } from "crypto";
|
|
35849
36316
|
var SCOPE_ACME = "project:acme";
|
|
35850
36317
|
var SCOPE_BETA = "project:beta";
|
|
35851
36318
|
var SCOPE_ALICE = "user:alice";
|
|
@@ -36328,7 +36795,7 @@ var BOUNDED_MEMORY_SMOKE_FIXTURE = [
|
|
|
36328
36795
|
function fixtureHash(tasks) {
|
|
36329
36796
|
const source = tasks ?? BOUNDED_MEMORY_FIXTURE;
|
|
36330
36797
|
const payload = JSON.stringify(source);
|
|
36331
|
-
return
|
|
36798
|
+
return createHash13("sha256").update(payload, "utf8").digest("hex");
|
|
36332
36799
|
}
|
|
36333
36800
|
|
|
36334
36801
|
// src/benchmarks/remnic/bounded-memory-contracts/agent.ts
|
|
@@ -37588,13 +38055,13 @@ function wrapJudgeWithCache(args) {
|
|
|
37588
38055
|
// differentiator is part of the prompt hash. Bumping
|
|
37589
38056
|
// JUDGE_CACHE_PROTOCOL_VERSION invalidates verdicts when judge
|
|
37590
38057
|
// prompt/parse semantics change (PR #1591, High).
|
|
37591
|
-
judgePromptHash:
|
|
38058
|
+
judgePromptHash: createHash14("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("").update(args.amaBenchJudgeProtocol).update("").update(args.role).digest("hex"),
|
|
37592
38059
|
judgeModelId: args.provider?.model !== void 0 && args.provider.model.length > 0 ? `${args.provider.model}${crossJudgeIdSuffix}` : `unknown-${args.role}-judge`,
|
|
37593
38060
|
// Full judge configuration, deterministically serialized (sorted
|
|
37594
38061
|
// keys) so provider/base-url/retry changes produce fresh cache
|
|
37595
38062
|
// keys. `role` is included so primary and cross judges never
|
|
37596
38063
|
// share a paramsHash.
|
|
37597
|
-
judgeParamsHash:
|
|
38064
|
+
judgeParamsHash: createHash14("sha256").update(
|
|
37598
38065
|
stableStringify2({
|
|
37599
38066
|
role: args.role,
|
|
37600
38067
|
provider: args.provider
|
|
@@ -38300,7 +38767,7 @@ function formatSignedScore(value) {
|
|
|
38300
38767
|
}
|
|
38301
38768
|
|
|
38302
38769
|
// src/stats/locomo-recall-delta.ts
|
|
38303
|
-
import { createHash as
|
|
38770
|
+
import { createHash as createHash15 } from "crypto";
|
|
38304
38771
|
import { basename } from "path";
|
|
38305
38772
|
var LOCOMO_FULL_TASK_COUNT = 1986;
|
|
38306
38773
|
var LOCOMO_RECALL_EXCERPT_CHARS = 240;
|
|
@@ -38781,7 +39248,7 @@ function normalizeText3(value) {
|
|
|
38781
39248
|
return value.replaceAll("\r\n", "\n").replaceAll("\r", "\n");
|
|
38782
39249
|
}
|
|
38783
39250
|
function sha2562(value) {
|
|
38784
|
-
return
|
|
39251
|
+
return createHash15("sha256").update(value).digest("hex");
|
|
38785
39252
|
}
|
|
38786
39253
|
function stableJson(value) {
|
|
38787
39254
|
return JSON.stringify(value);
|
|
@@ -38816,6 +39283,349 @@ function formatSignedScore2(value) {
|
|
|
38816
39283
|
return `${value >= 0 ? "+" : ""}${formatScore2(value)}`;
|
|
38817
39284
|
}
|
|
38818
39285
|
|
|
39286
|
+
// src/benchmarks/published/locomo/retrieval-trace-runner.ts
|
|
39287
|
+
var LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION = 1;
|
|
39288
|
+
var LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION = 1;
|
|
39289
|
+
var LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION = 1;
|
|
39290
|
+
async function preflightLoCoMoRetrievalTraceCapture(options) {
|
|
39291
|
+
assertCaptureOptions(options);
|
|
39292
|
+
assertProviderFreeRetrievalConfig(options.retrievalConfig);
|
|
39293
|
+
const loaded = await loadLoCoMoDataset("full", options.datasetDir);
|
|
39294
|
+
const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
|
|
39295
|
+
const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
|
|
39296
|
+
const selectable = plans.flatMap(
|
|
39297
|
+
(plan, planIndex) => plan.trials.map((trial) => ({ taskId: trial.taskId, planIndex }))
|
|
39298
|
+
);
|
|
39299
|
+
selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
|
|
39300
|
+
}
|
|
39301
|
+
function buildProviderFreeLoCoMoRetrievalConfig(retrievalConfig) {
|
|
39302
|
+
const sanitized = sanitizeProviderFreeRetrievalConfig(retrievalConfig);
|
|
39303
|
+
return assertProviderFreeRetrievalConfig({
|
|
39304
|
+
...sanitized,
|
|
39305
|
+
localLlmEnabled: false,
|
|
39306
|
+
localLlmFastEnabled: false,
|
|
39307
|
+
recallPlannerEnabled: false,
|
|
39308
|
+
embeddingFallbackEnabled: false,
|
|
39309
|
+
hostEmbeddingProviderEnabled: false,
|
|
39310
|
+
openaiApiKey: false,
|
|
39311
|
+
modelSource: "plugin"
|
|
39312
|
+
});
|
|
39313
|
+
}
|
|
39314
|
+
async function captureLoCoMoRetrievalTrace(options) {
|
|
39315
|
+
assertCaptureOptions(options);
|
|
39316
|
+
const retrievalConfig = assertProviderFreeRetrievalConfig(options.retrievalConfig);
|
|
39317
|
+
const recallWithTrace = options.system.recallWithTrace?.bind(options.system);
|
|
39318
|
+
if (!recallWithTrace) {
|
|
39319
|
+
throw new Error("LoCoMo retrieval trace capture requires system.recallWithTrace().");
|
|
39320
|
+
}
|
|
39321
|
+
const loaded = await loadLoCoMoDataset("full", options.datasetDir);
|
|
39322
|
+
const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
|
|
39323
|
+
const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
|
|
39324
|
+
const selectable = plans.flatMap(
|
|
39325
|
+
(plan, planIndex) => plan.trials.map(
|
|
39326
|
+
(trial) => ({
|
|
39327
|
+
taskId: trial.taskId,
|
|
39328
|
+
question: trial.question,
|
|
39329
|
+
recallSessionIds: [...trial.recallSessionIds],
|
|
39330
|
+
planIndex
|
|
39331
|
+
})
|
|
39332
|
+
)
|
|
39333
|
+
);
|
|
39334
|
+
const selection = selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
|
|
39335
|
+
const selectedIds = new Set(selection.selectedTaskIds);
|
|
39336
|
+
const tasks = [];
|
|
39337
|
+
for (let planIndex = 0; planIndex < plans.length; planIndex += 1) {
|
|
39338
|
+
const selected = selectable.filter((task) => task.planIndex === planIndex && selectedIds.has(task.taskId));
|
|
39339
|
+
if (selected.length === 0) continue;
|
|
39340
|
+
const plan = plans[planIndex];
|
|
39341
|
+
if (!plan) throw new Error(`Missing LoCoMo plan at index ${planIndex}.`);
|
|
39342
|
+
await options.system.reset();
|
|
39343
|
+
for (const session of plan.ingestSessions) {
|
|
39344
|
+
if (session.messages.length > 0) {
|
|
39345
|
+
await options.system.store(session.sessionId, session.messages);
|
|
39346
|
+
}
|
|
39347
|
+
}
|
|
39348
|
+
await options.system.drain?.();
|
|
39349
|
+
for (const selectedTask of selected) {
|
|
39350
|
+
const recallBudgetChars = benchmarkRecallBudgetForSessionCount(selectedTask.recallSessionIds.length);
|
|
39351
|
+
const recalled = await Promise.all(
|
|
39352
|
+
selectedTask.recallSessionIds.map(async (sessionId) => {
|
|
39353
|
+
const result = await recallWithTrace(sessionId, selectedTask.question, recallBudgetChars);
|
|
39354
|
+
return {
|
|
39355
|
+
text: result.text,
|
|
39356
|
+
receipt: {
|
|
39357
|
+
session: digestContent(sessionId),
|
|
39358
|
+
trace: sanitizeStructuralTrace(result.trace)
|
|
39359
|
+
}
|
|
39360
|
+
};
|
|
39361
|
+
})
|
|
39362
|
+
);
|
|
39363
|
+
const rawRecalledText = recalled.map((entry) => entry.text).filter(Boolean).join("\n\n");
|
|
39364
|
+
const sanitized = sanitizeLoCoMoRecallText({
|
|
39365
|
+
question: selectedTask.question,
|
|
39366
|
+
recalledText: rawRecalledText
|
|
39367
|
+
});
|
|
39368
|
+
const composition = prioritizeLoCoMoRecallTextWithTrace({
|
|
39369
|
+
question: selectedTask.question,
|
|
39370
|
+
recalledText: sanitized,
|
|
39371
|
+
multiHopRecallComposition
|
|
39372
|
+
});
|
|
39373
|
+
tasks.push({
|
|
39374
|
+
taskId: selectedTask.taskId,
|
|
39375
|
+
question: digestContent(selectedTask.question),
|
|
39376
|
+
recallBudgetChars,
|
|
39377
|
+
sessions: recalled.map((entry) => entry.receipt),
|
|
39378
|
+
composition: composition.receipt
|
|
39379
|
+
});
|
|
39380
|
+
}
|
|
39381
|
+
}
|
|
39382
|
+
const withoutHash = {
|
|
39383
|
+
schemaVersion: LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
|
|
39384
|
+
benchmarkId: "locomo",
|
|
39385
|
+
captureKind: "retrieval-only",
|
|
39386
|
+
sensitivity: {
|
|
39387
|
+
classification: "restricted",
|
|
39388
|
+
contentEncoding: "sha256+length",
|
|
39389
|
+
containsGold: false,
|
|
39390
|
+
containsRawContent: false
|
|
39391
|
+
},
|
|
39392
|
+
provenance: {
|
|
39393
|
+
gitSha: options.gitSha,
|
|
39394
|
+
remnicVersion: options.remnicVersion,
|
|
39395
|
+
runtimeProfile: options.runtimeProfile,
|
|
39396
|
+
adapterMode: "direct",
|
|
39397
|
+
replayExtractionMode: "skip",
|
|
39398
|
+
providerFree: true,
|
|
39399
|
+
dataset: { id: "locomo-10", sha256: loaded.sha256 },
|
|
39400
|
+
retrievalConfigSha256: hashCanonicalJson(retrievalConfig),
|
|
39401
|
+
recallBudget: {
|
|
39402
|
+
algorithm: "benchmarkRecallBudgetForSessionCount",
|
|
39403
|
+
version: LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION
|
|
39404
|
+
}
|
|
39405
|
+
},
|
|
39406
|
+
selection,
|
|
39407
|
+
tasks
|
|
39408
|
+
};
|
|
39409
|
+
return {
|
|
39410
|
+
...withoutHash,
|
|
39411
|
+
artifactHash: hashCanonicalJson(withoutHash)
|
|
39412
|
+
};
|
|
39413
|
+
}
|
|
39414
|
+
function selectLoCoMoRetrievalTraceTasks(tasks, selector) {
|
|
39415
|
+
const allIds = tasks.map((task) => task.taskId);
|
|
39416
|
+
if (new Set(allIds).size !== allIds.length) {
|
|
39417
|
+
throw new Error("LoCoMo retrieval trace task ids must be unique.");
|
|
39418
|
+
}
|
|
39419
|
+
let selected;
|
|
39420
|
+
let algorithm;
|
|
39421
|
+
let seed;
|
|
39422
|
+
const hasTaskIds = "taskIds" in selector && selector.taskIds !== void 0;
|
|
39423
|
+
const hasSampleSize = "sampleSize" in selector && selector.sampleSize !== void 0;
|
|
39424
|
+
if (Number(hasTaskIds) + Number(hasSampleSize) !== 1) {
|
|
39425
|
+
throw new Error("Choose exactly one LoCoMo retrieval trace selector.");
|
|
39426
|
+
}
|
|
39427
|
+
if (hasTaskIds && "seed" in selector && selector.seed !== void 0) {
|
|
39428
|
+
throw new Error("LoCoMo retrieval trace seed is valid only for seeded sampling.");
|
|
39429
|
+
}
|
|
39430
|
+
if (hasTaskIds) {
|
|
39431
|
+
algorithm = "explicit-task-ids";
|
|
39432
|
+
const requestedTaskIds = selector.taskIds;
|
|
39433
|
+
if (!requestedTaskIds) throw new Error("LoCoMo explicit task ids are required.");
|
|
39434
|
+
const requested = [...requestedTaskIds];
|
|
39435
|
+
if (requested.length === 0) {
|
|
39436
|
+
throw new Error("LoCoMo retrieval trace explicit task selection cannot be empty.");
|
|
39437
|
+
}
|
|
39438
|
+
if (new Set(requested).size !== requested.length) {
|
|
39439
|
+
throw new Error("LoCoMo retrieval trace explicit task ids must not contain duplicates.");
|
|
39440
|
+
}
|
|
39441
|
+
const available = new Set(allIds);
|
|
39442
|
+
const unknown = requested.filter((taskId) => !available.has(taskId));
|
|
39443
|
+
if (unknown.length > 0) {
|
|
39444
|
+
throw new Error(`Unknown LoCoMo retrieval trace task id: ${unknown[0]}`);
|
|
39445
|
+
}
|
|
39446
|
+
const requestedSet = new Set(requested);
|
|
39447
|
+
selected = allIds.filter((taskId) => requestedSet.has(taskId));
|
|
39448
|
+
} else {
|
|
39449
|
+
algorithm = "sha256-seeded-sample";
|
|
39450
|
+
const sampleSize = selector.sampleSize;
|
|
39451
|
+
seed = selector.seed;
|
|
39452
|
+
if (sampleSize === void 0 || seed === void 0) {
|
|
39453
|
+
throw new Error("LoCoMo seeded sampling requires sampleSize and seed.");
|
|
39454
|
+
}
|
|
39455
|
+
if (!Number.isSafeInteger(sampleSize) || sampleSize <= 0 || sampleSize > allIds.length) {
|
|
39456
|
+
throw new Error(`LoCoMo retrieval trace sampleSize must be an integer from 1 to ${allIds.length}.`);
|
|
39457
|
+
}
|
|
39458
|
+
if (!Number.isSafeInteger(seed) || seed < 0) {
|
|
39459
|
+
throw new Error("LoCoMo retrieval trace seed must be a non-negative safe integer.");
|
|
39460
|
+
}
|
|
39461
|
+
const sampled = [...allIds].sort((left, right) => {
|
|
39462
|
+
const leftHash = hashString(`${seed}\0${left}`);
|
|
39463
|
+
const rightHash = hashString(`${seed}\0${right}`);
|
|
39464
|
+
return leftHash.localeCompare(rightHash) || left.localeCompare(right);
|
|
39465
|
+
}).slice(0, sampleSize);
|
|
39466
|
+
const sampledSet = new Set(sampled);
|
|
39467
|
+
selected = allIds.filter((taskId) => sampledSet.has(taskId));
|
|
39468
|
+
}
|
|
39469
|
+
return {
|
|
39470
|
+
algorithm,
|
|
39471
|
+
version: LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
|
|
39472
|
+
...seed === void 0 ? {} : { seed },
|
|
39473
|
+
candidateCount: allIds.length,
|
|
39474
|
+
selectedCount: selected.length,
|
|
39475
|
+
selectedTaskIds: selected,
|
|
39476
|
+
selectedTaskIdsSha256: hashCanonicalJson(selected)
|
|
39477
|
+
};
|
|
39478
|
+
}
|
|
39479
|
+
function serializeLoCoMoRetrievalTraceReceipt(receipt) {
|
|
39480
|
+
return `${canonicalJsonStringify(receipt, 2)}
|
|
39481
|
+
`;
|
|
39482
|
+
}
|
|
39483
|
+
function sanitizeStructuralTrace(trace2) {
|
|
39484
|
+
return {
|
|
39485
|
+
schemaVersion: trace2.schemaVersion,
|
|
39486
|
+
sensitivity: { ...trace2.sensitivity },
|
|
39487
|
+
sections: trace2.sections.map((section) => ({ ...section })),
|
|
39488
|
+
selections: trace2.selections.map(({ summary, ...selection }) => ({
|
|
39489
|
+
...selection,
|
|
39490
|
+
...selection.archiveRowIds === void 0 ? {} : { archiveRowIds: [...selection.archiveRowIds] },
|
|
39491
|
+
...summary === void 0 ? {} : { summary: { depth: summary.depth, msgStart: summary.msgStart, msgEnd: summary.msgEnd } }
|
|
39492
|
+
})),
|
|
39493
|
+
lcmCandidates: trace2.lcmCandidates.map((candidate) => ({ ...candidate })),
|
|
39494
|
+
...trace2.coreCapture === void 0 ? {} : {
|
|
39495
|
+
coreCapture: {
|
|
39496
|
+
budget: { ...trace2.coreCapture.budget },
|
|
39497
|
+
filters: trace2.coreCapture.filters.map((filter) => ({ ...filter })),
|
|
39498
|
+
results: trace2.coreCapture.results.map((result) => {
|
|
39499
|
+
assertMemoryIdRef(result.memoryIdRef);
|
|
39500
|
+
const score = result.scoreDecomposition;
|
|
39501
|
+
return {
|
|
39502
|
+
memoryIdRef: {
|
|
39503
|
+
sha256: result.memoryIdRef.sha256,
|
|
39504
|
+
length: result.memoryIdRef.length
|
|
39505
|
+
},
|
|
39506
|
+
servedBy: result.servedBy,
|
|
39507
|
+
scoreDecomposition: {
|
|
39508
|
+
...score.vector === void 0 ? {} : { vector: score.vector },
|
|
39509
|
+
...score.bm25 === void 0 ? {} : { bm25: score.bm25 },
|
|
39510
|
+
...score.importance === void 0 ? {} : { importance: score.importance },
|
|
39511
|
+
...score.mmrPenalty === void 0 ? {} : { mmrPenalty: score.mmrPenalty },
|
|
39512
|
+
...score.tierPrior === void 0 ? {} : { tierPrior: score.tierPrior },
|
|
39513
|
+
...score.reinforcementBoost === void 0 ? {} : { reinforcementBoost: score.reinforcementBoost },
|
|
39514
|
+
final: score.final
|
|
39515
|
+
},
|
|
39516
|
+
admittedBy: [...result.admittedBy],
|
|
39517
|
+
...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
|
|
39518
|
+
...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
|
|
39519
|
+
...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
|
|
39520
|
+
};
|
|
39521
|
+
})
|
|
39522
|
+
}
|
|
39523
|
+
},
|
|
39524
|
+
budget: { ...trace2.budget }
|
|
39525
|
+
};
|
|
39526
|
+
}
|
|
39527
|
+
function digestContent(value) {
|
|
39528
|
+
return {
|
|
39529
|
+
sha256: hashString(value),
|
|
39530
|
+
charCount: value.length,
|
|
39531
|
+
lineCount: value.length === 0 ? 0 : value.split("\n").length
|
|
39532
|
+
};
|
|
39533
|
+
}
|
|
39534
|
+
function assertCaptureOptions(options) {
|
|
39535
|
+
if (!options.datasetDir.trim()) {
|
|
39536
|
+
throw new Error("LoCoMo retrieval trace capture requires datasetDir.");
|
|
39537
|
+
}
|
|
39538
|
+
if (options.runtimeProfile !== "baseline" && options.runtimeProfile !== "real") {
|
|
39539
|
+
throw new Error('LoCoMo retrieval trace runtimeProfile must be "baseline" or "real".');
|
|
39540
|
+
}
|
|
39541
|
+
if (!options.gitSha.trim() || !options.remnicVersion.trim() || options.gitSha === "unknown" || options.remnicVersion === "unknown") {
|
|
39542
|
+
throw new Error("LoCoMo retrieval trace provenance requires gitSha and remnicVersion.");
|
|
39543
|
+
}
|
|
39544
|
+
if (options.providerFreeConfirmed !== true) {
|
|
39545
|
+
throw new Error("LoCoMo retrieval trace capture requires explicit provider-free confirmation.");
|
|
39546
|
+
}
|
|
39547
|
+
}
|
|
39548
|
+
function assertProviderFreeRetrievalConfig(value) {
|
|
39549
|
+
const config = assertJsonConfig(value);
|
|
39550
|
+
for (const key of [
|
|
39551
|
+
"localLlmEnabled",
|
|
39552
|
+
"localLlmFastEnabled",
|
|
39553
|
+
"recallPlannerEnabled",
|
|
39554
|
+
"embeddingFallbackEnabled",
|
|
39555
|
+
"hostEmbeddingProviderEnabled",
|
|
39556
|
+
"openaiApiKey"
|
|
39557
|
+
]) {
|
|
39558
|
+
if (config[key] !== false) {
|
|
39559
|
+
throw new Error(`retrievalConfig.${key} must be false for provider-free capture.`);
|
|
39560
|
+
}
|
|
39561
|
+
}
|
|
39562
|
+
if (config.modelSource !== "plugin") {
|
|
39563
|
+
throw new Error('retrievalConfig.modelSource must be "plugin" for provider-free capture.');
|
|
39564
|
+
}
|
|
39565
|
+
return config;
|
|
39566
|
+
}
|
|
39567
|
+
function assertMemoryIdRef(value) {
|
|
39568
|
+
if (!value || typeof value !== "object" || !/^[0-9a-f]{64}$/u.test(value.sha256) || !Number.isSafeInteger(value.length) || value.length <= 0) {
|
|
39569
|
+
throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
|
|
39570
|
+
}
|
|
39571
|
+
}
|
|
39572
|
+
function assertJsonConfig(value, path40 = "retrievalConfig") {
|
|
39573
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
39574
|
+
if (typeof value === "number") {
|
|
39575
|
+
if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
|
|
39576
|
+
return value;
|
|
39577
|
+
}
|
|
39578
|
+
if (Array.isArray(value)) {
|
|
39579
|
+
return value.map((entry, index) => assertJsonConfig(entry, `${path40}[${index}]`));
|
|
39580
|
+
}
|
|
39581
|
+
if (!value || typeof value !== "object") {
|
|
39582
|
+
throw new Error(`${path40} must be JSON-serializable and provider-free.`);
|
|
39583
|
+
}
|
|
39584
|
+
const output = {};
|
|
39585
|
+
for (const key of Object.keys(value).sort()) {
|
|
39586
|
+
const child = value[key];
|
|
39587
|
+
if (key === "openaiApiKey") {
|
|
39588
|
+
if (child !== false) {
|
|
39589
|
+
throw new Error(`${path40}.${key} must be exactly false for provider-free capture.`);
|
|
39590
|
+
}
|
|
39591
|
+
output[key] = false;
|
|
39592
|
+
continue;
|
|
39593
|
+
}
|
|
39594
|
+
if (isSecretKey(key)) {
|
|
39595
|
+
throw new Error(`${path40}.${key} contains secret-bearing configuration.`);
|
|
39596
|
+
}
|
|
39597
|
+
if (child === void 0) continue;
|
|
39598
|
+
if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
|
|
39599
|
+
throw new Error(`${path40}.${key} is provider-capable configuration.`);
|
|
39600
|
+
}
|
|
39601
|
+
output[key] = assertJsonConfig(child, `${path40}.${key}`);
|
|
39602
|
+
}
|
|
39603
|
+
return output;
|
|
39604
|
+
}
|
|
39605
|
+
function sanitizeProviderFreeRetrievalConfig(value, path40 = "retrievalConfig") {
|
|
39606
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
39607
|
+
if (typeof value === "number") {
|
|
39608
|
+
if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
|
|
39609
|
+
return value;
|
|
39610
|
+
}
|
|
39611
|
+
if (Array.isArray(value)) {
|
|
39612
|
+
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path40}[${index}]`));
|
|
39613
|
+
}
|
|
39614
|
+
if (!value || typeof value !== "object") {
|
|
39615
|
+
throw new Error(`${path40} must be JSON-serializable.`);
|
|
39616
|
+
}
|
|
39617
|
+
const output = {};
|
|
39618
|
+
for (const key of Object.keys(value).sort()) {
|
|
39619
|
+
const child = value[key];
|
|
39620
|
+
if (child === void 0) continue;
|
|
39621
|
+
if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
|
|
39622
|
+
continue;
|
|
39623
|
+
}
|
|
39624
|
+
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path40}.${key}`);
|
|
39625
|
+
}
|
|
39626
|
+
return output;
|
|
39627
|
+
}
|
|
39628
|
+
|
|
38819
39629
|
// src/integrity/sealed-qrels.ts
|
|
38820
39630
|
import { readFile as readFile20 } from "fs/promises";
|
|
38821
39631
|
function isSealedQrelsArtifact(value) {
|
|
@@ -40219,7 +41029,7 @@ var chatFixture = {
|
|
|
40219
41029
|
};
|
|
40220
41030
|
|
|
40221
41031
|
// src/judges/calibration-slice.ts
|
|
40222
|
-
import { createHash as
|
|
41032
|
+
import { createHash as createHash16, randomBytes as randomBytes3 } from "crypto";
|
|
40223
41033
|
import { chmod as chmod2, lstat as lstat4, mkdir as mkdir18, open as open2, readFile as readFile22, rename as rename4, unlink as unlink4, writeFile as writeFile17 } from "fs/promises";
|
|
40224
41034
|
import path37 from "path";
|
|
40225
41035
|
|
|
@@ -40368,7 +41178,7 @@ function selectCalibrationSlice(questionIds, size = CALIBRATION_SLICE_SIZE) {
|
|
|
40368
41178
|
unique.push(id);
|
|
40369
41179
|
}
|
|
40370
41180
|
}
|
|
40371
|
-
return unique.map((id) => ({ id, digest:
|
|
41181
|
+
return unique.map((id) => ({ id, digest: createHash16("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
|
|
40372
41182
|
}
|
|
40373
41183
|
async function runJudgeCalibration(options) {
|
|
40374
41184
|
const binScore = options.binScore ?? ((score) => binarizeJudgeScore(score));
|
|
@@ -40485,7 +41295,7 @@ function hashOrderedQuestionIds(questionIds) {
|
|
|
40485
41295
|
if (questionIds.some((id) => typeof id !== "string" || id.length === 0)) {
|
|
40486
41296
|
throw new Error("hashOrderedQuestionIds: question ids must be non-empty strings.");
|
|
40487
41297
|
}
|
|
40488
|
-
return
|
|
41298
|
+
return createHash16("sha256").update(JSON.stringify(questionIds), "utf8").digest("hex");
|
|
40489
41299
|
}
|
|
40490
41300
|
function validatePinnedQuestionIds(ids, availableIds) {
|
|
40491
41301
|
if (ids.length === 0 || ids.length > CALIBRATION_SLICE_SIZE || ids.some((id) => typeof id !== "string" || id.length === 0) || new Set(ids).size !== ids.length) {
|
|
@@ -40498,7 +41308,7 @@ function validatePinnedQuestionIds(ids, availableIds) {
|
|
|
40498
41308
|
return [...ids];
|
|
40499
41309
|
}
|
|
40500
41310
|
function hashCalibrationAnswerSet(answers) {
|
|
40501
|
-
return
|
|
41311
|
+
return createHash16("sha256").update(JSON.stringify(answers.map((answer) => [
|
|
40502
41312
|
answer.questionId,
|
|
40503
41313
|
answer.question,
|
|
40504
41314
|
answer.predicted,
|
|
@@ -40564,7 +41374,7 @@ async function loadOrInitializeCheckpoint(benchmarkId, provenance, sliceQuestion
|
|
|
40564
41374
|
frontierJudgeConfigHash: provenance.frontierJudgeConfigHash,
|
|
40565
41375
|
binningIdentity: provenance.binningIdentity
|
|
40566
41376
|
};
|
|
40567
|
-
const contractHash =
|
|
41377
|
+
const contractHash = createHash16("sha256").update(stableJson2(contract)).digest("hex");
|
|
40568
41378
|
let raw;
|
|
40569
41379
|
try {
|
|
40570
41380
|
const info = await lstat4(checkpointPath);
|
|
@@ -42085,7 +42895,7 @@ function createMitigatedTarget(config) {
|
|
|
42085
42895
|
}
|
|
42086
42896
|
|
|
42087
42897
|
// src/coding-graph/generator.ts
|
|
42088
|
-
import { createHash as
|
|
42898
|
+
import { createHash as createHash17 } from "crypto";
|
|
42089
42899
|
function createSeededRng3(seed) {
|
|
42090
42900
|
let state = seed >>> 0;
|
|
42091
42901
|
return function rng() {
|
|
@@ -42114,7 +42924,7 @@ var EDGE_TYPE_WEIGHTS = [
|
|
|
42114
42924
|
var PROVENANCE_VALUES = ["heuristic", "heuristic", "heuristic", "trace"];
|
|
42115
42925
|
var AVG_BYTES_PER_LINE = 40;
|
|
42116
42926
|
function hashContent(input) {
|
|
42117
|
-
return
|
|
42927
|
+
return createHash17("sha256").update(input).digest("hex").slice(0, 16);
|
|
42118
42928
|
}
|
|
42119
42929
|
function generateSyntheticRepo(config) {
|
|
42120
42930
|
const rng = createSeededRng3(config.seed);
|
|
@@ -42686,6 +43496,9 @@ export {
|
|
|
42686
43496
|
LOCOMO_FULL_TASK_COUNT,
|
|
42687
43497
|
LOCOMO_RECALL_DIFF_LINE_LIMIT,
|
|
42688
43498
|
LOCOMO_RECALL_EXCERPT_CHARS,
|
|
43499
|
+
LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION,
|
|
43500
|
+
LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
|
|
43501
|
+
LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
|
|
42689
43502
|
LONG_MEM_EVAL_DATASET_FILENAMES,
|
|
42690
43503
|
LettaMemCorrectAdapter,
|
|
42691
43504
|
LocalLabPreflightError,
|
|
@@ -42742,10 +43555,12 @@ export {
|
|
|
42742
43555
|
buildCodexCreditReceipt,
|
|
42743
43556
|
buildJudgePayload,
|
|
42744
43557
|
buildOracleTrajectoryRecall,
|
|
43558
|
+
buildProviderFreeLoCoMoRetrievalConfig,
|
|
42745
43559
|
buildSchemaTierFixture,
|
|
42746
43560
|
buildSchemaTierSmokeFixture,
|
|
42747
43561
|
calendarFixture,
|
|
42748
43562
|
canonicalJsonStringify,
|
|
43563
|
+
captureLoCoMoRetrievalTrace,
|
|
42749
43564
|
captureMachineFingerprint,
|
|
42750
43565
|
chatFixture,
|
|
42751
43566
|
checkCodingGraphRegression,
|
|
@@ -42814,6 +43629,7 @@ export {
|
|
|
42814
43629
|
getAblationCell,
|
|
42815
43630
|
getBenchmark,
|
|
42816
43631
|
getBenchmarkLowerIsBetter,
|
|
43632
|
+
getGitSha,
|
|
42817
43633
|
getMemoryEvalDimension,
|
|
42818
43634
|
getProviderBackedJudgePromptIdentity,
|
|
42819
43635
|
getRemnicVersion,
|
|
@@ -42867,6 +43683,7 @@ export {
|
|
|
42867
43683
|
parseSealedQrels,
|
|
42868
43684
|
pickStableQualifiedName,
|
|
42869
43685
|
precisionAtK,
|
|
43686
|
+
preflightLoCoMoRetrievalTraceCapture,
|
|
42870
43687
|
preflightLocalLabRole,
|
|
42871
43688
|
projectFolderFixture,
|
|
42872
43689
|
recallAtK,
|
|
@@ -42922,6 +43739,7 @@ export {
|
|
|
42922
43739
|
selectFixtureVariant,
|
|
42923
43740
|
serializeBenchmarkArtifact,
|
|
42924
43741
|
serializeJsonl,
|
|
43742
|
+
serializeLoCoMoRetrievalTraceReceipt,
|
|
42925
43743
|
serializeSealedQrels,
|
|
42926
43744
|
shuffleTasks,
|
|
42927
43745
|
timed,
|