@remnic/bench 9.6.31 → 9.6.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.d.ts +254 -16
  2. package/dist/index.js +948 -130
  3. package/package.json +3 -3
package/dist/index.js CHANGED
@@ -597,7 +597,7 @@ var LettaMemCorrectAdapter = class {
597
597
  var REQUIRED_FRONTMATTER_FIELDS = ["title", "type", "state", "created", "see-also"];
598
598
 
599
599
  // src/adapters/remnic-adapter.ts
600
- import { createHash } from "crypto";
600
+ import { createHash as createHash2 } from "crypto";
601
601
  import { execFile } from "child_process";
602
602
  import {
603
603
  chmod,
@@ -629,6 +629,226 @@ import {
629
629
  parseEntityFile,
630
630
  serializeEntityFile
631
631
  } from "@remnic/core";
632
+ import {
633
+ lcmEvidenceIdentity
634
+ } from "@remnic/core/lcm";
635
+
636
+ // src/adapters/remnic-recall-trace.ts
637
+ import { createHash } from "crypto";
638
+ import {
639
+ lcmArchiveRowId
640
+ } from "@remnic/core/lcm";
641
+ function visibleRange(start, end, returnedChars) {
642
+ const visibleStart = Math.min(start, returnedChars);
643
+ return {
644
+ visibleStart,
645
+ visibleEnd: Math.max(visibleStart, Math.min(end, returnedChars))
646
+ };
647
+ }
648
+ function projectBenchCoreCapture(snapshot) {
649
+ return {
650
+ snapshotId: snapshot.snapshotId,
651
+ capturedAt: snapshot.capturedAt,
652
+ ...snapshot.traceId === void 0 ? {} : { traceId: snapshot.traceId },
653
+ budget: { chars: snapshot.budget.chars, used: snapshot.budget.used },
654
+ filters: snapshot.filters.map(({ name, considered, admitted }) => ({
655
+ name,
656
+ considered,
657
+ admitted
658
+ })),
659
+ results: snapshot.results.map((result) => {
660
+ const scores = result.scoreDecomposition;
661
+ return {
662
+ memoryIdRef: {
663
+ sha256: createHash("sha256").update(result.memoryId, "utf8").digest("hex"),
664
+ length: Buffer.byteLength(result.memoryId, "utf8")
665
+ },
666
+ servedBy: result.servedBy,
667
+ scoreDecomposition: {
668
+ ...typeof scores.vector === "number" ? { vector: scores.vector } : {},
669
+ ...typeof scores.bm25 === "number" ? { bm25: scores.bm25 } : {},
670
+ ...typeof scores.importance === "number" ? { importance: scores.importance } : {},
671
+ ...typeof scores.mmrPenalty === "number" ? { mmrPenalty: scores.mmrPenalty } : {},
672
+ ...typeof scores.tierPrior === "number" ? { tierPrior: scores.tierPrior } : {},
673
+ ...typeof scores.reinforcementBoost === "number" ? { reinforcementBoost: scores.reinforcementBoost } : {},
674
+ final: scores.final
675
+ },
676
+ admittedBy: [...result.admittedBy],
677
+ ...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
678
+ ...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
679
+ ...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
680
+ };
681
+ })
682
+ };
683
+ }
684
+ function createBenchRecallTraceRecorder(requestedChars) {
685
+ const sections = [];
686
+ const pendingSelections = [];
687
+ const lcmCandidates = [];
688
+ let composedChars = 0;
689
+ let coreCapture;
690
+ const sectionById = (sectionId) => {
691
+ const section = sections.find((entry) => entry.id === sectionId);
692
+ if (!section) throw new Error(`Unknown benchmark recall trace section: ${sectionId}`);
693
+ return section;
694
+ };
695
+ const appendRelativeSelection = (sectionId, kind, start, end, lineageStatus, fields = {}) => {
696
+ const section = sectionById(sectionId);
697
+ const contentLength = section.contentEnd - section.contentStart;
698
+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > contentLength) {
699
+ throw new Error(
700
+ `Invalid benchmark recall trace range for ${sectionId}: ${start}..${end}.`
701
+ );
702
+ }
703
+ pendingSelections.push({
704
+ sectionId,
705
+ kind,
706
+ lineageStatus,
707
+ composedStart: section.contentStart + start,
708
+ composedEnd: section.contentStart + end,
709
+ ...fields
710
+ });
711
+ };
712
+ return {
713
+ appendSection(id, source, renderedLength) {
714
+ if (!Number.isSafeInteger(renderedLength) || renderedLength < 0) {
715
+ throw new Error("Benchmark recall trace section length must be a non-negative integer.");
716
+ }
717
+ if (sections.some((section) => section.id === id)) {
718
+ throw new Error(`Duplicate benchmark recall trace section: ${id}`);
719
+ }
720
+ const separatorStart = composedChars;
721
+ const contentStart = composedChars + (sections.length === 0 ? 0 : 2);
722
+ const contentEnd = contentStart + renderedLength;
723
+ sections.push({
724
+ id,
725
+ source,
726
+ separatorStart,
727
+ contentStart,
728
+ contentEnd,
729
+ composedStart: separatorStart,
730
+ composedEnd: contentEnd,
731
+ visibleStart: 0,
732
+ visibleEnd: 0,
733
+ visibleChars: 0
734
+ });
735
+ composedChars = contentEnd;
736
+ },
737
+ recordEvidenceSelections(sectionId, receipts) {
738
+ for (const receipt of receipts) {
739
+ appendRelativeSelection(
740
+ sectionId,
741
+ "evidence-block",
742
+ receipt.blockStart,
743
+ receipt.blockEnd,
744
+ receipt.item.archiveRowId === void 0 ? "unavailable" : "exact",
745
+ {
746
+ ...receipt.item.archiveRowId === void 0 ? {} : { archiveRowIds: [receipt.item.archiveRowId] },
747
+ ...receipt.item.turnIndex === void 0 ? {} : { turnIndex: receipt.item.turnIndex },
748
+ ...receipt.item.role === void 0 ? {} : { role: receipt.item.role },
749
+ ...receipt.item.score === void 0 ? {} : { score: receipt.item.score }
750
+ }
751
+ );
752
+ }
753
+ },
754
+ recordTrajectorySelections(sectionId, receipts) {
755
+ for (const receipt of receipts) {
756
+ appendRelativeSelection(
757
+ sectionId,
758
+ "trajectory-line",
759
+ receipt.lineStart,
760
+ receipt.lineEnd,
761
+ receipt.lineageStatus,
762
+ {
763
+ archiveRowIds: [
764
+ ...receipt.actionArchiveRowIds,
765
+ ...receipt.observationArchiveRowIds
766
+ ]
767
+ }
768
+ );
769
+ }
770
+ },
771
+ recordSummarySelections(sectionId, receipts) {
772
+ for (const receipt of receipts) {
773
+ appendRelativeSelection(
774
+ sectionId,
775
+ "lcm-summary",
776
+ receipt.entryStart,
777
+ receipt.entryEnd,
778
+ "exact",
779
+ {
780
+ summary: {
781
+ id: receipt.id,
782
+ depth: receipt.depth,
783
+ msgStart: receipt.msgStart,
784
+ msgEnd: receipt.msgEnd
785
+ }
786
+ }
787
+ );
788
+ }
789
+ },
790
+ recordRawRow(sectionId, range, row) {
791
+ const archiveRowId = lcmArchiveRowId(row);
792
+ appendRelativeSelection(
793
+ sectionId,
794
+ "raw-row",
795
+ range.start,
796
+ range.end,
797
+ archiveRowId === void 0 ? "unavailable" : "exact",
798
+ {
799
+ ...archiveRowId === void 0 ? {} : { archiveRowIds: [archiveRowId] },
800
+ turnIndex: row.turn_index,
801
+ role: row.role
802
+ }
803
+ );
804
+ },
805
+ recordLcmCandidate(candidate) {
806
+ lcmCandidates.push({ ...candidate });
807
+ },
808
+ recordCoreCapture(snapshot) {
809
+ coreCapture = snapshot ? projectBenchCoreCapture(snapshot) : void 0;
810
+ },
811
+ finalize(returnedChars) {
812
+ const normalizedReturnedChars = Math.max(0, Math.min(returnedChars, composedChars));
813
+ return {
814
+ schemaVersion: 1,
815
+ sensitivity: {
816
+ classification: "restricted",
817
+ contentEncoding: "sha256+length",
818
+ containsGold: false
819
+ },
820
+ sections: sections.map((section) => {
821
+ const visible = visibleRange(
822
+ section.composedStart,
823
+ section.composedEnd,
824
+ normalizedReturnedChars
825
+ );
826
+ return {
827
+ ...section,
828
+ ...visible,
829
+ visibleChars: visible.visibleEnd - visible.visibleStart
830
+ };
831
+ }),
832
+ selections: pendingSelections.map((selection) => ({
833
+ ...selection,
834
+ ...visibleRange(
835
+ selection.composedStart,
836
+ selection.composedEnd,
837
+ normalizedReturnedChars
838
+ )
839
+ })),
840
+ lcmCandidates: lcmCandidates.map((candidate) => ({ ...candidate })),
841
+ ...coreCapture === void 0 ? {} : { coreCapture },
842
+ budget: {
843
+ requestedChars,
844
+ composedChars,
845
+ returnedChars: normalizedReturnedChars,
846
+ truncated: normalizedReturnedChars < composedChars
847
+ }
848
+ };
849
+ }
850
+ };
851
+ }
632
852
 
633
853
  // src/recall-budget.ts
634
854
  var DEFAULT_BENCH_RECALL_BUDGET_CHARS = 24e3;
@@ -1264,7 +1484,7 @@ function benchCoreMemoryTier(memory) {
1264
1484
  return memory.path.includes(`${path.sep}cold${path.sep}`) ? "cold" : "hot";
1265
1485
  }
1266
1486
  function benchCoreMemorySource(sessionId) {
1267
- return `bench-replay-${createHash("sha256").update(sessionId).digest("hex").slice(0, 16)}`;
1487
+ return `bench-replay-${createHash2("sha256").update(sessionId).digest("hex").slice(0, 16)}`;
1268
1488
  }
1269
1489
  function resolveSessionScopedCorrectionDecision(plan, ownedIds, phase) {
1270
1490
  const actionTargets = [];
@@ -2009,7 +2229,8 @@ function createAdapterFactory(mode) {
2009
2229
  }
2010
2230
  return correctionAccess.service;
2011
2231
  };
2012
- return {
2232
+ const composeRecall = /* @__PURE__ */ Symbol("benchRecallComposer");
2233
+ const adapter = {
2013
2234
  async store(sessionId, messages, control) {
2014
2235
  throwIfBenchPhaseAborted(control, "store");
2015
2236
  sessionId = normalizeBenchSessionId(sessionId);
@@ -2128,7 +2349,7 @@ function createAdapterFactory(mode) {
2128
2349
  throw error;
2129
2350
  }
2130
2351
  },
2131
- async recall(sessionId, query, budgetChars, recallOptions = {}, control) {
2352
+ async [composeRecall](sessionId, query, budgetChars, recallOptions = {}, control, traceRecorder) {
2132
2353
  throwIfBenchPhaseAborted(control, "recall");
2133
2354
  const waitForRecall = (promise) => withBenchPhaseAbort(promise, control, "recall");
2134
2355
  sessionId = normalizeBenchSessionId(sessionId);
@@ -2150,6 +2371,10 @@ function createAdapterFactory(mode) {
2150
2371
  );
2151
2372
  }
2152
2373
  const sections = [];
2374
+ const appendSection = (id, source, rendered) => {
2375
+ traceRecorder?.appendSection(id, source, rendered.length);
2376
+ sections.push(rendered);
2377
+ };
2153
2378
  let usedChars = 0;
2154
2379
  const explicitReferences = historicalRecall ? [] : collectExplicitTurnReferences(query);
2155
2380
  const hasExplicitReferences = explicitReferences.length > 0;
@@ -2175,7 +2400,7 @@ function createAdapterFactory(mode) {
2175
2400
  }));
2176
2401
  if (temporalIntervalEvidence) {
2177
2402
  hasTemporalIntervalEvidence = true;
2178
- sections.push(temporalIntervalEvidence);
2403
+ appendSection("temporal-interval", "derived", temporalIntervalEvidence);
2179
2404
  usedChars += temporalIntervalEvidence.length;
2180
2405
  }
2181
2406
  }
@@ -2187,7 +2412,7 @@ function createAdapterFactory(mode) {
2187
2412
  }));
2188
2413
  if (dependencyVersionEvidence) {
2189
2414
  hasDependencyVersionEvidence = true;
2190
- sections.push(dependencyVersionEvidence);
2415
+ appendSection("dependency-version", "derived", dependencyVersionEvidence);
2191
2416
  usedChars += dependencyVersionEvidence.length;
2192
2417
  }
2193
2418
  }
@@ -2199,7 +2424,7 @@ function createAdapterFactory(mode) {
2199
2424
  maxChars: Math.min(3e3, Math.floor(budget * 0.25))
2200
2425
  }));
2201
2426
  if (latestQuantitativeEvidence) {
2202
- sections.push(latestQuantitativeEvidence);
2427
+ appendSection("latest-quantitative", "derived", latestQuantitativeEvidence);
2203
2428
  usedChars += latestQuantitativeEvidence.length;
2204
2429
  }
2205
2430
  }
@@ -2211,10 +2436,11 @@ function createAdapterFactory(mode) {
2211
2436
  }));
2212
2437
  if (userImplementationTargetEvidence) {
2213
2438
  hasUserImplementationTargetEvidence = true;
2214
- sections.push(userImplementationTargetEvidence);
2439
+ appendSection("implementation-targets", "derived", userImplementationTargetEvidence);
2215
2440
  usedChars += userImplementationTargetEvidence.length;
2216
2441
  }
2217
2442
  }
2443
+ const explicitCueSelections = [];
2218
2444
  const exactReferenceEvidence = historicalRecall || hasDependencyVersionEvidence ? "" : await waitForRecall(buildExplicitCueRecallSection({
2219
2445
  engine,
2220
2446
  sessionId,
@@ -2223,12 +2449,17 @@ function createAdapterFactory(mode) {
2223
2449
  maxItemChars: CORE_EXPLICIT_CUE_MAX_ITEM_CHARS,
2224
2450
  maxReferences: CORE_EXPLICIT_CUE_MAX_REFERENCES,
2225
2451
  includeBenchmarkAnchorCues: sessionId.startsWith("beam-"),
2226
- includeStructuredPlanCues: sessionId.startsWith("arena-")
2452
+ includeStructuredPlanCues: sessionId.startsWith("arena-"),
2453
+ ...traceRecorder ? { onEvidenceSelected: (receipt) => {
2454
+ explicitCueSelections.push(receipt);
2455
+ } } : {}
2227
2456
  }));
2228
2457
  if (exactReferenceEvidence) {
2229
- sections.push(exactReferenceEvidence);
2458
+ appendSection("explicit-cue", "explicit-cue", exactReferenceEvidence);
2459
+ traceRecorder?.recordEvidenceSelections("explicit-cue", explicitCueSelections);
2230
2460
  usedChars += exactReferenceEvidence.length;
2231
2461
  }
2462
+ const trajectorySelections = [];
2232
2463
  const trajectoryAnalysisEvidence = !historicalRecall && sessionId.startsWith("ama-") ? await waitForRecall(buildTrajectoryAnalysisRecallSection({
2233
2464
  engine,
2234
2465
  sessionId,
@@ -2236,10 +2467,17 @@ function createAdapterFactory(mode) {
2236
2467
  maxChars: Math.min(
2237
2468
  CORE_TRAJECTORY_ANALYSIS_MAX_CHARS,
2238
2469
  Math.max(0, Math.floor((budget - usedChars) * 0.55))
2239
- )
2470
+ ),
2471
+ ...traceRecorder ? { onLineSelected: (receipt) => {
2472
+ trajectorySelections.push(receipt);
2473
+ } } : {}
2240
2474
  })) : "";
2241
2475
  if (trajectoryAnalysisEvidence) {
2242
- sections.push(trajectoryAnalysisEvidence);
2476
+ appendSection("trajectory-analysis", "trajectory-analysis", trajectoryAnalysisEvidence);
2477
+ traceRecorder?.recordTrajectorySelections(
2478
+ "trajectory-analysis",
2479
+ trajectorySelections
2480
+ );
2243
2481
  usedChars += trajectoryAnalysisEvidence.length;
2244
2482
  }
2245
2483
  if (includeCoreRecall && !requireDirectPersonalHistoryEvidence && !requireDirectTemporalEvidence && !hasTemporalIntervalEvidence && !hasDependencyVersionEvidence && !hasUserImplementationTargetEvidence) {
@@ -2252,17 +2490,27 @@ function createAdapterFactory(mode) {
2252
2490
  )
2253
2491
  )
2254
2492
  );
2255
- const coreRecall = await waitForRecall(
2256
- state.orchestrator.recall(query, sessionId, {
2257
- budgetCharsOverride: coreBudget,
2258
- mode: "full",
2259
- ...recallAsOf ? { asOf: recallAsOf } : {}
2260
- })
2493
+ const coreOptions = {
2494
+ budgetCharsOverride: coreBudget,
2495
+ mode: "full",
2496
+ ...control?.signal ? { abortSignal: control.signal } : {},
2497
+ ...recallAsOf ? { asOf: recallAsOf } : {}
2498
+ };
2499
+ const coreRecall = traceRecorder ? await withBenchPhaseAbort(
2500
+ state.orchestrator.recallWithXrayCapture(query, sessionId, coreOptions),
2501
+ control,
2502
+ "recall",
2503
+ { waitForCompletionOnAbort: true }
2504
+ ).then((capture) => {
2505
+ traceRecorder.recordCoreCapture(capture.snapshot);
2506
+ return capture.result;
2507
+ }) : await waitForRecall(
2508
+ state.orchestrator.recall(query, sessionId, coreOptions)
2261
2509
  );
2262
2510
  if (coreRecall.trim().length > 0) {
2263
2511
  const section = `## Remnic recall pipeline
2264
2512
  ${coreRecall.trim()}`;
2265
- sections.push(section);
2513
+ appendSection("core", "core", section);
2266
2514
  usedChars += section.length;
2267
2515
  }
2268
2516
  }
@@ -2273,7 +2521,7 @@ ${coreRecall.trim()}`;
2273
2521
  "## Remnic historical recall",
2274
2522
  `No historically valid Remnic memories matched this query as of ${recallAsOf}.`
2275
2523
  ].join("\n");
2276
- sections.push(section);
2524
+ appendSection("historical-empty", "derived", section);
2277
2525
  usedChars += section.length;
2278
2526
  }
2279
2527
  }
@@ -2289,6 +2537,30 @@ ${coreRecall.trim()}`;
2289
2537
  sessionId
2290
2538
  )
2291
2539
  );
2540
+ const exactSearchHits = /* @__PURE__ */ new Map();
2541
+ const uniqueLegacySearchHits = /* @__PURE__ */ new Map();
2542
+ const ambiguousLegacySearchHitIds = /* @__PURE__ */ new Set();
2543
+ searchResults.forEach((result, rank) => {
2544
+ const identity = lcmEvidenceIdentity(result, result.session_id);
2545
+ if (identity.archiveRowId !== void 0 && !exactSearchHits.has(identity.id)) {
2546
+ exactSearchHits.set(identity.id, result);
2547
+ } else if (identity.archiveRowId === void 0) {
2548
+ if (uniqueLegacySearchHits.has(identity.id)) {
2549
+ uniqueLegacySearchHits.delete(identity.id);
2550
+ ambiguousLegacySearchHitIds.add(identity.id);
2551
+ } else if (!ambiguousLegacySearchHitIds.has(identity.id)) {
2552
+ uniqueLegacySearchHits.set(identity.id, result);
2553
+ }
2554
+ }
2555
+ traceRecorder?.recordLcmCandidate({
2556
+ rank: rank + 1,
2557
+ ...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
2558
+ turnIndex: result.turn_index,
2559
+ role: result.role,
2560
+ ...typeof result.score === "number" ? { score: result.score } : {},
2561
+ lineageStatus: identity.archiveRowId === void 0 ? "unavailable" : "exact"
2562
+ });
2563
+ });
2292
2564
  if (searchResults.length > 0) {
2293
2565
  const evidenceItems = [];
2294
2566
  const directTemporalEvidenceItems = [];
@@ -2307,8 +2579,20 @@ ${coreRecall.trim()}`;
2307
2579
  includeCoreRecall ? 1600 : 600
2308
2580
  )
2309
2581
  );
2582
+ const expandedIdentityCounts = /* @__PURE__ */ new Map();
2583
+ for (const message of expanded) {
2584
+ const expandedIdentity = lcmEvidenceIdentity(
2585
+ message,
2586
+ result.session_id
2587
+ );
2588
+ expandedIdentityCounts.set(
2589
+ expandedIdentity.id,
2590
+ (expandedIdentityCounts.get(expandedIdentity.id) ?? 0) + 1
2591
+ );
2592
+ }
2310
2593
  if (expanded.length === 0) {
2311
- const id = `${result.session_id}:${result.turn_index}`;
2594
+ const identity = lcmEvidenceIdentity(result, result.session_id);
2595
+ const { id } = identity;
2312
2596
  if (!directTemporalTurnIds.has(id) && shouldIncludeDirectTemporalEvidence(
2313
2597
  result.content,
2314
2598
  query,
@@ -2317,6 +2601,7 @@ ${coreRecall.trim()}`;
2317
2601
  directTemporalTurnIds.add(id);
2318
2602
  directTemporalEvidenceItems.push({
2319
2603
  id,
2604
+ ...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
2320
2605
  sessionId: result.session_id,
2321
2606
  turnIndex: result.turn_index,
2322
2607
  role: result.role,
@@ -2336,6 +2621,7 @@ ${coreRecall.trim()}`;
2336
2621
  seenTurns.add(id);
2337
2622
  evidenceItems.push({
2338
2623
  id,
2624
+ ...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
2339
2625
  sessionId: result.session_id,
2340
2626
  turnIndex: result.turn_index,
2341
2627
  role: result.role,
@@ -2346,7 +2632,10 @@ ${coreRecall.trim()}`;
2346
2632
  continue;
2347
2633
  }
2348
2634
  for (const message of expanded) {
2349
- const id = `${result.session_id}:${message.turn_index}`;
2635
+ const identity = lcmEvidenceIdentity(message, result.session_id);
2636
+ const { id } = identity;
2637
+ const attributableSearchHit = identity.archiveRowId === void 0 ? expandedIdentityCounts.get(identity.id) === 1 ? uniqueLegacySearchHits.get(identity.id) : void 0 : exactSearchHits.get(identity.id);
2638
+ const attributableScore = typeof attributableSearchHit?.score === "number" ? attributableSearchHit.score : void 0;
2350
2639
  if (seenTurns.has(id)) continue;
2351
2640
  if (!directTemporalTurnIds.has(id) && shouldIncludeDirectTemporalEvidence(
2352
2641
  message.content,
@@ -2356,11 +2645,12 @@ ${coreRecall.trim()}`;
2356
2645
  directTemporalTurnIds.add(id);
2357
2646
  directTemporalEvidenceItems.push({
2358
2647
  id,
2648
+ ...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
2359
2649
  sessionId: result.session_id,
2360
2650
  turnIndex: message.turn_index,
2361
2651
  role: message.role,
2362
2652
  content: message.content,
2363
- ...message.turn_index === result.turn_index && typeof result.score === "number" ? { score: result.score } : {}
2653
+ ...attributableScore === void 0 ? {} : { score: attributableScore }
2364
2654
  });
2365
2655
  }
2366
2656
  if (!shouldIncludeFocusedSearchEvidence(
@@ -2377,18 +2667,23 @@ ${coreRecall.trim()}`;
2377
2667
  seenTurns.add(id);
2378
2668
  evidenceItems.push({
2379
2669
  id,
2670
+ ...identity.archiveRowId === void 0 ? {} : { archiveRowId: identity.archiveRowId },
2380
2671
  sessionId: result.session_id,
2381
2672
  turnIndex: message.turn_index,
2382
2673
  role: message.role,
2383
2674
  content: message.content,
2384
- ...message.turn_index === result.turn_index && typeof result.score === "number" ? { score: result.score } : {}
2675
+ ...attributableScore === void 0 ? {} : { score: attributableScore }
2385
2676
  });
2386
2677
  }
2387
2678
  }
2679
+ const directTemporalSelections = [];
2388
2680
  const directTemporalEvidence = buildEvidencePack(directTemporalEvidenceItems, {
2389
2681
  title: "Direct temporal evidence",
2390
2682
  maxChars: Math.min(searchBudget, 3e3),
2391
- maxItemChars: 900
2683
+ maxItemChars: 900,
2684
+ ...traceRecorder ? { onSelection: (receipt) => {
2685
+ directTemporalSelections.push(receipt);
2686
+ } } : {}
2392
2687
  });
2393
2688
  let remainingSearchBudget = searchBudget;
2394
2689
  if (directTemporalEvidence) {
@@ -2396,7 +2691,11 @@ ${coreRecall.trim()}`;
2396
2691
  directTemporalEvidence,
2397
2692
  "These direct temporal statements match the question wording. Prefer them over indirect schedule-update context unless the question asks for the latest or current value."
2398
2693
  ].join("\n\n");
2399
- sections.push(section);
2694
+ appendSection("direct-temporal", "evidence-pack", section);
2695
+ traceRecorder?.recordEvidenceSelections(
2696
+ "direct-temporal",
2697
+ directTemporalSelections
2698
+ );
2400
2699
  usedChars += section.length;
2401
2700
  remainingSearchBudget = 0;
2402
2701
  }
@@ -2405,19 +2704,27 @@ ${coreRecall.trim()}`;
2405
2704
  evidenceItems
2406
2705
  );
2407
2706
  if (contradictionGuidance) {
2408
- sections.push(contradictionGuidance);
2707
+ appendSection("contradiction-guidance", "derived", contradictionGuidance);
2409
2708
  usedChars += contradictionGuidance.length;
2410
2709
  }
2710
+ const searchSelections = [];
2411
2711
  const searchEvidence = buildEvidencePack(
2412
2712
  directTemporalEvidence ? evidenceItems.filter((item) => !directTemporalTurnIds.has(item.id)) : evidenceItems,
2413
2713
  {
2414
2714
  title: "Search evidence",
2415
2715
  maxChars: remainingSearchBudget,
2416
- maxItemChars: 900
2716
+ maxItemChars: 900,
2717
+ ...traceRecorder ? { onSelection: (receipt) => {
2718
+ searchSelections.push(receipt);
2719
+ } } : {}
2417
2720
  }
2418
2721
  );
2419
2722
  if (searchEvidence) {
2420
- sections.push(searchEvidence);
2723
+ appendSection("search-evidence", "evidence-pack", searchEvidence);
2724
+ traceRecorder?.recordEvidenceSelections(
2725
+ "search-evidence",
2726
+ searchSelections
2727
+ );
2421
2728
  usedChars += searchEvidence.length;
2422
2729
  }
2423
2730
  }
@@ -2429,17 +2736,26 @@ ${coreRecall.trim()}`;
2429
2736
  "## Remnic recall sufficiency",
2430
2737
  "No direct evidence found for the requested personal background or previous development projects in this session."
2431
2738
  ].join("\n");
2432
- sections.push(section);
2739
+ appendSection("personal-history-empty", "derived", section);
2433
2740
  usedChars += section.length;
2434
2741
  }
2435
2742
  }
2436
2743
  if (!suppressBroadSummary) {
2437
2744
  const summaryBudget = Math.max(0, budget - usedChars - 4);
2438
- const recallText = await waitForRecall(
2745
+ const summaryCapture = traceRecorder && engine.assembleRecallWithTrace ? await waitForRecall(
2746
+ engine.assembleRecallWithTrace(sessionId, summaryBudget)
2747
+ ) : void 0;
2748
+ const recallText = summaryCapture?.text ?? await waitForRecall(
2439
2749
  engine.assembleRecall(sessionId, summaryBudget)
2440
2750
  );
2441
2751
  if (recallText) {
2442
- sections.push(recallText);
2752
+ appendSection("lcm-summary", "lcm-summary", recallText);
2753
+ if (summaryCapture) {
2754
+ traceRecorder?.recordSummarySelections(
2755
+ "lcm-summary",
2756
+ summaryCapture.selectedSummaries
2757
+ );
2758
+ }
2443
2759
  }
2444
2760
  }
2445
2761
  if (!historicalRecall && sections.length === 0) {
@@ -2455,16 +2771,50 @@ ${coreRecall.trim()}`;
2455
2771
  )
2456
2772
  );
2457
2773
  if (expanded.length > 0) {
2458
- sections.push(
2459
- `## Raw messages
2460
- ${expanded.map((message) => `[${message.role}]: ${message.content}`).join("\n")}`
2774
+ const prefix = "## Raw messages\n";
2775
+ const rows = expanded.map(
2776
+ (message) => `[${message.role}]: ${message.content}`
2461
2777
  );
2778
+ const rawSection = `${prefix}${rows.join("\n")}`;
2779
+ appendSection("raw-messages", "raw-row", rawSection);
2780
+ let rowStart = prefix.length;
2781
+ expanded.forEach((message, index) => {
2782
+ const rowEnd = rowStart + rows[index].length;
2783
+ traceRecorder?.recordRawRow(
2784
+ "raw-messages",
2785
+ { start: rowStart, end: rowEnd },
2786
+ message
2787
+ );
2788
+ rowStart = rowEnd + 1;
2789
+ });
2462
2790
  }
2463
2791
  }
2464
2792
  }
2465
2793
  const joined = sections.join("\n\n");
2466
2794
  return joined.length > budget ? joined.slice(0, budget) : joined;
2467
2795
  },
2796
+ recall(sessionId, query, budgetChars, recallOptions = {}, control) {
2797
+ return adapter[composeRecall](
2798
+ sessionId,
2799
+ query,
2800
+ budgetChars,
2801
+ recallOptions,
2802
+ control
2803
+ );
2804
+ },
2805
+ async recallWithTrace(sessionId, query, budgetChars, recallOptions = {}, control) {
2806
+ const budget = budgetChars ?? DEFAULT_BENCH_RECALL_BUDGET_CHARS;
2807
+ const traceRecorder = createBenchRecallTraceRecorder(Math.max(0, budget));
2808
+ const text = await adapter[composeRecall](
2809
+ sessionId,
2810
+ query,
2811
+ budgetChars,
2812
+ recallOptions,
2813
+ control,
2814
+ traceRecorder
2815
+ );
2816
+ return { text, trace: traceRecorder.finalize(text.length) };
2817
+ },
2468
2818
  async assessRecallSupport(request, control) {
2469
2819
  throwIfBenchPhaseAborted(control, "assessRecallSupport");
2470
2820
  return assessRemnicRecallSupport(request, answerSupportMinCoverage);
@@ -2699,6 +3049,7 @@ ${expanded.map((message) => `[${message.role}]: ${message.content}`).join("\n")}
2699
3049
  responder: options.responder,
2700
3050
  judge: options.judge
2701
3051
  };
3052
+ return adapter;
2702
3053
  };
2703
3054
  }
2704
3055
  var createLightweightAdapter = createAdapterFactory("lightweight");
@@ -3434,7 +3785,7 @@ function extractStructuredTrajectoryCueNumber(content) {
3434
3785
  function nextBenchTranscriptTurnId(counters, sessionId, message) {
3435
3786
  const index = counters.get(sessionId) ?? 0;
3436
3787
  counters.set(sessionId, index + 1);
3437
- const digest = createHash("sha256").update(`${sessionId}
3788
+ const digest = createHash2("sha256").update(`${sessionId}
3438
3789
  ${index}
3439
3790
  ${message.role}
3440
3791
  ${message.content}`).digest("hex").slice(0, 16);
@@ -4301,6 +4652,33 @@ function createTimeoutGuardedAdapter(adapter, options) {
4301
4652
  return adapter.destroy();
4302
4653
  }
4303
4654
  };
4655
+ if (adapter.recallWithTrace) {
4656
+ wrapped.recallWithTrace = (sessionId, query, budgetChars, recallOptions, control) => {
4657
+ if (phaseTimeoutMs === void 0) {
4658
+ return adapter.recallWithTrace(
4659
+ sessionId,
4660
+ query,
4661
+ budgetChars,
4662
+ recallOptions,
4663
+ control
4664
+ );
4665
+ }
4666
+ return run(`recallWithTrace session=${sessionId}`, async (signal) => {
4667
+ const merged = mergeBenchPhaseControl(signal, control);
4668
+ try {
4669
+ return await adapter.recallWithTrace(
4670
+ sessionId,
4671
+ query,
4672
+ budgetChars,
4673
+ recallOptions,
4674
+ merged.control
4675
+ );
4676
+ } finally {
4677
+ merged.cleanup();
4678
+ }
4679
+ });
4680
+ };
4681
+ }
4304
4682
  if (adapter.drain) {
4305
4683
  wrapped.drain = (control) => drainTimeoutMs === void 0 ? adapter.drain(control) : run(
4306
4684
  "drain",
@@ -5101,7 +5479,7 @@ function listMemoryEvalBenchmarkIds() {
5101
5479
  import {
5102
5480
  createCipheriv,
5103
5481
  createDecipheriv,
5104
- createHash as createHash2,
5482
+ createHash as createHash3,
5105
5483
  randomBytes,
5106
5484
  timingSafeEqual
5107
5485
  } from "crypto";
@@ -5112,10 +5490,10 @@ var AES_TAG_LENGTH = 16;
5112
5490
  var INTEGRITY_HASH_ALGORITHM = "sha256";
5113
5491
  var INTEGRITY_CIPHER_ALGORITHM = "aes-256-gcm";
5114
5492
  function hashString(value) {
5115
- return createHash2(INTEGRITY_HASH_ALGORITHM).update(value, "utf8").digest("hex");
5493
+ return createHash3(INTEGRITY_HASH_ALGORITHM).update(value, "utf8").digest("hex");
5116
5494
  }
5117
5495
  function hashBytes(value) {
5118
- return createHash2(INTEGRITY_HASH_ALGORITHM).update(value).digest("hex");
5496
+ return createHash3(INTEGRITY_HASH_ALGORITHM).update(value).digest("hex");
5119
5497
  }
5120
5498
  function canonicalJsonStringify(value, space) {
5121
5499
  return JSON.stringify(value, canonicalReplacer, space);
@@ -5497,14 +5875,14 @@ var BENCHMARK_RESULT_SCHEMA = {
5497
5875
 
5498
5876
  // src/repro-manifest.ts
5499
5877
  import { execFileSync } from "child_process";
5500
- import { createHash as createHash4 } from "crypto";
5878
+ import { createHash as createHash5 } from "crypto";
5501
5879
  import { createReadStream } from "fs";
5502
5880
  import { lstat as lstat3, mkdir as mkdir4, readFile as readFile5, readdir as readdir4, readlink, realpath as realpath3, stat as stat2, writeFile as writeFile4 } from "fs/promises";
5503
5881
  import os3 from "os";
5504
5882
  import path5 from "path";
5505
5883
 
5506
5884
  // src/providers/codex-credit-budget.ts
5507
- import { createHash as createHash3 } from "crypto";
5885
+ import { createHash as createHash4 } from "crypto";
5508
5886
  import { mkdir as mkdir2, open, readFile as readFile3, rename as rename2, rmdir, unlink as unlink2, writeFile as writeFile2 } from "fs/promises";
5509
5887
  import os from "os";
5510
5888
  import path3 from "path";
@@ -6123,7 +6501,7 @@ function isSha256(value) {
6123
6501
  return typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
6124
6502
  }
6125
6503
  function sha256(value) {
6126
- return createHash3("sha256").update(value).digest("hex");
6504
+ return createHash4("sha256").update(value).digest("hex");
6127
6505
  }
6128
6506
  function normalizeZero(value) {
6129
6507
  return Object.is(value, -0) ? 0 : value;
@@ -7270,13 +7648,13 @@ var CODEX_CREDIT_REPRO_ENV_KEYS = [
7270
7648
  "REMNIC_BENCH_RUN_ID"
7271
7649
  ];
7272
7650
  function sha256String(value) {
7273
- return createHash4("sha256").update(value).digest("hex");
7651
+ return createHash5("sha256").update(value).digest("hex");
7274
7652
  }
7275
7653
  function sha256Buffer(value) {
7276
- return createHash4("sha256").update(value).digest("hex");
7654
+ return createHash5("sha256").update(value).digest("hex");
7277
7655
  }
7278
7656
  async function sha256File(filePath) {
7279
- const hash = createHash4("sha256");
7657
+ const hash = createHash5("sha256");
7280
7658
  await new Promise((resolve, reject) => {
7281
7659
  const stream = createReadStream(filePath);
7282
7660
  stream.on("data", (chunk) => hash.update(chunk));
@@ -8197,7 +8575,7 @@ async function writeBenchmarkReproManifest(resultsDir, options = {}) {
8197
8575
  }
8198
8576
 
8199
8577
  // src/published-artifact.ts
8200
- import { createHash as createHash5 } from "crypto";
8578
+ import { createHash as createHash6 } from "crypto";
8201
8579
  import { mkdir as mkdir5, readFile as readFile6, writeFile as writeFile5 } from "fs/promises";
8202
8580
  import path6 from "path";
8203
8581
  var BENCHMARK_ARTIFACT_SCHEMA_VERSION = 1;
@@ -8343,7 +8721,7 @@ function serializeBenchmarkArtifact(artifact) {
8343
8721
  `;
8344
8722
  }
8345
8723
  function hashBenchmarkArtifact(artifact) {
8346
- return createHash5("sha256").update(serializeBenchmarkArtifact(artifact)).digest("hex");
8724
+ return createHash6("sha256").update(serializeBenchmarkArtifact(artifact)).digest("hex");
8347
8725
  }
8348
8726
  async function writeBenchmarkArtifact(artifact, outputDir) {
8349
8727
  await mkdir5(outputDir, { recursive: true });
@@ -8361,7 +8739,7 @@ async function writeBenchmarkArtifact(artifact, outputDir) {
8361
8739
  return {
8362
8740
  path: abs,
8363
8741
  filename,
8364
- sha256: createHash5("sha256").update(body).digest("hex"),
8742
+ sha256: createHash6("sha256").update(body).digest("hex"),
8365
8743
  bytes: Buffer.byteLength(body, "utf8")
8366
8744
  };
8367
8745
  }
@@ -8475,7 +8853,7 @@ async function loadBenchmarkArtifact(filePath) {
8475
8853
  const artifact = parseBenchmarkArtifact(raw);
8476
8854
  return {
8477
8855
  artifact,
8478
- sha256: createHash5("sha256").update(raw).digest("hex"),
8856
+ sha256: createHash6("sha256").update(raw).digest("hex"),
8479
8857
  bytes: Buffer.byteLength(raw, "utf8")
8480
8858
  };
8481
8859
  }
@@ -9638,7 +10016,7 @@ function createClaudeCliProvider(config, deps) {
9638
10016
 
9639
10017
  // src/providers/codex-cli.ts
9640
10018
  import { spawn as spawn2 } from "child_process";
9641
- import { createHash as createHash6, randomUUID } from "crypto";
10019
+ import { createHash as createHash7, randomUUID } from "crypto";
9642
10020
  import { mkdir as mkdir6, mkdtemp as mkdtemp3, readFile as readFile7, rm as rm3, writeFile as writeFile6 } from "fs/promises";
9643
10021
  import os5 from "os";
9644
10022
  import path8 from "path";
@@ -10506,7 +10884,7 @@ function resolveCodexCliDiagnosticsMode(config) {
10506
10884
  }
10507
10885
  function inspectCodexCompletionPrompt(prompt) {
10508
10886
  const stats = {
10509
- sha256: createHash6("sha256").update(prompt).digest("hex"),
10887
+ sha256: createHash7("sha256").update(prompt).digest("hex"),
10510
10888
  chars: prompt.length,
10511
10889
  lines: prompt.length === 0 ? 0 : prompt.split("\n").length
10512
10890
  };
@@ -12642,7 +13020,7 @@ function asStringArray(value) {
12642
13020
  }
12643
13021
 
12644
13022
  // src/responders.ts
12645
- import { createHash as createHash7 } from "crypto";
13023
+ import { createHash as createHash8 } from "crypto";
12646
13024
  import { FallbackLlmClient } from "@remnic/core";
12647
13025
 
12648
13026
  // src/providers/openai-responses.ts
@@ -13115,7 +13493,7 @@ function getProviderBackedJudgePromptIdentity(config) {
13115
13493
  temperature: 0,
13116
13494
  maxTokens: 16
13117
13495
  };
13118
- return `sha256:${createHash7("sha256").update(JSON.stringify(contract)).digest("hex")}`;
13496
+ return `sha256:${createHash8("sha256").update(JSON.stringify(contract)).digest("hex")}`;
13119
13497
  }
13120
13498
  var AMA_BENCH_RECOMMENDED_JUDGE_SYSTEM_PROMPT = [
13121
13499
  "You are evaluating an AMA-Bench long-horizon memory question.",
@@ -15456,11 +15834,11 @@ async function resolveLocalLabRuntimeProfile(options) {
15456
15834
  // src/benchmark.ts
15457
15835
  import fs2 from "fs";
15458
15836
  import path35 from "path";
15459
- import { createHash as createHash13 } from "crypto";
15837
+ import { createHash as createHash14 } from "crypto";
15460
15838
  import { expandTildePath as expandTildePath3 } from "@remnic/core";
15461
15839
 
15462
15840
  // src/judges/judge-cache.ts
15463
- import { createHash as createHash8, randomBytes as randomBytes2 } from "crypto";
15841
+ import { createHash as createHash9, randomBytes as randomBytes2 } from "crypto";
15464
15842
  import {
15465
15843
  mkdir as mkdir9,
15466
15844
  readFile as readFile11,
@@ -15503,8 +15881,8 @@ var JudgeCache = class {
15503
15881
  }
15504
15882
  /** Compute the sha256-hex key for a set of parts. Pure, sync, side-effect-free. */
15505
15883
  computeKey(parts) {
15506
- const fieldDigest = (value) => createHash8("sha256").update(value).digest();
15507
- return createHash8("sha256").update(fieldDigest(parts.benchmarkId)).update(fieldDigest(parts.datasetVersion)).update(fieldDigest(parts.questionId)).update(fieldDigest(parts.answerText)).update(fieldDigest(parts.judgePromptHash)).update(fieldDigest(parts.judgeModelId)).update(fieldDigest(parts.judgeParamsHash)).digest("hex");
15884
+ const fieldDigest = (value) => createHash9("sha256").update(value).digest();
15885
+ return createHash9("sha256").update(fieldDigest(parts.benchmarkId)).update(fieldDigest(parts.datasetVersion)).update(fieldDigest(parts.questionId)).update(fieldDigest(parts.answerText)).update(fieldDigest(parts.judgePromptHash)).update(fieldDigest(parts.judgeModelId)).update(fieldDigest(parts.judgeParamsHash)).digest("hex");
15508
15886
  }
15509
15887
  /**
15510
15888
  * Read a previously-stored verdict. Returns `undefined` on miss, corrupted
@@ -15699,7 +16077,7 @@ function runJudgeWithCache(options) {
15699
16077
  // Binary prompts are content-sensitive: two distinct prompts of
15700
16078
  // the same character length would collide on the previous
15701
16079
  // `binary:N` key, so key on a sha256 prefix of the prompt body.
15702
- questionId: `binary:${createHash8("sha256").update(prompt).digest("hex").slice(0, 16)}`,
16080
+ questionId: `binary:${createHash9("sha256").update(prompt).digest("hex").slice(0, 16)}`,
15703
16081
  answerText: prompt,
15704
16082
  judgePromptHash: keyExtras.judgePromptHash ?? "unknown-prompt",
15705
16083
  judgeModelId: keyExtras.judgeModelId ?? "unknown-judge",
@@ -19469,26 +19847,6 @@ import { collectTemporalLexicalCues } from "@remnic/core";
19469
19847
  import { readFile as readFile15 } from "fs/promises";
19470
19848
  import path17 from "path";
19471
19849
 
19472
- // src/benchmarks/published/longmemeval/fixture.ts
19473
- var LONG_MEM_EVAL_SMOKE_FIXTURE = [
19474
- {
19475
- question_id: 1,
19476
- question_type: "single-session-user",
19477
- question: "What city does the user live in?",
19478
- answer: "Paris",
19479
- question_date: "2025-01-01",
19480
- haystack_dates: ["2024-12-01"],
19481
- haystack_session_ids: ["session-1"],
19482
- haystack_sessions: [
19483
- [
19484
- { role: "user", content: "I moved to Paris last year." },
19485
- { role: "assistant", content: "Paris sounds great." }
19486
- ]
19487
- ],
19488
- answer_session_ids: ["session-1"]
19489
- }
19490
- ];
19491
-
19492
19850
  // src/benchmarks/published/locomo/fixture.ts
19493
19851
  var LOCOMO_SMOKE_FIXTURE = [
19494
19852
  {
@@ -19538,6 +19896,26 @@ var LOCOMO_SMOKE_FIXTURE = [
19538
19896
  }
19539
19897
  ];
19540
19898
 
19899
+ // src/benchmarks/published/longmemeval/fixture.ts
19900
+ var LONG_MEM_EVAL_SMOKE_FIXTURE = [
19901
+ {
19902
+ question_id: 1,
19903
+ question_type: "single-session-user",
19904
+ question: "What city does the user live in?",
19905
+ answer: "Paris",
19906
+ question_date: "2025-01-01",
19907
+ haystack_dates: ["2024-12-01"],
19908
+ haystack_session_ids: ["session-1"],
19909
+ haystack_sessions: [
19910
+ [
19911
+ { role: "user", content: "I moved to Paris last year." },
19912
+ { role: "assistant", content: "Paris sounds great." }
19913
+ ]
19914
+ ],
19915
+ answer_session_ids: ["session-1"]
19916
+ }
19917
+ ];
19918
+
19541
19919
  // src/benchmarks/published/dataset-loader.ts
19542
19920
  var LONG_MEM_EVAL_DATASET_FILENAMES = Object.freeze([
19543
19921
  "longmemeval_oracle.json",
@@ -19585,6 +19963,7 @@ async function loadDataset4(options) {
19585
19963
  return {
19586
19964
  source: "dataset",
19587
19965
  filename,
19966
+ sha256: hashString(raw),
19588
19967
  items: applyLimit4(parsed, limit),
19589
19968
  errors
19590
19969
  };
@@ -19600,6 +19979,7 @@ async function loadDataset4(options) {
19600
19979
  }
19601
19980
  return {
19602
19981
  source: "smoke",
19982
+ sha256: hashCanonicalJson(options.smokeFixture),
19603
19983
  items: applyLimit4([...options.smokeFixture], limit),
19604
19984
  errors
19605
19985
  };
@@ -20885,11 +21265,12 @@ var locomoDefinition = {
20885
21265
  }
20886
21266
  };
20887
21267
  async function runLoCoMoBenchmark(options) {
20888
- const conversations = await loadDataset6(
21268
+ const loaded = await loadLoCoMoDataset(
20889
21269
  options.mode,
20890
21270
  options.datasetDir,
20891
21271
  options.limit
20892
21272
  );
21273
+ const conversations = loaded.items;
20893
21274
  const trialLimit = resolveTrialLimit(options.benchmarkOptions?.trialLimit);
20894
21275
  const multiHopRecallComposition = resolveLoCoMoBooleanOption(
20895
21276
  options.benchmarkOptions?.multiHopRecallComposition,
@@ -20898,7 +21279,7 @@ async function runLoCoMoBenchmark(options) {
20898
21279
  );
20899
21280
  const plans = applyTrialLimit(
20900
21281
  conversations.map(
20901
- (conversation) => buildPlan2(conversation, multiHopRecallComposition)
21282
+ (conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition)
20902
21283
  ),
20903
21284
  trialLimit
20904
21285
  );
@@ -20969,7 +21350,7 @@ function applyTrialLimit(plans, trialLimit) {
20969
21350
  }
20970
21351
  return limitedPlans;
20971
21352
  }
20972
- function buildPlan2(conversation, multiHopRecallComposition) {
21353
+ function buildLoCoMoPlan(conversation, multiHopRecallComposition) {
20973
21354
  const sessions = extractSessions(conversation.conversation);
20974
21355
  const speakerA = typeof conversation.conversation.speaker_a === "string" ? conversation.conversation.speaker_a : "Speaker A";
20975
21356
  const ingestSessions = [];
@@ -21005,9 +21386,9 @@ function buildTrial(conversationId, qa, questionIndex, sessionIds, multiHopRecal
21005
21386
  expected: qa.answer,
21006
21387
  recallSessionIds: sessionIds,
21007
21388
  answerFormat: "short-with-specifics",
21008
- recallTextTransform: ({ question, recalledText }) => prioritizeLoCoMoRecallText({
21389
+ recallTextTransform: ({ question, recalledText }) => transformLoCoMoRecallText({
21009
21390
  question,
21010
- recalledText: sanitizeLoCoMoRecallText({ question, recalledText }),
21391
+ recalledText,
21011
21392
  multiHopRecallComposition
21012
21393
  }),
21013
21394
  answerFallback: ({ question, recalledText }) => answerLoCoMoFromRecall(question, recalledText),
@@ -21208,10 +21589,20 @@ function sanitizeLoCoMoRecallText(args) {
21208
21589
  (id) => queryVisibleIds.has(id) ? id : ""
21209
21590
  );
21210
21591
  }
21211
- function prioritizeLoCoMoRecallText(args) {
21212
- const lines = dedupePreserveOrder(
21213
- args.recalledText.replaceAll("\r\n", "\n").replaceAll("\r", "\n").split("\n").map((line) => line.trim()).filter((line) => line.length > 0)
21214
- );
21592
+ function transformLoCoMoRecallText(args) {
21593
+ const sanitized = sanitizeLoCoMoRecallText(args);
21594
+ return prioritizeLoCoMoRecallTextWithTrace({
21595
+ ...args,
21596
+ recalledText: sanitized
21597
+ }).text;
21598
+ }
21599
+ function prioritizeLoCoMoRecallTextWithTrace(args) {
21600
+ const inputLines = args.recalledText.replaceAll("\r\n", "\n").replaceAll("\r", "\n").split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
21601
+ const lines = dedupePreserveOrder(inputLines);
21602
+ const inputOrdinalByLine = /* @__PURE__ */ new Map();
21603
+ inputLines.forEach((line, index) => {
21604
+ if (!inputOrdinalByLine.has(line)) inputOrdinalByLine.set(line, index);
21605
+ });
21215
21606
  const questionTokens = expandLoCoMoQuestionTokens(
21216
21607
  tokenizeForLoCoMo(args.question)
21217
21608
  );
@@ -21244,25 +21635,90 @@ function prioritizeLoCoMoRecallText(args) {
21244
21635
  )
21245
21636
  ];
21246
21637
  if (direct.length === 0 && linkedHops.length === 0) {
21247
- return truncateLoCoMoContext(
21638
+ const { text: text2 } = truncateLoCoMoContext(
21248
21639
  args.recalledText,
21249
21640
  LOCOMO_FALLBACK_CONTEXT_MAX_CHARS
21250
21641
  );
21642
+ return {
21643
+ text: text2,
21644
+ receipt: {
21645
+ schemaVersion: 1,
21646
+ mode: "fallback",
21647
+ multiHopRecallComposition: args.multiHopRecallComposition,
21648
+ input: digestLoCoMoContent(args.recalledText),
21649
+ output: digestLoCoMoContent(text2),
21650
+ selectedLines: []
21651
+ }
21652
+ };
21251
21653
  }
21252
- const sections = [
21253
- "## LoCoMo Question-Focused Evidence",
21254
- ...direct.map((entry) => truncateLoCoMoLine(entry.line))
21255
- ];
21654
+ const sections = ["## LoCoMo Question-Focused Evidence"];
21655
+ const selectedRanges = [];
21656
+ const appendSelectedLine = (input, stage, hop) => {
21657
+ const output = truncateLoCoMoLine(input);
21658
+ const inputOrdinal = inputOrdinalByLine.get(input);
21659
+ if (inputOrdinal === void 0) {
21660
+ throw new Error("LoCoMo composition selected a line outside its normalized input.");
21661
+ }
21662
+ const outputStart = sections.join("\n").length + 1;
21663
+ sections.push(output);
21664
+ selectedRanges.push({
21665
+ input,
21666
+ output,
21667
+ inputOrdinal,
21668
+ stage,
21669
+ ...hop === void 0 ? {} : { hop },
21670
+ outputStart,
21671
+ outputEnd: outputStart + output.length
21672
+ });
21673
+ };
21674
+ for (const entry of direct) appendSelectedLine(entry.line, "direct");
21256
21675
  for (const hop of linkedHops) {
21257
- sections.push(
21258
- `## LoCoMo Linked Evidence (hop ${hop.hop})`,
21259
- ...hop.lines.map(truncateLoCoMoLine)
21260
- );
21676
+ sections.push(`## LoCoMo Linked Evidence (hop ${hop.hop})`);
21677
+ for (const line of hop.lines) appendSelectedLine(line, "linked", hop.hop);
21261
21678
  }
21262
- return truncateLoCoMoContext(
21679
+ const truncation = truncateLoCoMoContext(
21263
21680
  sections.join("\n"),
21264
21681
  LOCOMO_FOCUSED_CONTEXT_MAX_CHARS
21265
21682
  );
21683
+ const { text, safePrefixEnd } = truncation;
21684
+ const selectedLines = selectedRanges.map((entry) => {
21685
+ const visibleStart = Math.min(entry.outputStart, safePrefixEnd);
21686
+ const visibleEnd = Math.min(entry.outputEnd, safePrefixEnd);
21687
+ const visible = visibleEnd - visibleStart === entry.output.length;
21688
+ return buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd);
21689
+ });
21690
+ return {
21691
+ text,
21692
+ receipt: {
21693
+ schemaVersion: 1,
21694
+ mode: "focused",
21695
+ multiHopRecallComposition: args.multiHopRecallComposition,
21696
+ input: digestLoCoMoContent(args.recalledText),
21697
+ output: digestLoCoMoContent(text),
21698
+ selectedLines
21699
+ }
21700
+ };
21701
+ }
21702
+ function buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd) {
21703
+ return {
21704
+ inputOrdinal: entry.inputOrdinal,
21705
+ input: digestLoCoMoContent(entry.input),
21706
+ output: digestLoCoMoContent(entry.output),
21707
+ stage: entry.stage,
21708
+ ...entry.hop === void 0 ? {} : { hop: entry.hop },
21709
+ visible,
21710
+ outputStart: entry.outputStart,
21711
+ outputEnd: entry.outputEnd,
21712
+ visibleStart,
21713
+ visibleEnd
21714
+ };
21715
+ }
21716
+ function digestLoCoMoContent(value) {
21717
+ return {
21718
+ sha256: hashString(value),
21719
+ charCount: value.length,
21720
+ lineCount: value.length === 0 ? 0 : value.split("\n").length
21721
+ };
21266
21722
  }
21267
21723
  function composeLoCoMoLinkedEvidence(args) {
21268
21724
  if (args.direct.length === 0 || args.remainingLineBudget <= 0) {
@@ -21520,13 +21976,16 @@ function truncateLoCoMoLine(line) {
21520
21976
  }
21521
21977
  function truncateLoCoMoContext(text, maxChars) {
21522
21978
  if (text.length <= maxChars) {
21523
- return text;
21979
+ return { text, safePrefixEnd: text.length };
21524
21980
  }
21525
21981
  const truncated = text.slice(0, maxChars);
21526
21982
  const lastNewline = truncated.lastIndexOf("\n");
21527
21983
  const safePrefix = lastNewline > 0 ? truncated.slice(0, lastNewline) : truncated;
21528
- return `${safePrefix}
21529
- [LoCoMo context truncated to ${maxChars} characters]`;
21984
+ return {
21985
+ text: `${safePrefix}
21986
+ [LoCoMo context truncated to ${maxChars} characters]`,
21987
+ safePrefixEnd: safePrefix.length
21988
+ };
21530
21989
  }
21531
21990
  function countHiddenEvidenceIdsInRecall(evidence, question, recalledText) {
21532
21991
  const queryVisibleIds = collectDialogueIds(question);
@@ -21547,7 +22006,7 @@ function collectDialogueIds(text) {
21547
22006
  function escapeRegExp2(value) {
21548
22007
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
21549
22008
  }
21550
- async function loadDataset6(mode, datasetDir, limit) {
22009
+ async function loadLoCoMoDataset(mode, datasetDir, limit) {
21551
22010
  const loaded = await loadLoCoMo10({
21552
22011
  mode,
21553
22012
  datasetDir,
@@ -21579,7 +22038,15 @@ async function loadDataset6(mode, datasetDir, limit) {
21579
22038
  "[remnic-bench] LoCoMo falling back to smoke fixture: " + loaded.errors.join(" | ")
21580
22039
  );
21581
22040
  }
21582
- return loaded.items;
22041
+ if (!loaded.sha256) {
22042
+ throw new Error("LoCoMo dataset loader did not provide a content hash.");
22043
+ }
22044
+ return {
22045
+ source: loaded.source,
22046
+ ...loaded.filename === void 0 ? {} : { filename: loaded.filename },
22047
+ sha256: loaded.sha256,
22048
+ items: loaded.items
22049
+ };
21583
22050
  }
21584
22051
  function parseDataset2(raw, filename) {
21585
22052
  const parsed = JSON.parse(raw);
@@ -21863,7 +22330,7 @@ async function loadBeamDatasetPreview(options) {
21863
22330
  }
21864
22331
  let dataset;
21865
22332
  try {
21866
- dataset = await loadDataset7(
22333
+ dataset = await loadDataset6(
21867
22334
  options.mode === "quick" ? "quick" : "full",
21868
22335
  options.datasetDir,
21869
22336
  options.limit
@@ -21902,7 +22369,7 @@ async function loadBeamDatasetPreview(options) {
21902
22369
  };
21903
22370
  }
21904
22371
  async function runBeamBenchmark(options) {
21905
- const dataset = await loadDataset7(options.mode, options.datasetDir, options.limit);
22372
+ const dataset = await loadDataset6(options.mode, options.datasetDir, options.limit);
21906
22373
  const tasks = [];
21907
22374
  const taskFilter = normalizeBeamTaskFilter(
21908
22375
  options.benchmarkOptions?.taskFilter
@@ -22089,7 +22556,7 @@ async function runBeamBenchmark(options) {
22089
22556
  }
22090
22557
  };
22091
22558
  }
22092
- async function loadDataset7(mode, datasetDir, limit) {
22559
+ async function loadDataset6(mode, datasetDir, limit) {
22093
22560
  const normalizedLimit = normalizeLimit5(limit);
22094
22561
  const ensureDatasetEntries = (entryCount) => {
22095
22562
  if (entryCount === 0) {
@@ -23190,7 +23657,7 @@ var StructuredLiteralParser = class {
23190
23657
  };
23191
23658
 
23192
23659
  // src/benchmarks/published/personamem/runner.ts
23193
- import { createHash as createHash9, randomUUID as randomUUID7 } from "crypto";
23660
+ import { createHash as createHash10, randomUUID as randomUUID7 } from "crypto";
23194
23661
  import { readFile as readFile16, realpath as realpath4 } from "fs/promises";
23195
23662
  import path19 from "path";
23196
23663
 
@@ -23275,7 +23742,7 @@ var personaMemDefinition = {
23275
23742
  }
23276
23743
  };
23277
23744
  async function runPersonaMemBenchmark(options) {
23278
- const samples = await loadDataset8(options.mode, options.datasetDir, options.limit);
23745
+ const samples = await loadDataset7(options.mode, options.datasetDir, options.limit);
23279
23746
  const tasks = [];
23280
23747
  const totalTasks = samples.length;
23281
23748
  for (let sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
@@ -23455,7 +23922,7 @@ async function runPersonaMemBenchmark(options) {
23455
23922
  }
23456
23923
  };
23457
23924
  }
23458
- async function loadDataset8(mode, datasetDir, limit) {
23925
+ async function loadDataset7(mode, datasetDir, limit) {
23459
23926
  const normalizedLimit = normalizeLimit6(limit);
23460
23927
  const ensureDatasetSamples = (samples) => {
23461
23928
  if (samples.length === 0) {
@@ -23795,7 +24262,7 @@ function buildMcqPrompt(sample, seed) {
23795
24262
  function deterministicShuffle(values, seedMaterial) {
23796
24263
  return values.map((value, index) => ({
23797
24264
  value,
23798
- key: createHash9("sha256").update(`${seedMaterial}:${index}:${value}`).digest("hex"),
24265
+ key: createHash10("sha256").update(`${seedMaterial}:${index}:${value}`).digest("hex"),
23799
24266
  index
23800
24267
  })).sort((left, right) => {
23801
24268
  const byKey = left.key.localeCompare(right.key);
@@ -24063,7 +24530,7 @@ var memBenchDefinition = {
24063
24530
  }
24064
24531
  };
24065
24532
  async function runMemBenchBenchmark(options) {
24066
- const dataset = await loadDataset9(options.mode, options.datasetDir, options.limit);
24533
+ const dataset = await loadDataset8(options.mode, options.datasetDir, options.limit);
24067
24534
  const tasks = [];
24068
24535
  const totalTasks = dataset.length;
24069
24536
  for (const testCase of dataset) {
@@ -24237,7 +24704,7 @@ async function runMemBenchBenchmark(options) {
24237
24704
  }
24238
24705
  };
24239
24706
  }
24240
- async function loadDataset9(mode, datasetDir, limit) {
24707
+ async function loadDataset8(mode, datasetDir, limit) {
24241
24708
  const normalizedLimit = normalizeLimit7(limit);
24242
24709
  const ensureDatasetCases = (cases) => {
24243
24710
  if (cases.length === 0) {
@@ -25268,7 +25735,7 @@ var memoryAgentBenchDefinition = {
25268
25735
  }
25269
25736
  };
25270
25737
  async function runMemoryAgentBenchBenchmark(options) {
25271
- const rawDataset = await loadDataset10(options.mode, options.datasetDir, options.limit);
25738
+ const rawDataset = await loadDataset9(options.mode, options.datasetDir, options.limit);
25272
25739
  const trialLimit = resolveTrialLimit2(options.benchmarkOptions?.trialLimit);
25273
25740
  const benchmarkOptions = trialLimit === void 0 ? options.benchmarkOptions : { ...options.benchmarkOptions ?? {}, trialLimit };
25274
25741
  const dataset = applyTrialLimit2(rawDataset, trialLimit);
@@ -26241,7 +26708,7 @@ function decodeUrlComponentSafely(value) {
26241
26708
  return value;
26242
26709
  }
26243
26710
  }
26244
- async function loadDataset10(mode, datasetDir, limit) {
26711
+ async function loadDataset9(mode, datasetDir, limit) {
26245
26712
  const normalizedLimit = normalizeLimit8(limit);
26246
26713
  const ensureDatasetItems = (items) => {
26247
26714
  if (items.length === 0) {
@@ -32254,7 +32721,7 @@ function pairedDeltaConfidenceInterval(candidateValues, baselineValues, options
32254
32721
  }
32255
32722
 
32256
32723
  // src/judges/sealed-rubric.ts
32257
- import { createHash as createHash10 } from "crypto";
32724
+ import { createHash as createHash11 } from "crypto";
32258
32725
  import { appendFileSync, mkdirSync } from "fs";
32259
32726
  import path31 from "path";
32260
32727
 
@@ -32363,7 +32830,7 @@ function loadSealedRubric(id = DEFAULT_ASSISTANT_RUBRIC_ID, options = {}) {
32363
32830
  if (typeof prompt !== "string" || prompt.length === 0) {
32364
32831
  throw new Error(`sealed rubric not found in registry: ${id}`);
32365
32832
  }
32366
- const sha2563 = createHash10("sha256").update(prompt, "utf8").digest("hex");
32833
+ const sha2563 = createHash11("sha256").update(prompt, "utf8").digest("hex");
32367
32834
  const version = parseVersionFromId(id);
32368
32835
  return { id, version, prompt, sha256: sha2563 };
32369
32836
  }
@@ -34566,7 +35033,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
34566
35033
  import { randomUUID as randomUUID31 } from "crypto";
34567
35034
 
34568
35035
  // src/benchmarks/remnic/memcorrect/generator.ts
34569
- import { createHash as createHash11 } from "crypto";
35036
+ import { createHash as createHash12 } from "crypto";
34570
35037
 
34571
35038
  // src/benchmarks/remnic/memcorrect/token-pools.ts
34572
35039
  var PERSONAS = [
@@ -34890,7 +35357,7 @@ function corpusHash(corpus) {
34890
35357
  uptakeLatencyCap: corpus.options.uptakeLatencyCap,
34891
35358
  scenarios: corpus.scenarios
34892
35359
  });
34893
- return createHash11("sha256").update(canonical).digest("hex");
35360
+ return createHash12("sha256").update(canonical).digest("hex");
34894
35361
  }
34895
35362
 
34896
35363
  // src/benchmarks/remnic/memcorrect/schema.ts
@@ -35845,7 +36312,7 @@ import { mkdir as mkdir17, writeFile as writeFile16 } from "fs/promises";
35845
36312
  import path34 from "path";
35846
36313
 
35847
36314
  // src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
35848
- import { createHash as createHash12 } from "crypto";
36315
+ import { createHash as createHash13 } from "crypto";
35849
36316
  var SCOPE_ACME = "project:acme";
35850
36317
  var SCOPE_BETA = "project:beta";
35851
36318
  var SCOPE_ALICE = "user:alice";
@@ -36328,7 +36795,7 @@ var BOUNDED_MEMORY_SMOKE_FIXTURE = [
36328
36795
  function fixtureHash(tasks) {
36329
36796
  const source = tasks ?? BOUNDED_MEMORY_FIXTURE;
36330
36797
  const payload = JSON.stringify(source);
36331
- return createHash12("sha256").update(payload, "utf8").digest("hex");
36798
+ return createHash13("sha256").update(payload, "utf8").digest("hex");
36332
36799
  }
36333
36800
 
36334
36801
  // src/benchmarks/remnic/bounded-memory-contracts/agent.ts
@@ -37588,13 +38055,13 @@ function wrapJudgeWithCache(args) {
37588
38055
  // differentiator is part of the prompt hash. Bumping
37589
38056
  // JUDGE_CACHE_PROTOCOL_VERSION invalidates verdicts when judge
37590
38057
  // prompt/parse semantics change (PR #1591, High).
37591
- judgePromptHash: createHash13("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("").update(args.amaBenchJudgeProtocol).update("").update(args.role).digest("hex"),
38058
+ judgePromptHash: createHash14("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("").update(args.amaBenchJudgeProtocol).update("").update(args.role).digest("hex"),
37592
38059
  judgeModelId: args.provider?.model !== void 0 && args.provider.model.length > 0 ? `${args.provider.model}${crossJudgeIdSuffix}` : `unknown-${args.role}-judge`,
37593
38060
  // Full judge configuration, deterministically serialized (sorted
37594
38061
  // keys) so provider/base-url/retry changes produce fresh cache
37595
38062
  // keys. `role` is included so primary and cross judges never
37596
38063
  // share a paramsHash.
37597
- judgeParamsHash: createHash13("sha256").update(
38064
+ judgeParamsHash: createHash14("sha256").update(
37598
38065
  stableStringify2({
37599
38066
  role: args.role,
37600
38067
  provider: args.provider
@@ -38300,7 +38767,7 @@ function formatSignedScore(value) {
38300
38767
  }
38301
38768
 
38302
38769
  // src/stats/locomo-recall-delta.ts
38303
- import { createHash as createHash14 } from "crypto";
38770
+ import { createHash as createHash15 } from "crypto";
38304
38771
  import { basename } from "path";
38305
38772
  var LOCOMO_FULL_TASK_COUNT = 1986;
38306
38773
  var LOCOMO_RECALL_EXCERPT_CHARS = 240;
@@ -38781,7 +39248,7 @@ function normalizeText3(value) {
38781
39248
  return value.replaceAll("\r\n", "\n").replaceAll("\r", "\n");
38782
39249
  }
38783
39250
  function sha2562(value) {
38784
- return createHash14("sha256").update(value).digest("hex");
39251
+ return createHash15("sha256").update(value).digest("hex");
38785
39252
  }
38786
39253
  function stableJson(value) {
38787
39254
  return JSON.stringify(value);
@@ -38816,6 +39283,349 @@ function formatSignedScore2(value) {
38816
39283
  return `${value >= 0 ? "+" : ""}${formatScore2(value)}`;
38817
39284
  }
38818
39285
 
39286
+ // src/benchmarks/published/locomo/retrieval-trace-runner.ts
39287
+ var LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION = 1;
39288
+ var LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION = 1;
39289
+ var LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION = 1;
39290
+ async function preflightLoCoMoRetrievalTraceCapture(options) {
39291
+ assertCaptureOptions(options);
39292
+ assertProviderFreeRetrievalConfig(options.retrievalConfig);
39293
+ const loaded = await loadLoCoMoDataset("full", options.datasetDir);
39294
+ const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
39295
+ const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
39296
+ const selectable = plans.flatMap(
39297
+ (plan, planIndex) => plan.trials.map((trial) => ({ taskId: trial.taskId, planIndex }))
39298
+ );
39299
+ selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
39300
+ }
39301
+ function buildProviderFreeLoCoMoRetrievalConfig(retrievalConfig) {
39302
+ const sanitized = sanitizeProviderFreeRetrievalConfig(retrievalConfig);
39303
+ return assertProviderFreeRetrievalConfig({
39304
+ ...sanitized,
39305
+ localLlmEnabled: false,
39306
+ localLlmFastEnabled: false,
39307
+ recallPlannerEnabled: false,
39308
+ embeddingFallbackEnabled: false,
39309
+ hostEmbeddingProviderEnabled: false,
39310
+ openaiApiKey: false,
39311
+ modelSource: "plugin"
39312
+ });
39313
+ }
39314
+ async function captureLoCoMoRetrievalTrace(options) {
39315
+ assertCaptureOptions(options);
39316
+ const retrievalConfig = assertProviderFreeRetrievalConfig(options.retrievalConfig);
39317
+ const recallWithTrace = options.system.recallWithTrace?.bind(options.system);
39318
+ if (!recallWithTrace) {
39319
+ throw new Error("LoCoMo retrieval trace capture requires system.recallWithTrace().");
39320
+ }
39321
+ const loaded = await loadLoCoMoDataset("full", options.datasetDir);
39322
+ const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
39323
+ const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
39324
+ const selectable = plans.flatMap(
39325
+ (plan, planIndex) => plan.trials.map(
39326
+ (trial) => ({
39327
+ taskId: trial.taskId,
39328
+ question: trial.question,
39329
+ recallSessionIds: [...trial.recallSessionIds],
39330
+ planIndex
39331
+ })
39332
+ )
39333
+ );
39334
+ const selection = selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
39335
+ const selectedIds = new Set(selection.selectedTaskIds);
39336
+ const tasks = [];
39337
+ for (let planIndex = 0; planIndex < plans.length; planIndex += 1) {
39338
+ const selected = selectable.filter((task) => task.planIndex === planIndex && selectedIds.has(task.taskId));
39339
+ if (selected.length === 0) continue;
39340
+ const plan = plans[planIndex];
39341
+ if (!plan) throw new Error(`Missing LoCoMo plan at index ${planIndex}.`);
39342
+ await options.system.reset();
39343
+ for (const session of plan.ingestSessions) {
39344
+ if (session.messages.length > 0) {
39345
+ await options.system.store(session.sessionId, session.messages);
39346
+ }
39347
+ }
39348
+ await options.system.drain?.();
39349
+ for (const selectedTask of selected) {
39350
+ const recallBudgetChars = benchmarkRecallBudgetForSessionCount(selectedTask.recallSessionIds.length);
39351
+ const recalled = await Promise.all(
39352
+ selectedTask.recallSessionIds.map(async (sessionId) => {
39353
+ const result = await recallWithTrace(sessionId, selectedTask.question, recallBudgetChars);
39354
+ return {
39355
+ text: result.text,
39356
+ receipt: {
39357
+ session: digestContent(sessionId),
39358
+ trace: sanitizeStructuralTrace(result.trace)
39359
+ }
39360
+ };
39361
+ })
39362
+ );
39363
+ const rawRecalledText = recalled.map((entry) => entry.text).filter(Boolean).join("\n\n");
39364
+ const sanitized = sanitizeLoCoMoRecallText({
39365
+ question: selectedTask.question,
39366
+ recalledText: rawRecalledText
39367
+ });
39368
+ const composition = prioritizeLoCoMoRecallTextWithTrace({
39369
+ question: selectedTask.question,
39370
+ recalledText: sanitized,
39371
+ multiHopRecallComposition
39372
+ });
39373
+ tasks.push({
39374
+ taskId: selectedTask.taskId,
39375
+ question: digestContent(selectedTask.question),
39376
+ recallBudgetChars,
39377
+ sessions: recalled.map((entry) => entry.receipt),
39378
+ composition: composition.receipt
39379
+ });
39380
+ }
39381
+ }
39382
+ const withoutHash = {
39383
+ schemaVersion: LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
39384
+ benchmarkId: "locomo",
39385
+ captureKind: "retrieval-only",
39386
+ sensitivity: {
39387
+ classification: "restricted",
39388
+ contentEncoding: "sha256+length",
39389
+ containsGold: false,
39390
+ containsRawContent: false
39391
+ },
39392
+ provenance: {
39393
+ gitSha: options.gitSha,
39394
+ remnicVersion: options.remnicVersion,
39395
+ runtimeProfile: options.runtimeProfile,
39396
+ adapterMode: "direct",
39397
+ replayExtractionMode: "skip",
39398
+ providerFree: true,
39399
+ dataset: { id: "locomo-10", sha256: loaded.sha256 },
39400
+ retrievalConfigSha256: hashCanonicalJson(retrievalConfig),
39401
+ recallBudget: {
39402
+ algorithm: "benchmarkRecallBudgetForSessionCount",
39403
+ version: LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION
39404
+ }
39405
+ },
39406
+ selection,
39407
+ tasks
39408
+ };
39409
+ return {
39410
+ ...withoutHash,
39411
+ artifactHash: hashCanonicalJson(withoutHash)
39412
+ };
39413
+ }
39414
+ function selectLoCoMoRetrievalTraceTasks(tasks, selector) {
39415
+ const allIds = tasks.map((task) => task.taskId);
39416
+ if (new Set(allIds).size !== allIds.length) {
39417
+ throw new Error("LoCoMo retrieval trace task ids must be unique.");
39418
+ }
39419
+ let selected;
39420
+ let algorithm;
39421
+ let seed;
39422
+ const hasTaskIds = "taskIds" in selector && selector.taskIds !== void 0;
39423
+ const hasSampleSize = "sampleSize" in selector && selector.sampleSize !== void 0;
39424
+ if (Number(hasTaskIds) + Number(hasSampleSize) !== 1) {
39425
+ throw new Error("Choose exactly one LoCoMo retrieval trace selector.");
39426
+ }
39427
+ if (hasTaskIds && "seed" in selector && selector.seed !== void 0) {
39428
+ throw new Error("LoCoMo retrieval trace seed is valid only for seeded sampling.");
39429
+ }
39430
+ if (hasTaskIds) {
39431
+ algorithm = "explicit-task-ids";
39432
+ const requestedTaskIds = selector.taskIds;
39433
+ if (!requestedTaskIds) throw new Error("LoCoMo explicit task ids are required.");
39434
+ const requested = [...requestedTaskIds];
39435
+ if (requested.length === 0) {
39436
+ throw new Error("LoCoMo retrieval trace explicit task selection cannot be empty.");
39437
+ }
39438
+ if (new Set(requested).size !== requested.length) {
39439
+ throw new Error("LoCoMo retrieval trace explicit task ids must not contain duplicates.");
39440
+ }
39441
+ const available = new Set(allIds);
39442
+ const unknown = requested.filter((taskId) => !available.has(taskId));
39443
+ if (unknown.length > 0) {
39444
+ throw new Error(`Unknown LoCoMo retrieval trace task id: ${unknown[0]}`);
39445
+ }
39446
+ const requestedSet = new Set(requested);
39447
+ selected = allIds.filter((taskId) => requestedSet.has(taskId));
39448
+ } else {
39449
+ algorithm = "sha256-seeded-sample";
39450
+ const sampleSize = selector.sampleSize;
39451
+ seed = selector.seed;
39452
+ if (sampleSize === void 0 || seed === void 0) {
39453
+ throw new Error("LoCoMo seeded sampling requires sampleSize and seed.");
39454
+ }
39455
+ if (!Number.isSafeInteger(sampleSize) || sampleSize <= 0 || sampleSize > allIds.length) {
39456
+ throw new Error(`LoCoMo retrieval trace sampleSize must be an integer from 1 to ${allIds.length}.`);
39457
+ }
39458
+ if (!Number.isSafeInteger(seed) || seed < 0) {
39459
+ throw new Error("LoCoMo retrieval trace seed must be a non-negative safe integer.");
39460
+ }
39461
+ const sampled = [...allIds].sort((left, right) => {
39462
+ const leftHash = hashString(`${seed}\0${left}`);
39463
+ const rightHash = hashString(`${seed}\0${right}`);
39464
+ return leftHash.localeCompare(rightHash) || left.localeCompare(right);
39465
+ }).slice(0, sampleSize);
39466
+ const sampledSet = new Set(sampled);
39467
+ selected = allIds.filter((taskId) => sampledSet.has(taskId));
39468
+ }
39469
+ return {
39470
+ algorithm,
39471
+ version: LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
39472
+ ...seed === void 0 ? {} : { seed },
39473
+ candidateCount: allIds.length,
39474
+ selectedCount: selected.length,
39475
+ selectedTaskIds: selected,
39476
+ selectedTaskIdsSha256: hashCanonicalJson(selected)
39477
+ };
39478
+ }
39479
+ function serializeLoCoMoRetrievalTraceReceipt(receipt) {
39480
+ return `${canonicalJsonStringify(receipt, 2)}
39481
+ `;
39482
+ }
39483
+ function sanitizeStructuralTrace(trace2) {
39484
+ return {
39485
+ schemaVersion: trace2.schemaVersion,
39486
+ sensitivity: { ...trace2.sensitivity },
39487
+ sections: trace2.sections.map((section) => ({ ...section })),
39488
+ selections: trace2.selections.map(({ summary, ...selection }) => ({
39489
+ ...selection,
39490
+ ...selection.archiveRowIds === void 0 ? {} : { archiveRowIds: [...selection.archiveRowIds] },
39491
+ ...summary === void 0 ? {} : { summary: { depth: summary.depth, msgStart: summary.msgStart, msgEnd: summary.msgEnd } }
39492
+ })),
39493
+ lcmCandidates: trace2.lcmCandidates.map((candidate) => ({ ...candidate })),
39494
+ ...trace2.coreCapture === void 0 ? {} : {
39495
+ coreCapture: {
39496
+ budget: { ...trace2.coreCapture.budget },
39497
+ filters: trace2.coreCapture.filters.map((filter) => ({ ...filter })),
39498
+ results: trace2.coreCapture.results.map((result) => {
39499
+ assertMemoryIdRef(result.memoryIdRef);
39500
+ const score = result.scoreDecomposition;
39501
+ return {
39502
+ memoryIdRef: {
39503
+ sha256: result.memoryIdRef.sha256,
39504
+ length: result.memoryIdRef.length
39505
+ },
39506
+ servedBy: result.servedBy,
39507
+ scoreDecomposition: {
39508
+ ...score.vector === void 0 ? {} : { vector: score.vector },
39509
+ ...score.bm25 === void 0 ? {} : { bm25: score.bm25 },
39510
+ ...score.importance === void 0 ? {} : { importance: score.importance },
39511
+ ...score.mmrPenalty === void 0 ? {} : { mmrPenalty: score.mmrPenalty },
39512
+ ...score.tierPrior === void 0 ? {} : { tierPrior: score.tierPrior },
39513
+ ...score.reinforcementBoost === void 0 ? {} : { reinforcementBoost: score.reinforcementBoost },
39514
+ final: score.final
39515
+ },
39516
+ admittedBy: [...result.admittedBy],
39517
+ ...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
39518
+ ...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
39519
+ ...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
39520
+ };
39521
+ })
39522
+ }
39523
+ },
39524
+ budget: { ...trace2.budget }
39525
+ };
39526
+ }
39527
+ function digestContent(value) {
39528
+ return {
39529
+ sha256: hashString(value),
39530
+ charCount: value.length,
39531
+ lineCount: value.length === 0 ? 0 : value.split("\n").length
39532
+ };
39533
+ }
39534
+ function assertCaptureOptions(options) {
39535
+ if (!options.datasetDir.trim()) {
39536
+ throw new Error("LoCoMo retrieval trace capture requires datasetDir.");
39537
+ }
39538
+ if (options.runtimeProfile !== "baseline" && options.runtimeProfile !== "real") {
39539
+ throw new Error('LoCoMo retrieval trace runtimeProfile must be "baseline" or "real".');
39540
+ }
39541
+ if (!options.gitSha.trim() || !options.remnicVersion.trim() || options.gitSha === "unknown" || options.remnicVersion === "unknown") {
39542
+ throw new Error("LoCoMo retrieval trace provenance requires gitSha and remnicVersion.");
39543
+ }
39544
+ if (options.providerFreeConfirmed !== true) {
39545
+ throw new Error("LoCoMo retrieval trace capture requires explicit provider-free confirmation.");
39546
+ }
39547
+ }
39548
+ function assertProviderFreeRetrievalConfig(value) {
39549
+ const config = assertJsonConfig(value);
39550
+ for (const key of [
39551
+ "localLlmEnabled",
39552
+ "localLlmFastEnabled",
39553
+ "recallPlannerEnabled",
39554
+ "embeddingFallbackEnabled",
39555
+ "hostEmbeddingProviderEnabled",
39556
+ "openaiApiKey"
39557
+ ]) {
39558
+ if (config[key] !== false) {
39559
+ throw new Error(`retrievalConfig.${key} must be false for provider-free capture.`);
39560
+ }
39561
+ }
39562
+ if (config.modelSource !== "plugin") {
39563
+ throw new Error('retrievalConfig.modelSource must be "plugin" for provider-free capture.');
39564
+ }
39565
+ return config;
39566
+ }
39567
+ function assertMemoryIdRef(value) {
39568
+ if (!value || typeof value !== "object" || !/^[0-9a-f]{64}$/u.test(value.sha256) || !Number.isSafeInteger(value.length) || value.length <= 0) {
39569
+ throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
39570
+ }
39571
+ }
39572
+ function assertJsonConfig(value, path40 = "retrievalConfig") {
39573
+ if (value === null || typeof value === "string" || typeof value === "boolean") return value;
39574
+ if (typeof value === "number") {
39575
+ if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
39576
+ return value;
39577
+ }
39578
+ if (Array.isArray(value)) {
39579
+ return value.map((entry, index) => assertJsonConfig(entry, `${path40}[${index}]`));
39580
+ }
39581
+ if (!value || typeof value !== "object") {
39582
+ throw new Error(`${path40} must be JSON-serializable and provider-free.`);
39583
+ }
39584
+ const output = {};
39585
+ for (const key of Object.keys(value).sort()) {
39586
+ const child = value[key];
39587
+ if (key === "openaiApiKey") {
39588
+ if (child !== false) {
39589
+ throw new Error(`${path40}.${key} must be exactly false for provider-free capture.`);
39590
+ }
39591
+ output[key] = false;
39592
+ continue;
39593
+ }
39594
+ if (isSecretKey(key)) {
39595
+ throw new Error(`${path40}.${key} contains secret-bearing configuration.`);
39596
+ }
39597
+ if (child === void 0) continue;
39598
+ if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
39599
+ throw new Error(`${path40}.${key} is provider-capable configuration.`);
39600
+ }
39601
+ output[key] = assertJsonConfig(child, `${path40}.${key}`);
39602
+ }
39603
+ return output;
39604
+ }
39605
+ function sanitizeProviderFreeRetrievalConfig(value, path40 = "retrievalConfig") {
39606
+ if (value === null || typeof value === "string" || typeof value === "boolean") return value;
39607
+ if (typeof value === "number") {
39608
+ if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
39609
+ return value;
39610
+ }
39611
+ if (Array.isArray(value)) {
39612
+ return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path40}[${index}]`));
39613
+ }
39614
+ if (!value || typeof value !== "object") {
39615
+ throw new Error(`${path40} must be JSON-serializable.`);
39616
+ }
39617
+ const output = {};
39618
+ for (const key of Object.keys(value).sort()) {
39619
+ const child = value[key];
39620
+ if (child === void 0) continue;
39621
+ if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
39622
+ continue;
39623
+ }
39624
+ output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path40}.${key}`);
39625
+ }
39626
+ return output;
39627
+ }
39628
+
38819
39629
  // src/integrity/sealed-qrels.ts
38820
39630
  import { readFile as readFile20 } from "fs/promises";
38821
39631
  function isSealedQrelsArtifact(value) {
@@ -40219,7 +41029,7 @@ var chatFixture = {
40219
41029
  };
40220
41030
 
40221
41031
  // src/judges/calibration-slice.ts
40222
- import { createHash as createHash15, randomBytes as randomBytes3 } from "crypto";
41032
+ import { createHash as createHash16, randomBytes as randomBytes3 } from "crypto";
40223
41033
  import { chmod as chmod2, lstat as lstat4, mkdir as mkdir18, open as open2, readFile as readFile22, rename as rename4, unlink as unlink4, writeFile as writeFile17 } from "fs/promises";
40224
41034
  import path37 from "path";
40225
41035
 
@@ -40368,7 +41178,7 @@ function selectCalibrationSlice(questionIds, size = CALIBRATION_SLICE_SIZE) {
40368
41178
  unique.push(id);
40369
41179
  }
40370
41180
  }
40371
- return unique.map((id) => ({ id, digest: createHash15("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
41181
+ return unique.map((id) => ({ id, digest: createHash16("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
40372
41182
  }
40373
41183
  async function runJudgeCalibration(options) {
40374
41184
  const binScore = options.binScore ?? ((score) => binarizeJudgeScore(score));
@@ -40485,7 +41295,7 @@ function hashOrderedQuestionIds(questionIds) {
40485
41295
  if (questionIds.some((id) => typeof id !== "string" || id.length === 0)) {
40486
41296
  throw new Error("hashOrderedQuestionIds: question ids must be non-empty strings.");
40487
41297
  }
40488
- return createHash15("sha256").update(JSON.stringify(questionIds), "utf8").digest("hex");
41298
+ return createHash16("sha256").update(JSON.stringify(questionIds), "utf8").digest("hex");
40489
41299
  }
40490
41300
  function validatePinnedQuestionIds(ids, availableIds) {
40491
41301
  if (ids.length === 0 || ids.length > CALIBRATION_SLICE_SIZE || ids.some((id) => typeof id !== "string" || id.length === 0) || new Set(ids).size !== ids.length) {
@@ -40498,7 +41308,7 @@ function validatePinnedQuestionIds(ids, availableIds) {
40498
41308
  return [...ids];
40499
41309
  }
40500
41310
  function hashCalibrationAnswerSet(answers) {
40501
- return createHash15("sha256").update(JSON.stringify(answers.map((answer) => [
41311
+ return createHash16("sha256").update(JSON.stringify(answers.map((answer) => [
40502
41312
  answer.questionId,
40503
41313
  answer.question,
40504
41314
  answer.predicted,
@@ -40564,7 +41374,7 @@ async function loadOrInitializeCheckpoint(benchmarkId, provenance, sliceQuestion
40564
41374
  frontierJudgeConfigHash: provenance.frontierJudgeConfigHash,
40565
41375
  binningIdentity: provenance.binningIdentity
40566
41376
  };
40567
- const contractHash = createHash15("sha256").update(stableJson2(contract)).digest("hex");
41377
+ const contractHash = createHash16("sha256").update(stableJson2(contract)).digest("hex");
40568
41378
  let raw;
40569
41379
  try {
40570
41380
  const info = await lstat4(checkpointPath);
@@ -42085,7 +42895,7 @@ function createMitigatedTarget(config) {
42085
42895
  }
42086
42896
 
42087
42897
  // src/coding-graph/generator.ts
42088
- import { createHash as createHash16 } from "crypto";
42898
+ import { createHash as createHash17 } from "crypto";
42089
42899
  function createSeededRng3(seed) {
42090
42900
  let state = seed >>> 0;
42091
42901
  return function rng() {
@@ -42114,7 +42924,7 @@ var EDGE_TYPE_WEIGHTS = [
42114
42924
  var PROVENANCE_VALUES = ["heuristic", "heuristic", "heuristic", "trace"];
42115
42925
  var AVG_BYTES_PER_LINE = 40;
42116
42926
  function hashContent(input) {
42117
- return createHash16("sha256").update(input).digest("hex").slice(0, 16);
42927
+ return createHash17("sha256").update(input).digest("hex").slice(0, 16);
42118
42928
  }
42119
42929
  function generateSyntheticRepo(config) {
42120
42930
  const rng = createSeededRng3(config.seed);
@@ -42686,6 +43496,9 @@ export {
42686
43496
  LOCOMO_FULL_TASK_COUNT,
42687
43497
  LOCOMO_RECALL_DIFF_LINE_LIMIT,
42688
43498
  LOCOMO_RECALL_EXCERPT_CHARS,
43499
+ LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION,
43500
+ LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
43501
+ LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
42689
43502
  LONG_MEM_EVAL_DATASET_FILENAMES,
42690
43503
  LettaMemCorrectAdapter,
42691
43504
  LocalLabPreflightError,
@@ -42742,10 +43555,12 @@ export {
42742
43555
  buildCodexCreditReceipt,
42743
43556
  buildJudgePayload,
42744
43557
  buildOracleTrajectoryRecall,
43558
+ buildProviderFreeLoCoMoRetrievalConfig,
42745
43559
  buildSchemaTierFixture,
42746
43560
  buildSchemaTierSmokeFixture,
42747
43561
  calendarFixture,
42748
43562
  canonicalJsonStringify,
43563
+ captureLoCoMoRetrievalTrace,
42749
43564
  captureMachineFingerprint,
42750
43565
  chatFixture,
42751
43566
  checkCodingGraphRegression,
@@ -42814,6 +43629,7 @@ export {
42814
43629
  getAblationCell,
42815
43630
  getBenchmark,
42816
43631
  getBenchmarkLowerIsBetter,
43632
+ getGitSha,
42817
43633
  getMemoryEvalDimension,
42818
43634
  getProviderBackedJudgePromptIdentity,
42819
43635
  getRemnicVersion,
@@ -42867,6 +43683,7 @@ export {
42867
43683
  parseSealedQrels,
42868
43684
  pickStableQualifiedName,
42869
43685
  precisionAtK,
43686
+ preflightLoCoMoRetrievalTraceCapture,
42870
43687
  preflightLocalLabRole,
42871
43688
  projectFolderFixture,
42872
43689
  recallAtK,
@@ -42922,6 +43739,7 @@ export {
42922
43739
  selectFixtureVariant,
42923
43740
  serializeBenchmarkArtifact,
42924
43741
  serializeJsonl,
43742
+ serializeLoCoMoRetrievalTraceReceipt,
42925
43743
  serializeSealedQrels,
42926
43744
  shuffleTasks,
42927
43745
  timed,