@remnic/bench 9.45.2 → 9.45.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.d.ts +50 -1
  2. package/dist/index.js +1586 -1071
  3. package/package.json +3 -3
package/dist/index.js CHANGED
@@ -1977,6 +1977,63 @@ function isProviderConfigLike(value) {
1977
1977
  }
1978
1978
  return isObjectRecord(value) && typeof value.provider === "string" && typeof value.model === "string";
1979
1979
  }
1980
+ function isNonEmptyString(value) {
1981
+ return typeof value === "string" && value.trim().length > 0;
1982
+ }
1983
+ function isNonNegativeInteger(value) {
1984
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
1985
+ }
1986
+ function isUniqueMemoryIdList(value) {
1987
+ if (value === null) return true;
1988
+ if (!Array.isArray(value)) return false;
1989
+ const ids = /* @__PURE__ */ new Set();
1990
+ for (const id of value) {
1991
+ if (!isNonEmptyString(id) || ids.has(id)) return false;
1992
+ ids.add(id);
1993
+ }
1994
+ return true;
1995
+ }
1996
+ function isTaskAttributionWitnessLike(value, goldMemories) {
1997
+ if (!isObjectRecord(value) || value.schemaVersion !== 1) {
1998
+ return false;
1999
+ }
2000
+ const runtime = value.runtime;
2001
+ if (!isObjectRecord(runtime) || !isNonEmptyString(runtime.qmdCollection) || !isNonEmptyString(runtime.qmdIndex) || !isNonNegativeInteger(runtime.qmdMaxResults) || typeof runtime.attributionThreshold !== "number" || !Number.isFinite(runtime.attributionThreshold) || runtime.attributionThreshold < 0 || runtime.attributionThreshold > 1) {
2002
+ return false;
2003
+ }
2004
+ if (!Array.isArray(value.golds) || goldMemories !== void 0 && value.golds.length !== goldMemories.length) {
2005
+ return false;
2006
+ }
2007
+ for (let index = 0; index < value.golds.length; index += 1) {
2008
+ const gold = value.golds[index];
2009
+ if (!isObjectRecord(gold) || !isNonEmptyString(gold.goldMemory) || goldMemories !== void 0 && gold.goldMemory !== goldMemories[index] || !isUniqueMemoryIdList(gold.storeMemoryIds) || !isUniqueMemoryIdList(gold.oracleMemoryIds)) {
2010
+ return false;
2011
+ }
2012
+ }
2013
+ if (!Array.isArray(value.retrievals)) {
2014
+ return false;
2015
+ }
2016
+ const sessionIds = /* @__PURE__ */ new Set();
2017
+ for (const retrieval of value.retrievals) {
2018
+ if (!isObjectRecord(retrieval) || !isNonEmptyString(retrieval.sessionId) || sessionIds.has(retrieval.sessionId) || !(retrieval.appliedCap === null || isNonNegativeInteger(retrieval.appliedCap)) || !isUniqueMemoryIdList(retrieval.atCapMemoryIds) || !isUniqueMemoryIdList(retrieval.headroomMemoryIds)) {
2019
+ return false;
2020
+ }
2021
+ sessionIds.add(retrieval.sessionId);
2022
+ const appliedCap = retrieval.appliedCap;
2023
+ const atCapIds = retrieval.atCapMemoryIds;
2024
+ const headroomIds = retrieval.headroomMemoryIds;
2025
+ if (appliedCap === null && (atCapIds !== null || headroomIds !== null) || appliedCap !== null && atCapIds !== null && atCapIds.length > appliedCap || appliedCap !== null && atCapIds !== null && atCapIds.length < appliedCap && headroomIds === null || appliedCap !== null && headroomIds !== null && headroomIds.length > 0 && (atCapIds === null || atCapIds.length !== appliedCap)) {
2026
+ return false;
2027
+ }
2028
+ if (atCapIds !== null && headroomIds !== null) {
2029
+ const atCapSet = new Set(atCapIds);
2030
+ if (headroomIds.some((id) => atCapSet.has(id))) {
2031
+ return false;
2032
+ }
2033
+ }
2034
+ }
2035
+ return true;
2036
+ }
1980
2037
  function isBenchmarkResult(value) {
1981
2038
  if (!isObjectRecord(value)) {
1982
2039
  return false;
@@ -2013,6 +2070,9 @@ function isTaskResultLike(value) {
2013
2070
  if (goldMemories !== void 0 && (!Array.isArray(goldMemories) || !goldMemories.every((item) => typeof item === "string"))) {
2014
2071
  return false;
2015
2072
  }
2073
+ if (value.attributionWitness !== void 0 && !isTaskAttributionWitnessLike(value.attributionWitness, goldMemories)) {
2074
+ return false;
2075
+ }
2016
2076
  return typeof value.taskId === "string" && typeof value.question === "string" && typeof value.expected === "string" && typeof value.actual === "string" && isObjectRecord(value.scores) && Object.values(value.scores).every(isFiniteNumber) && isFiniteNumber(value.latencyMs) && isObjectRecord(tokens) && isFiniteNumber(tokens.input) && isFiniteNumber(tokens.output);
2017
2077
  }
2018
2078
  function isStoredBenchmarkBaseline(value) {
@@ -5587,7 +5647,8 @@ import {
5587
5647
  parseConfig,
5588
5648
  parseEntityFile,
5589
5649
  serializeEntityFile,
5590
- StorageManager
5650
+ StorageManager,
5651
+ withRawEntityPageMutation
5591
5652
  } from "@remnic/core";
5592
5653
 
5593
5654
  // src/adapters/with-bench-core-memory-source.ts
@@ -5625,344 +5686,1132 @@ async function withBenchCoreMemorySource(orchestrator, source, task) {
5625
5686
  }
5626
5687
  }
5627
5688
 
5628
- // src/adapters/remnic-adapter.ts
5629
- import {
5630
- lcmEvidenceIdentity
5631
- } from "@remnic/core/lcm";
5632
-
5633
- // src/adapters/remnic-recall-trace.ts
5634
- import { createHash as createHash6 } from "crypto";
5635
- import {
5636
- lcmArchiveRowId
5637
- } from "@remnic/core/lcm";
5638
- function visibleRange(start, end, returnedChars) {
5639
- const visibleStart = Math.min(start, returnedChars);
5640
- return {
5641
- visibleStart,
5642
- visibleEnd: Math.max(visibleStart, Math.min(end, returnedChars))
5643
- };
5644
- }
5645
- function projectBenchCoreCapture(snapshot) {
5646
- return {
5647
- snapshotId: snapshot.snapshotId,
5648
- capturedAt: snapshot.capturedAt,
5649
- ...snapshot.traceId === void 0 ? {} : { traceId: snapshot.traceId },
5650
- budget: { chars: snapshot.budget.chars, used: snapshot.budget.used },
5651
- filters: snapshot.filters.map(({ name, considered, admitted }) => ({
5652
- name,
5653
- considered,
5654
- admitted
5655
- })),
5656
- results: snapshot.results.map((result) => {
5657
- const scores = result.scoreDecomposition;
5658
- return {
5659
- memoryIdRef: {
5660
- sha256: createHash6("sha256").update(result.memoryId, "utf8").digest("hex"),
5661
- length: Buffer.byteLength(result.memoryId, "utf8")
5662
- },
5663
- servedBy: result.servedBy,
5664
- scoreDecomposition: {
5665
- ...typeof scores.vector === "number" ? { vector: scores.vector } : {},
5666
- ...typeof scores.bm25 === "number" ? { bm25: scores.bm25 } : {},
5667
- ...typeof scores.importance === "number" ? { importance: scores.importance } : {},
5668
- ...typeof scores.mmrPenalty === "number" ? { mmrPenalty: scores.mmrPenalty } : {},
5669
- ...typeof scores.tierPrior === "number" ? { tierPrior: scores.tierPrior } : {},
5670
- ...typeof scores.reinforcementBoost === "number" ? { reinforcementBoost: scores.reinforcementBoost } : {},
5671
- final: scores.final
5672
- },
5673
- admittedBy: [...result.admittedBy],
5674
- ...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
5675
- ...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
5676
- ...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
5677
- };
5678
- })
5679
- };
5680
- }
5681
- function createBenchRecallTraceRecorder(requestedChars) {
5682
- const sections = [];
5683
- const pendingSelections = [];
5684
- const lcmCandidates = [];
5685
- let composedChars = 0;
5686
- let coreCapture;
5687
- const sectionById = (sectionId) => {
5688
- const section = sections.find((entry) => entry.id === sectionId);
5689
- if (!section) throw new Error(`Unknown benchmark recall trace section: ${sectionId}`);
5690
- return section;
5691
- };
5692
- const appendRelativeSelection = (sectionId, kind, start, end, lineageStatus, fields = {}) => {
5693
- const section = sectionById(sectionId);
5694
- const contentLength = section.contentEnd - section.contentStart;
5695
- if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > contentLength) {
5696
- throw new Error(
5697
- `Invalid benchmark recall trace range for ${sectionId}: ${start}..${end}.`
5698
- );
5699
- }
5700
- pendingSelections.push({
5701
- sectionId,
5702
- kind,
5703
- lineageStatus,
5704
- composedStart: section.contentStart + start,
5705
- composedEnd: section.contentStart + end,
5706
- ...fields
5707
- });
5708
- };
5709
- return {
5710
- appendSection(id, source, renderedLength) {
5711
- if (!Number.isSafeInteger(renderedLength) || renderedLength < 0) {
5712
- throw new Error("Benchmark recall trace section length must be a non-negative integer.");
5713
- }
5714
- if (sections.some((section) => section.id === id)) {
5715
- throw new Error(`Duplicate benchmark recall trace section: ${id}`);
5716
- }
5717
- const separatorStart = composedChars;
5718
- const contentStart = composedChars + (sections.length === 0 ? 0 : 2);
5719
- const contentEnd = contentStart + renderedLength;
5720
- sections.push({
5721
- id,
5722
- source,
5723
- separatorStart,
5724
- contentStart,
5725
- contentEnd,
5726
- composedStart: separatorStart,
5727
- composedEnd: contentEnd,
5728
- visibleStart: 0,
5729
- visibleEnd: 0,
5730
- visibleChars: 0
5731
- });
5732
- composedChars = contentEnd;
5733
- },
5734
- recordEvidenceSelections(sectionId, receipts) {
5735
- for (const receipt of receipts) {
5736
- appendRelativeSelection(
5737
- sectionId,
5738
- "evidence-block",
5739
- receipt.blockStart,
5740
- receipt.blockEnd,
5741
- receipt.item.archiveRowId === void 0 ? "unavailable" : "exact",
5742
- {
5743
- ...receipt.item.archiveRowId === void 0 ? {} : { archiveRowIds: [receipt.item.archiveRowId] },
5744
- ...receipt.item.turnIndex === void 0 ? {} : { turnIndex: receipt.item.turnIndex },
5745
- ...receipt.item.role === void 0 ? {} : { role: receipt.item.role },
5746
- ...receipt.item.score === void 0 ? {} : { score: receipt.item.score }
5747
- }
5748
- );
5749
- }
5750
- },
5751
- recordTrajectorySelections(sectionId, receipts) {
5752
- for (const receipt of receipts) {
5753
- appendRelativeSelection(
5754
- sectionId,
5755
- "trajectory-line",
5756
- receipt.lineStart,
5757
- receipt.lineEnd,
5758
- receipt.lineageStatus,
5759
- {
5760
- archiveRowIds: [
5761
- ...receipt.actionArchiveRowIds,
5762
- ...receipt.observationArchiveRowIds
5763
- ]
5764
- }
5765
- );
5766
- }
5767
- },
5768
- recordSummarySelections(sectionId, receipts) {
5769
- for (const receipt of receipts) {
5770
- appendRelativeSelection(
5771
- sectionId,
5772
- "lcm-summary",
5773
- receipt.entryStart,
5774
- receipt.entryEnd,
5775
- "exact",
5776
- {
5777
- summary: {
5778
- id: receipt.id,
5779
- depth: receipt.depth,
5780
- msgStart: receipt.msgStart,
5781
- msgEnd: receipt.msgEnd
5782
- }
5783
- }
5784
- );
5785
- }
5786
- },
5787
- recordRawRow(sectionId, range, row) {
5788
- const archiveRowId = lcmArchiveRowId(row);
5789
- appendRelativeSelection(
5790
- sectionId,
5791
- "raw-row",
5792
- range.start,
5793
- range.end,
5794
- archiveRowId === void 0 ? "unavailable" : "exact",
5795
- {
5796
- ...archiveRowId === void 0 ? {} : { archiveRowIds: [archiveRowId] },
5797
- turnIndex: row.turn_index,
5798
- role: row.role
5799
- }
5800
- );
5801
- },
5802
- recordLcmCandidate(candidate) {
5803
- lcmCandidates.push({ ...candidate });
5804
- },
5805
- recordCoreCapture(snapshot) {
5806
- coreCapture = snapshot ? projectBenchCoreCapture(snapshot) : void 0;
5807
- },
5808
- finalize(returnedChars) {
5809
- const normalizedReturnedChars = Math.max(0, Math.min(returnedChars, composedChars));
5810
- return {
5811
- schemaVersion: 1,
5812
- sensitivity: {
5813
- classification: "restricted",
5814
- contentEncoding: "sha256+length",
5815
- containsGold: false
5816
- },
5817
- sections: sections.map((section) => {
5818
- const visible = visibleRange(
5819
- section.composedStart,
5820
- section.composedEnd,
5821
- normalizedReturnedChars
5822
- );
5823
- return {
5824
- ...section,
5825
- ...visible,
5826
- visibleChars: visible.visibleEnd - visible.visibleStart
5827
- };
5828
- }),
5829
- selections: pendingSelections.map((selection) => ({
5830
- ...selection,
5831
- ...visibleRange(
5832
- selection.composedStart,
5833
- selection.composedEnd,
5834
- normalizedReturnedChars
5835
- )
5836
- })),
5837
- lcmCandidates: lcmCandidates.map((candidate) => ({ ...candidate })),
5838
- ...coreCapture === void 0 ? {} : { coreCapture },
5839
- budget: {
5840
- requestedChars,
5841
- composedChars,
5842
- returnedChars: normalizedReturnedChars,
5843
- truncated: normalizedReturnedChars < composedChars
5844
- }
5845
- };
5846
- }
5847
- };
5848
- }
5849
-
5850
- // src/recall-budget.ts
5851
- var DEFAULT_BENCH_RECALL_BUDGET_CHARS = 24e3;
5852
- var MAX_COMBINED_RECALL_BUDGET_CHARS = 36e3;
5853
- function benchmarkRecallBudgetForSessionCount(sessionCount) {
5854
- if (!Number.isInteger(sessionCount) || sessionCount <= 0) {
5855
- return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
5856
- }
5857
- if (sessionCount === 1) {
5858
- return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
5859
- }
5860
- return Math.floor(MAX_COMBINED_RECALL_BUDGET_CHARS / sessionCount);
5861
- }
5862
-
5863
- // src/adapters/remnic-adapter.ts
5864
- var DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE = 0.34;
5865
- var ANSWER_SUPPORT_STOP_WORDS = /* @__PURE__ */ new Set([
5866
- "about",
5867
- "after",
5868
- "again",
5869
- "also",
5870
- "answer",
5871
- "before",
5872
- "being",
5873
- "could",
5874
- "does",
5689
+ // src/attribution.ts
5690
+ var DEFAULT_ATTRIBUTION_THRESHOLD = 0.6;
5691
+ var DEFAULT_STOPWORDS = /* @__PURE__ */ new Set([
5692
+ "a",
5693
+ "an",
5694
+ "the",
5695
+ "in",
5696
+ "on",
5697
+ "at",
5698
+ "to",
5699
+ "for",
5700
+ "of",
5701
+ "with",
5702
+ "by",
5875
5703
  "from",
5876
- "have",
5877
- "information",
5704
+ "up",
5705
+ "about",
5878
5706
  "into",
5879
- "just",
5880
- "know",
5881
- "memory",
5882
- "might",
5883
- "please",
5884
- "question",
5885
- "recall",
5886
- "remember",
5887
- "should",
5707
+ "through",
5708
+ "during",
5709
+ "before",
5710
+ "after",
5711
+ "above",
5712
+ "below",
5713
+ "and",
5714
+ "or",
5715
+ "but",
5716
+ "if",
5717
+ "then",
5718
+ "else",
5719
+ "when",
5720
+ "where",
5721
+ "why",
5722
+ "how",
5723
+ "all",
5724
+ "any",
5725
+ "both",
5726
+ "each",
5727
+ "few",
5728
+ "more",
5729
+ "most",
5730
+ "other",
5731
+ "some",
5732
+ "such",
5733
+ "no",
5734
+ "nor",
5735
+ "not",
5736
+ "only",
5737
+ "own",
5738
+ "same",
5739
+ "so",
5740
+ "than",
5741
+ "too",
5742
+ "very",
5743
+ "this",
5888
5744
  "that",
5889
- "their",
5890
- "there",
5891
5745
  "these",
5892
- "they",
5893
- "this",
5894
5746
  "those",
5895
- "user",
5896
- "using",
5897
- "what",
5898
- "when",
5899
- "where",
5900
- "which",
5901
- "while",
5902
- "with",
5903
- "would",
5904
- "your"
5747
+ "it",
5748
+ "its",
5749
+ "is",
5750
+ "are",
5751
+ "was",
5752
+ "were",
5753
+ "be",
5754
+ "been",
5755
+ "being",
5756
+ "have",
5757
+ "has",
5758
+ "had",
5759
+ "do",
5760
+ "does",
5761
+ "did"
5905
5762
  ]);
5906
- function resolveAnswerSupportMinCoverage(config) {
5907
- const raw = config?.answerSupportMinCoverage;
5908
- if (raw === void 0) return DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE;
5909
- const parsed = typeof raw === "number" ? raw : Number(raw);
5910
- if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) {
5911
- throw new Error("answerSupportMinCoverage must be a finite number greater than 0 and at most 1.");
5763
+ function extractContentWords(text) {
5764
+ const cleaned = text.toLowerCase().replace(/[^a-z0-9\s]/g, " ");
5765
+ const tokens = cleaned.split(/\s+/).filter(Boolean);
5766
+ return tokens.filter((t) => !DEFAULT_STOPWORDS.has(t));
5767
+ }
5768
+ function lexicalSimilarity(a, b) {
5769
+ const goldWords = extractContentWords(a);
5770
+ if (goldWords.length === 0) {
5771
+ return 0;
5912
5772
  }
5913
- return parsed;
5773
+ const candWords = new Set(extractContentWords(b));
5774
+ let matchCount = 0;
5775
+ for (const word of goldWords) {
5776
+ if (candWords.has(word)) {
5777
+ matchCount++;
5778
+ }
5779
+ }
5780
+ return matchCount / goldWords.length;
5914
5781
  }
5915
- function resolveSkipExtractionLcmFirst(config) {
5916
- const raw = config?.skipExtractionLcmFirst;
5917
- if (raw === void 0) return true;
5918
- if (typeof raw === "boolean") return raw;
5919
- if (typeof raw === "string") {
5920
- const normalized = raw.trim().toLowerCase();
5921
- if (["true", "1", "yes", "on"].includes(normalized)) return true;
5922
- if (["false", "0", "no", "off"].includes(normalized)) return false;
5782
+ var CLASS_RANK = {
5783
+ extraction_miss: 1,
5784
+ index_miss: 2,
5785
+ retrieval_miss: 3,
5786
+ use_miss: 4,
5787
+ unattributed: 5
5788
+ };
5789
+ var RETRIEVAL_STAGE_RANK = {
5790
+ cap: 1,
5791
+ rank: 2,
5792
+ filter: 3,
5793
+ unknown: 4
5794
+ };
5795
+ function computeOverallLabel(golds) {
5796
+ if (golds.length === 0) {
5797
+ return { class: "unattributed", reason: "no gold memories" };
5923
5798
  }
5924
- throw new Error(
5925
- "skipExtractionLcmFirst must be a boolean or one of true/false, 1/0, yes/no, on/off."
5926
- );
5799
+ let bestGold = golds[0];
5800
+ for (let i = 1; i < golds.length; i++) {
5801
+ const current = golds[i];
5802
+ const bestRank = CLASS_RANK[bestGold.label.class];
5803
+ const currRank = CLASS_RANK[current.label.class];
5804
+ if (currRank < bestRank) {
5805
+ bestGold = current;
5806
+ } else if (currRank === bestRank && current.label.class === "retrieval_miss") {
5807
+ const bestRetRank = RETRIEVAL_STAGE_RANK[bestGold.label.retrievalStage ?? "unknown"];
5808
+ const currRetRank = RETRIEVAL_STAGE_RANK[current.label.retrievalStage ?? "unknown"];
5809
+ if (currRetRank < bestRetRank) {
5810
+ bestGold = current;
5811
+ }
5812
+ }
5813
+ }
5814
+ return { ...bestGold.label };
5927
5815
  }
5928
- function shouldIncludeCoreRecallForReplay(options) {
5929
- return options.useCoreMemoryPipeline && (options.replayExtractionMode !== "skip" || !options.skipExtractionLcmFirst);
5816
+ function isDiagnosticScore(name) {
5817
+ return name.endsWith("_agreement") || name.includes("_id_leak") || name === "search_hits";
5930
5818
  }
5931
- function normalizeSupportToken(value) {
5932
- if (value.length > 5 && value.endsWith("ing")) return value.slice(0, -3);
5933
- if (value.length > 4 && value.endsWith("ed")) {
5934
- const base = value.slice(0, -2);
5935
- return /[vs]$/.test(base) ? `${base}e` : base;
5819
+ function isTaskFailed(task) {
5820
+ if (!task.scores || Object.keys(task.scores).length === 0) {
5821
+ return true;
5936
5822
  }
5937
- if (value.length > 4 && value.endsWith("es")) return value.slice(0, -2);
5938
- if (value.length > 3 && value.endsWith("s")) return value.slice(0, -1);
5939
- return value;
5940
- }
5941
- function supportTerms(value) {
5942
- return [...new Set(
5943
- (value.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}_-]{2,}/gu) ?? []).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term)).map(normalizeSupportToken).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term) && !/^\d+$/.test(term))
5944
- )];
5823
+ if ("overall" in task.scores && typeof task.scores.overall === "number") {
5824
+ return task.scores.overall < 1;
5825
+ }
5826
+ const primaryScores = Object.entries(task.scores).filter(([name, score]) => typeof score === "number" && !isDiagnosticScore(name)).map(([, score]) => score);
5827
+ return primaryScores.length === 0 || Math.min(...primaryScores) < 1;
5945
5828
  }
5946
- function exactContextEvidenceLines(recalledText) {
5947
- return recalledText.split(/\r?\n/).map((line) => line.trim()).filter((line) => {
5948
- if (!line || /^#{1,6}\s/.test(line)) return false;
5949
- return !/^(?:answer guidance:|distinct user-stated targets found:|no (?:direct|historically valid)|these direct temporal statements|this is the most recent|use this list|when answering)/i.test(line);
5950
- });
5829
+ function withMemoizedListMemories(env) {
5830
+ let cache = null;
5831
+ return {
5832
+ ...env,
5833
+ listMemories() {
5834
+ if (!cache) {
5835
+ cache = env.listMemories();
5836
+ }
5837
+ return cache;
5838
+ }
5839
+ };
5951
5840
  }
5952
- function assessRemnicRecallSupport(request, supportThreshold = DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE) {
5953
- if (request.recalledText.trim().length === 0) {
5954
- return { status: "empty", reason: "exact responder context is empty", evidenceCount: 0 };
5841
+ async function attributeGoldMemory(goldStatement, question, env, options = {}, recalledText) {
5842
+ if (options.threshold !== void 0) {
5843
+ if (typeof options.threshold !== "number" || !Number.isFinite(options.threshold) || options.threshold < 0 || options.threshold > 1) {
5844
+ throw new RangeError("attribution threshold must be a finite number between 0 and 1");
5845
+ }
5955
5846
  }
5956
- const queryTerms = supportTerms(request.query);
5957
- if (queryTerms.length < 2) {
5958
- return {
5959
- status: "unavailable",
5960
- reason: "query has fewer than two distinctive terms for conservative support scoring"
5961
- };
5847
+ const threshold = options.threshold ?? DEFAULT_ATTRIBUTION_THRESHOLD;
5848
+ const simFn = options.similarity ?? lexicalSimilarity;
5849
+ const goldInRecalledText = typeof recalledText === "string" && simFn(goldStatement, recalledText) >= threshold;
5850
+ const stages = {
5851
+ extraction: { status: "unavailable" },
5852
+ index: { status: "unavailable" },
5853
+ retrieval: { status: "unavailable" },
5854
+ use: { status: "unavailable" }
5855
+ };
5856
+ let memories = [];
5857
+ let extractionRan = false;
5858
+ let extractionErrorDetail;
5859
+ if (typeof env.listMemories === "function") {
5860
+ try {
5861
+ memories = await env.listMemories();
5862
+ extractionRan = true;
5863
+ } catch {
5864
+ extractionRan = false;
5865
+ extractionErrorDetail = "listMemories failed";
5866
+ }
5962
5867
  }
5963
- const evidenceLines = exactContextEvidenceLines(request.recalledText);
5964
- const evidenceTermSets = evidenceLines.map((line) => new Set(supportTerms(line)));
5965
- const matchedTerms = queryTerms.filter(
5868
+ let bestSim = -1;
5869
+ let matchedMem = null;
5870
+ if (extractionRan) {
5871
+ if (memories.length === 0) {
5872
+ if (goldInRecalledText) {
5873
+ const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
5874
+ stages.extraction = { status: "pass", detail: impliedDetail };
5875
+ stages.index = { status: "pass", detail: impliedDetail };
5876
+ stages.retrieval = { status: "pass", detail: impliedDetail };
5877
+ stages.use = {
5878
+ status: "fail",
5879
+ detail: "Gold memory present in context but answer was incorrect"
5880
+ };
5881
+ return {
5882
+ goldMemory: goldStatement,
5883
+ label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
5884
+ stages
5885
+ };
5886
+ }
5887
+ const detail = "store contains no memories";
5888
+ stages.extraction = { status: "fail", detail };
5889
+ stages.index = { status: "unavailable", detail: "not reached" };
5890
+ stages.retrieval = { status: "unavailable", detail: "not reached" };
5891
+ stages.use = { status: "unavailable", detail: "not reached" };
5892
+ return {
5893
+ goldMemory: goldStatement,
5894
+ label: { class: "extraction_miss", reason: detail },
5895
+ stages
5896
+ };
5897
+ }
5898
+ for (const mem of memories) {
5899
+ const sim = simFn(goldStatement, mem.content);
5900
+ if (sim > bestSim) {
5901
+ bestSim = sim;
5902
+ matchedMem = mem;
5903
+ }
5904
+ }
5905
+ if (bestSim < threshold || !matchedMem) {
5906
+ if (goldInRecalledText) {
5907
+ const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
5908
+ stages.extraction = { status: "pass", detail: impliedDetail };
5909
+ stages.index = { status: "pass", detail: impliedDetail };
5910
+ stages.retrieval = { status: "pass", detail: impliedDetail };
5911
+ stages.use = {
5912
+ status: "fail",
5913
+ detail: "Gold memory present in context but answer was incorrect"
5914
+ };
5915
+ return {
5916
+ goldMemory: goldStatement,
5917
+ label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
5918
+ stages
5919
+ };
5920
+ }
5921
+ const detail = `Best similarity ${bestSim >= 0 ? bestSim.toFixed(3) : 0} below threshold ${threshold}`;
5922
+ stages.extraction = { status: "fail", detail };
5923
+ stages.index = { status: "unavailable", detail: "not reached" };
5924
+ stages.retrieval = { status: "unavailable", detail: "not reached" };
5925
+ stages.use = { status: "unavailable", detail: "not reached" };
5926
+ return {
5927
+ goldMemory: goldStatement,
5928
+ label: { class: "extraction_miss", reason: detail },
5929
+ stages
5930
+ };
5931
+ }
5932
+ stages.extraction = {
5933
+ status: "pass",
5934
+ detail: `Matched memory ${matchedMem.id} (sim ${bestSim.toFixed(3)})`
5935
+ };
5936
+ } else {
5937
+ stages.extraction = {
5938
+ status: "unavailable",
5939
+ detail: extractionErrorDetail ?? "listMemories unavailable"
5940
+ };
5941
+ }
5942
+ const matchedMemoryId = matchedMem ? matchedMem.id : void 0;
5943
+ const recallLimit = env.recallLimit;
5944
+ const replayLimit = env.replayLimit ?? Math.max(25, recallLimit * 5);
5945
+ let indexCheckPassed = false;
5946
+ let indexCheckFailed = false;
5947
+ if (typeof env.oracleSearch === "function" && extractionRan) {
5948
+ try {
5949
+ const oracleResults = await env.oracleSearch(goldStatement, replayLimit);
5950
+ const idMatched = matchedMemoryId ? oracleResults.some((r) => r.id === matchedMemoryId) : false;
5951
+ if (idMatched) {
5952
+ indexCheckPassed = true;
5953
+ } else {
5954
+ const memMap = new Map(memories.map((m) => [m.id, m]));
5955
+ indexCheckPassed = oracleResults.some((r) => {
5956
+ const mem = memMap.get(r.id);
5957
+ return mem ? simFn(goldStatement, mem.content) >= threshold : false;
5958
+ });
5959
+ }
5960
+ if (indexCheckPassed) {
5961
+ stages.index = { status: "pass", detail: "Found in oracle search" };
5962
+ } else {
5963
+ indexCheckFailed = true;
5964
+ stages.index = { status: "fail", detail: "Not found in oracle search" };
5965
+ }
5966
+ } catch {
5967
+ stages.index = { status: "unavailable", detail: "oracleSearch threw error" };
5968
+ }
5969
+ } else if (typeof env.oracleSearch === "function") {
5970
+ stages.index = {
5971
+ status: "unavailable",
5972
+ detail: "extraction check unavailable; oracle result would be ambiguous"
5973
+ };
5974
+ } else {
5975
+ stages.index = { status: "unavailable", detail: "index check unavailable" };
5976
+ }
5977
+ let retrievalCheckPassed = false;
5978
+ let retrievalStageMiss = void 0;
5979
+ if (typeof env.recall === "function") {
5980
+ try {
5981
+ const recallResults = await env.recall(question, recallLimit);
5982
+ const isGoldInRecall = recallResults.some(
5983
+ (m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
5984
+ );
5985
+ if (isGoldInRecall) {
5986
+ retrievalCheckPassed = true;
5987
+ stages.retrieval = { status: "pass", detail: `Recalled within recallLimit ${recallLimit}` };
5988
+ } else {
5989
+ const replayResults = await env.recall(question, replayLimit);
5990
+ const replayIndex = replayResults.findIndex(
5991
+ (m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
5992
+ );
5993
+ if (replayIndex >= 0) {
5994
+ const rank = replayIndex + 1;
5995
+ retrievalStageMiss = "cap";
5996
+ stages.retrieval = {
5997
+ status: "fail",
5998
+ detail: `Rank ${rank} exceeds recallLimit ${recallLimit}`
5999
+ };
6000
+ } else {
6001
+ retrievalStageMiss = "unknown";
6002
+ stages.retrieval = {
6003
+ status: "fail",
6004
+ detail: `absent from recall at replayLimit ${replayLimit}; filter vs rank indistinguishable without candidate-stage evidence`
6005
+ };
6006
+ }
6007
+ }
6008
+ } catch {
6009
+ stages.retrieval = { status: "unavailable", detail: "recall threw error" };
6010
+ }
6011
+ } else {
6012
+ stages.retrieval = { status: "unavailable", detail: "retrieval check unavailable" };
6013
+ }
6014
+ if (!retrievalCheckPassed && goldInRecalledText) {
6015
+ retrievalCheckPassed = true;
6016
+ stages.retrieval = { status: "pass", detail: "Found in recalledText context" };
6017
+ }
6018
+ if (retrievalCheckPassed) {
6019
+ if (indexCheckFailed) {
6020
+ stages.index = { status: "pass", detail: "implied pass from retrieval (oracle query missed)" };
6021
+ indexCheckPassed = true;
6022
+ } else if (stages.index.status === "unavailable") {
6023
+ stages.index = { status: "pass", detail: "implied pass from retrieval" };
6024
+ indexCheckPassed = true;
6025
+ }
6026
+ }
6027
+ if (retrievalCheckPassed) {
6028
+ stages.use = {
6029
+ status: "fail",
6030
+ detail: "Gold memory present in context but answer was incorrect"
6031
+ };
6032
+ return {
6033
+ goldMemory: goldStatement,
6034
+ label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
6035
+ stages
6036
+ };
6037
+ }
6038
+ if (stages.extraction.status === "pass" && indexCheckFailed) {
6039
+ stages.use = { status: "unavailable", detail: "not reached" };
6040
+ return {
6041
+ goldMemory: goldStatement,
6042
+ label: { class: "index_miss", reason: "Gold statement missing from search index" },
6043
+ stages
6044
+ };
6045
+ }
6046
+ if (stages.extraction.status === "pass" && stages.index.status === "pass" && stages.retrieval.status === "fail") {
6047
+ stages.use = { status: "unavailable", detail: "not reached" };
6048
+ return {
6049
+ goldMemory: goldStatement,
6050
+ label: {
6051
+ class: "retrieval_miss",
6052
+ retrievalStage: retrievalStageMiss ?? "unknown",
6053
+ reason: stages.retrieval.detail
6054
+ },
6055
+ stages
6056
+ };
6057
+ }
6058
+ const missingReason = stages.extraction.status === "unavailable" ? `extraction check unavailable (${stages.extraction.detail})` : stages.index.status === "unavailable" && stages.retrieval.status === "unavailable" ? "index/retrieval checks unavailable in this attribution environment" : stages.index.status === "unavailable" ? "index check unavailable; a retrieval miss cannot be isolated from an index miss" : "retrieval check unavailable";
6059
+ stages.use = { status: "unavailable", detail: "not reached" };
6060
+ return {
6061
+ goldMemory: goldStatement,
6062
+ label: { class: "unattributed", reason: missingReason },
6063
+ stages
6064
+ };
6065
+ }
6066
+ function attributeGoldMemoryFromWitness(goldStatement, goldWitness, retrievals, witnessThreshold, options, recalledText) {
6067
+ const threshold = options.threshold ?? DEFAULT_ATTRIBUTION_THRESHOLD;
6068
+ if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
6069
+ throw new RangeError("attribution threshold must be a finite number between 0 and 1");
6070
+ }
6071
+ const similarity = options.similarity ?? lexicalSimilarity;
6072
+ const stages = {
6073
+ extraction: { status: "unavailable" },
6074
+ index: { status: "unavailable" },
6075
+ retrieval: { status: "unavailable" },
6076
+ use: { status: "unavailable" }
6077
+ };
6078
+ if (typeof recalledText === "string" && similarity(goldStatement, recalledText) >= threshold) {
6079
+ const detail = "implied pass from recalled context";
6080
+ stages.extraction = { status: "pass", detail };
6081
+ stages.index = { status: "pass", detail };
6082
+ stages.retrieval = { status: "pass", detail: "Found in recalledText context" };
6083
+ stages.use = { status: "fail", detail: "Gold memory present in context but answer was incorrect" };
6084
+ return {
6085
+ goldMemory: goldStatement,
6086
+ label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
6087
+ stages
6088
+ };
6089
+ }
6090
+ if (options.similarity || threshold !== witnessThreshold) {
6091
+ const detail = "stored extraction witness uses a different similarity policy";
6092
+ stages.extraction = { status: "unavailable", detail };
6093
+ stages.index = { status: "unavailable", detail: "extraction check unavailable" };
6094
+ stages.retrieval = { status: "unavailable", detail: "extraction check unavailable" };
6095
+ stages.use = { status: "unavailable", detail: "not reached" };
6096
+ return {
6097
+ goldMemory: goldStatement,
6098
+ label: { class: "unattributed", reason: detail },
6099
+ stages
6100
+ };
6101
+ }
6102
+ const storeIds = goldWitness.storeMemoryIds;
6103
+ if (storeIds === null) {
6104
+ stages.extraction = { status: "unavailable", detail: "stored extraction witness unavailable" };
6105
+ stages.index = { status: "unavailable", detail: "extraction check unavailable" };
6106
+ stages.retrieval = { status: "unavailable", detail: "extraction check unavailable" };
6107
+ stages.use = { status: "unavailable", detail: "not reached" };
6108
+ return {
6109
+ goldMemory: goldStatement,
6110
+ label: { class: "unattributed", reason: "extraction check unavailable (stored witness)" },
6111
+ stages
6112
+ };
6113
+ }
6114
+ if (storeIds.length === 0) {
6115
+ const detail = "stored witness found no matching memory";
6116
+ stages.extraction = { status: "fail", detail };
6117
+ stages.index = { status: "unavailable", detail: "not reached" };
6118
+ stages.retrieval = { status: "unavailable", detail: "not reached" };
6119
+ stages.use = { status: "unavailable", detail: "not reached" };
6120
+ return {
6121
+ goldMemory: goldStatement,
6122
+ label: { class: "extraction_miss", reason: detail },
6123
+ stages
6124
+ };
6125
+ }
6126
+ const storeIdSet = new Set(storeIds);
6127
+ stages.extraction = {
6128
+ status: "pass",
6129
+ detail: `Stored witness matched ${storeIds.length} memory id${storeIds.length === 1 ? "" : "s"}`
6130
+ };
6131
+ const oracleIds = goldWitness.oracleMemoryIds;
6132
+ let indexPassed = false;
6133
+ let indexFailed = false;
6134
+ if (oracleIds === null) {
6135
+ stages.index = { status: "unavailable", detail: "stored oracle witness unavailable" };
6136
+ } else if (oracleIds.some((id) => storeIdSet.has(id))) {
6137
+ indexPassed = true;
6138
+ stages.index = { status: "pass", detail: "Found in stored oracle witness" };
6139
+ } else {
6140
+ indexFailed = true;
6141
+ stages.index = { status: "fail", detail: "Not found in stored oracle witness" };
6142
+ }
6143
+ let appliedHit;
6144
+ let atCapUnavailable = retrievals.length === 0;
6145
+ for (const retrieval of retrievals) {
6146
+ if (retrieval.atCapMemoryIds === null) {
6147
+ atCapUnavailable = true;
6148
+ continue;
6149
+ }
6150
+ const index = retrieval.atCapMemoryIds.findIndex((id) => storeIdSet.has(id));
6151
+ if (index >= 0) {
6152
+ appliedHit = { sessionId: retrieval.sessionId, rank: index + 1 };
6153
+ break;
6154
+ }
6155
+ }
6156
+ let headroomHit;
6157
+ if (!appliedHit) {
6158
+ for (const retrieval of retrievals) {
6159
+ if (retrieval.headroomMemoryIds === null || retrieval.appliedCap === null) {
6160
+ continue;
6161
+ }
6162
+ const index = retrieval.headroomMemoryIds.findIndex((id) => storeIdSet.has(id));
6163
+ if (index >= 0) {
6164
+ headroomHit = {
6165
+ sessionId: retrieval.sessionId,
6166
+ rank: retrieval.appliedCap + index + 1,
6167
+ appliedCap: retrieval.appliedCap
6168
+ };
6169
+ break;
6170
+ }
6171
+ }
6172
+ }
6173
+ if (appliedHit) {
6174
+ stages.retrieval = {
6175
+ status: "pass",
6176
+ detail: `Found at rank ${appliedHit.rank} in stored session ${appliedHit.sessionId}`
6177
+ };
6178
+ if (!indexPassed) {
6179
+ indexPassed = true;
6180
+ indexFailed = false;
6181
+ stages.index = { status: "pass", detail: "implied pass from retrieval" };
6182
+ }
6183
+ stages.use = { status: "fail", detail: "Gold memory present in context but answer was incorrect" };
6184
+ return {
6185
+ goldMemory: goldStatement,
6186
+ label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
6187
+ stages
6188
+ };
6189
+ }
6190
+ let retrievalStageMiss = "unknown";
6191
+ if (atCapUnavailable) {
6192
+ stages.retrieval = { status: "unavailable", detail: "stored at-cap witness unavailable" };
6193
+ } else if (headroomHit) {
6194
+ retrievalStageMiss = "cap";
6195
+ stages.retrieval = {
6196
+ status: "fail",
6197
+ detail: `Headroom rank ${headroomHit.rank} exceeds applied cap ${headroomHit.appliedCap} in session ${headroomHit.sessionId}`
6198
+ };
6199
+ if (!indexPassed) {
6200
+ indexPassed = true;
6201
+ indexFailed = false;
6202
+ stages.index = { status: "pass", detail: "implied pass from retrieval headroom" };
6203
+ }
6204
+ } else {
6205
+ stages.retrieval = { status: "fail", detail: "absent from every stored at-cap retrieval witness" };
6206
+ }
6207
+ if (indexFailed && stages.retrieval.status === "fail") {
6208
+ stages.use = { status: "unavailable", detail: "not reached" };
6209
+ return {
6210
+ goldMemory: goldStatement,
6211
+ label: { class: "index_miss", reason: "Gold statement missing from search index" },
6212
+ stages
6213
+ };
6214
+ }
6215
+ if (indexPassed && stages.retrieval.status === "fail") {
6216
+ stages.use = { status: "unavailable", detail: "not reached" };
6217
+ return {
6218
+ goldMemory: goldStatement,
6219
+ label: {
6220
+ class: "retrieval_miss",
6221
+ retrievalStage: retrievalStageMiss,
6222
+ reason: stages.retrieval.detail
6223
+ },
6224
+ stages
6225
+ };
6226
+ }
6227
+ stages.use = { status: "unavailable", detail: "not reached" };
6228
+ const reason = stages.index.status === "unavailable" ? "index check unavailable; a retrieval miss cannot be isolated from an index miss" : "retrieval check unavailable";
6229
+ return {
6230
+ goldMemory: goldStatement,
6231
+ label: { class: "unattributed", reason },
6232
+ stages
6233
+ };
6234
+ }
6235
+ async function attributeTask(task, env, options = {}) {
6236
+ const golds = task.goldMemories ?? task.attributionWitness?.golds.map((gold) => gold.goldMemory);
6237
+ if (!golds || golds.length === 0) {
6238
+ return null;
6239
+ }
6240
+ const recalledText = typeof task.details?.recalledText === "string" ? task.details.recalledText : void 0;
6241
+ if (task.attributionWitness) {
6242
+ const goldAttributions2 = [];
6243
+ for (let index = 0; index < golds.length; index += 1) {
6244
+ const gold = golds[index];
6245
+ const goldWitness = task.attributionWitness.golds[index];
6246
+ if (!goldWitness || goldWitness.goldMemory !== gold) {
6247
+ throw new Error("attribution witness golds must match task goldMemories in length and order");
6248
+ }
6249
+ goldAttributions2.push(attributeGoldMemoryFromWitness(
6250
+ gold,
6251
+ goldWitness,
6252
+ task.attributionWitness.retrievals,
6253
+ task.attributionWitness.runtime.attributionThreshold,
6254
+ options,
6255
+ recalledText
6256
+ ));
6257
+ }
6258
+ if (task.attributionWitness.golds.length !== golds.length) {
6259
+ throw new Error("attribution witness golds must match task goldMemories in length and order");
6260
+ }
6261
+ return {
6262
+ taskId: task.taskId,
6263
+ question: task.question,
6264
+ golds: goldAttributions2,
6265
+ overall: computeOverallLabel(goldAttributions2)
6266
+ };
6267
+ }
6268
+ const memoizedEnv = withMemoizedListMemories(env);
6269
+ const goldAttributions = [];
6270
+ for (const gold of golds) {
6271
+ const attr = await attributeGoldMemory(gold, task.question, memoizedEnv, options, recalledText);
6272
+ goldAttributions.push(attr);
6273
+ }
6274
+ const overall = computeOverallLabel(goldAttributions);
6275
+ return {
6276
+ taskId: task.taskId,
6277
+ question: task.question,
6278
+ golds: goldAttributions,
6279
+ overall
6280
+ };
6281
+ }
6282
+ async function attributeRun(result, env, options = {}) {
6283
+ const runId = result.meta?.runId ?? result.meta?.id ?? "unknown-run";
6284
+ let memoizedEnv;
6285
+ const totals = {
6286
+ extraction_miss: 0,
6287
+ index_miss: 0,
6288
+ retrieval_miss: 0,
6289
+ use_miss: 0,
6290
+ unattributed: 0
6291
+ };
6292
+ const retrievalStages = {
6293
+ filter: 0,
6294
+ cap: 0,
6295
+ rank: 0,
6296
+ unknown: 0
6297
+ };
6298
+ const items = [];
6299
+ const skippedTasks = [];
6300
+ for (const task of result.results.tasks) {
6301
+ if (task.details?.benchmarkFailure && typeof task.details.benchmarkFailure === "object") {
6302
+ skippedTasks.push({
6303
+ taskId: task.taskId,
6304
+ reason: "trial execution failure (not an answer failure)"
6305
+ });
6306
+ continue;
6307
+ }
6308
+ const goldCount = task.goldMemories?.length ?? task.attributionWitness?.golds.length ?? 0;
6309
+ if (goldCount === 0) {
6310
+ skippedTasks.push({
6311
+ taskId: task.taskId,
6312
+ reason: "No goldMemories specified"
6313
+ });
6314
+ continue;
6315
+ }
6316
+ if (!isTaskFailed(task)) {
6317
+ skippedTasks.push({
6318
+ taskId: task.taskId,
6319
+ reason: "Task passed (score >= 1)"
6320
+ });
6321
+ continue;
6322
+ }
6323
+ const taskEnv = task.attributionWitness ? env : memoizedEnv ??= withMemoizedListMemories(env);
6324
+ const taskAttr = await attributeTask(task, taskEnv, options);
6325
+ if (taskAttr) {
6326
+ items.push(taskAttr);
6327
+ }
6328
+ }
6329
+ items.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
6330
+ skippedTasks.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
6331
+ for (const item of items) {
6332
+ totals[item.overall.class]++;
6333
+ if (item.overall.class === "retrieval_miss") {
6334
+ const stage = item.overall.retrievalStage ?? "unknown";
6335
+ retrievalStages[stage]++;
6336
+ }
6337
+ }
6338
+ return {
6339
+ runId,
6340
+ totals,
6341
+ retrievalStages,
6342
+ attributedTasks: items.length,
6343
+ skippedTasks,
6344
+ items
6345
+ };
6346
+ }
6347
+ function renderAttributionReportTable(report) {
6348
+ const lines = [];
6349
+ lines.push(`Attribution Report (Run: ${report.runId})`);
6350
+ lines.push(`Failed-task predicate: minimum primary answer score < 1 (scores.overall or non-diagnostic scores)`);
6351
+ lines.push(`Attributed tasks: ${report.attributedTasks}, Skipped tasks: ${report.skippedTasks.length}`);
6352
+ lines.push("");
6353
+ lines.push("Totals by Class:");
6354
+ lines.push(` extraction_miss: ${report.totals.extraction_miss}`);
6355
+ lines.push(` index_miss: ${report.totals.index_miss}`);
6356
+ lines.push(` retrieval_miss: ${report.totals.retrieval_miss}`);
6357
+ lines.push(` use_miss: ${report.totals.use_miss}`);
6358
+ lines.push(` unattributed: ${report.totals.unattributed}`);
6359
+ lines.push("");
6360
+ lines.push("Retrieval Miss Stages:");
6361
+ lines.push(` filter: ${report.retrievalStages.filter}`);
6362
+ lines.push(` cap: ${report.retrievalStages.cap}`);
6363
+ lines.push(` rank: ${report.retrievalStages.rank}`);
6364
+ lines.push(` unknown: ${report.retrievalStages.unknown}`);
6365
+ lines.push("");
6366
+ lines.push("Task Attributions:");
6367
+ if (report.items.length === 0) {
6368
+ lines.push(" (none)");
6369
+ } else {
6370
+ for (const item of report.items) {
6371
+ const stageStr = item.overall.retrievalStage ? ` (${item.overall.retrievalStage})` : "";
6372
+ const labelStr = `${item.overall.class}${stageStr}`;
6373
+ const reasonStr = item.overall.reason ? ` - ${item.overall.reason}` : "";
6374
+ lines.push(` ${item.taskId.padEnd(20)} ${labelStr.padEnd(24)}${reasonStr}`);
6375
+ }
6376
+ }
6377
+ if (report.skippedTasks.length > 0) {
6378
+ lines.push("");
6379
+ lines.push("Skipped Tasks:");
6380
+ for (const skipped of report.skippedTasks) {
6381
+ lines.push(` ${skipped.taskId.padEnd(20)} ${skipped.reason}`);
6382
+ }
6383
+ }
6384
+ return `${lines.join("\n")}
6385
+ `;
6386
+ }
6387
+ function serializeAttributionReport(report) {
6388
+ return `${JSON.stringify(report, null, 2)}
6389
+ `;
6390
+ }
6391
+
6392
+ // src/adapters/attribution-witness.ts
6393
+ async function readCoreMemories(orchestrator) {
6394
+ return [
6395
+ ...await orchestrator.storage.readAllMemories(),
6396
+ ...await orchestrator.storage.readAllColdMemories()
6397
+ ];
6398
+ }
6399
+ async function canonicalizeMemoryIds(orchestrator, results) {
6400
+ const memories = await Promise.all(
6401
+ results.map((result) => orchestrator.qmdResultResolver.readQmdResultMemory(
6402
+ result.path,
6403
+ orchestrator.storage,
6404
+ [],
6405
+ result.namespace
6406
+ ))
6407
+ );
6408
+ if (memories.some((memory) => memory === null)) return null;
6409
+ return memories.map((memory) => memory.frontmatter.id);
6410
+ }
6411
+ async function captureRecallAttribution(orchestrator, sessionId, snapshot) {
6412
+ const [atCapMemoryIds, headroomMemoryIds] = await Promise.all([
6413
+ canonicalizeMemoryIds(orchestrator, snapshot.appliedResults),
6414
+ canonicalizeMemoryIds(orchestrator, snapshot.headroomResults)
6415
+ ]);
6416
+ if (atCapMemoryIds === null || headroomMemoryIds === null) return void 0;
6417
+ return {
6418
+ sessionId,
6419
+ appliedCap: snapshot.appliedResultLimit,
6420
+ atCapMemoryIds,
6421
+ headroomMemoryIds
6422
+ };
6423
+ }
6424
+ async function captureTaskAttributionWitness(options) {
6425
+ const qmdMaxResults = options.orchestrator.config.qmdMaxResults;
6426
+ let storedMemories;
6427
+ try {
6428
+ storedMemories = await readCoreMemories(options.orchestrator);
6429
+ } catch {
6430
+ storedMemories = null;
6431
+ }
6432
+ const golds = await Promise.all(
6433
+ options.goldMemories.map(async (goldMemory) => {
6434
+ const storeMemoryIds = storedMemories === null ? null : [
6435
+ ...new Set(
6436
+ storedMemories.filter((memory) => lexicalSimilarity(goldMemory, memory.content) >= 0.6).map((memory) => memory.frontmatter.id)
6437
+ )
6438
+ ];
6439
+ let oracleMemoryIds = null;
6440
+ if (qmdMaxResults > 0) {
6441
+ try {
6442
+ const oracleResults = await options.orchestrator.searchAcrossNamespaces({
6443
+ query: goldMemory,
6444
+ maxResults: qmdMaxResults,
6445
+ mode: "search"
6446
+ });
6447
+ const canonicalIds = await canonicalizeMemoryIds(
6448
+ options.orchestrator,
6449
+ oracleResults
6450
+ );
6451
+ oracleMemoryIds = canonicalIds === null ? null : [...new Set(canonicalIds)];
6452
+ } catch {
6453
+ oracleMemoryIds = null;
6454
+ }
6455
+ }
6456
+ return { goldMemory, storeMemoryIds, oracleMemoryIds };
6457
+ })
6458
+ );
6459
+ return {
6460
+ schemaVersion: 1,
6461
+ runtime: {
6462
+ qmdCollection: options.qmdCollection,
6463
+ qmdIndex: options.qmdIndex,
6464
+ qmdMaxResults,
6465
+ attributionThreshold: DEFAULT_ATTRIBUTION_THRESHOLD
6466
+ },
6467
+ golds,
6468
+ retrievals: options.retrievals.map((retrieval) => ({
6469
+ sessionId: retrieval.sessionId,
6470
+ appliedCap: retrieval.appliedCap,
6471
+ atCapMemoryIds: retrieval.atCapMemoryIds === null ? null : [...retrieval.atCapMemoryIds],
6472
+ headroomMemoryIds: retrieval.headroomMemoryIds === null ? null : [...retrieval.headroomMemoryIds]
6473
+ }))
6474
+ };
6475
+ }
6476
+
6477
+ // src/adapters/remnic-adapter.ts
6478
+ import {
6479
+ lcmEvidenceIdentity
6480
+ } from "@remnic/core/lcm";
6481
+
6482
+ // src/adapters/remnic-recall-trace.ts
6483
+ import { createHash as createHash6 } from "crypto";
6484
+ import {
6485
+ lcmArchiveRowId
6486
+ } from "@remnic/core/lcm";
6487
+ function visibleRange(start, end, returnedChars) {
6488
+ const visibleStart = Math.min(start, returnedChars);
6489
+ return {
6490
+ visibleStart,
6491
+ visibleEnd: Math.max(visibleStart, Math.min(end, returnedChars))
6492
+ };
6493
+ }
6494
+ function projectBenchCoreCapture(snapshot) {
6495
+ return {
6496
+ snapshotId: snapshot.snapshotId,
6497
+ capturedAt: snapshot.capturedAt,
6498
+ ...snapshot.traceId === void 0 ? {} : { traceId: snapshot.traceId },
6499
+ budget: { chars: snapshot.budget.chars, used: snapshot.budget.used },
6500
+ filters: snapshot.filters.map(({ name, considered, admitted }) => ({
6501
+ name,
6502
+ considered,
6503
+ admitted
6504
+ })),
6505
+ results: snapshot.results.map((result) => {
6506
+ const scores = result.scoreDecomposition;
6507
+ return {
6508
+ memoryIdRef: {
6509
+ sha256: createHash6("sha256").update(result.memoryId, "utf8").digest("hex"),
6510
+ length: Buffer.byteLength(result.memoryId, "utf8")
6511
+ },
6512
+ servedBy: result.servedBy,
6513
+ scoreDecomposition: {
6514
+ ...typeof scores.vector === "number" ? { vector: scores.vector } : {},
6515
+ ...typeof scores.bm25 === "number" ? { bm25: scores.bm25 } : {},
6516
+ ...typeof scores.importance === "number" ? { importance: scores.importance } : {},
6517
+ ...typeof scores.mmrPenalty === "number" ? { mmrPenalty: scores.mmrPenalty } : {},
6518
+ ...typeof scores.tierPrior === "number" ? { tierPrior: scores.tierPrior } : {},
6519
+ ...typeof scores.reinforcementBoost === "number" ? { reinforcementBoost: scores.reinforcementBoost } : {},
6520
+ final: scores.final
6521
+ },
6522
+ admittedBy: [...result.admittedBy],
6523
+ ...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
6524
+ ...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
6525
+ ...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
6526
+ };
6527
+ })
6528
+ };
6529
+ }
6530
+ function createBenchRecallTraceRecorder(requestedChars) {
6531
+ const sections = [];
6532
+ const pendingSelections = [];
6533
+ const lcmCandidates = [];
6534
+ let composedChars = 0;
6535
+ let coreCapture;
6536
+ const sectionById = (sectionId) => {
6537
+ const section = sections.find((entry) => entry.id === sectionId);
6538
+ if (!section) throw new Error(`Unknown benchmark recall trace section: ${sectionId}`);
6539
+ return section;
6540
+ };
6541
+ const appendRelativeSelection = (sectionId, kind, start, end, lineageStatus, fields = {}) => {
6542
+ const section = sectionById(sectionId);
6543
+ const contentLength = section.contentEnd - section.contentStart;
6544
+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > contentLength) {
6545
+ throw new Error(
6546
+ `Invalid benchmark recall trace range for ${sectionId}: ${start}..${end}.`
6547
+ );
6548
+ }
6549
+ pendingSelections.push({
6550
+ sectionId,
6551
+ kind,
6552
+ lineageStatus,
6553
+ composedStart: section.contentStart + start,
6554
+ composedEnd: section.contentStart + end,
6555
+ ...fields
6556
+ });
6557
+ };
6558
+ return {
6559
+ appendSection(id, source, renderedLength) {
6560
+ if (!Number.isSafeInteger(renderedLength) || renderedLength < 0) {
6561
+ throw new Error("Benchmark recall trace section length must be a non-negative integer.");
6562
+ }
6563
+ if (sections.some((section) => section.id === id)) {
6564
+ throw new Error(`Duplicate benchmark recall trace section: ${id}`);
6565
+ }
6566
+ const separatorStart = composedChars;
6567
+ const contentStart = composedChars + (sections.length === 0 ? 0 : 2);
6568
+ const contentEnd = contentStart + renderedLength;
6569
+ sections.push({
6570
+ id,
6571
+ source,
6572
+ separatorStart,
6573
+ contentStart,
6574
+ contentEnd,
6575
+ composedStart: separatorStart,
6576
+ composedEnd: contentEnd,
6577
+ visibleStart: 0,
6578
+ visibleEnd: 0,
6579
+ visibleChars: 0
6580
+ });
6581
+ composedChars = contentEnd;
6582
+ },
6583
+ recordEvidenceSelections(sectionId, receipts) {
6584
+ for (const receipt of receipts) {
6585
+ appendRelativeSelection(
6586
+ sectionId,
6587
+ "evidence-block",
6588
+ receipt.blockStart,
6589
+ receipt.blockEnd,
6590
+ receipt.item.archiveRowId === void 0 ? "unavailable" : "exact",
6591
+ {
6592
+ ...receipt.item.archiveRowId === void 0 ? {} : { archiveRowIds: [receipt.item.archiveRowId] },
6593
+ ...receipt.item.turnIndex === void 0 ? {} : { turnIndex: receipt.item.turnIndex },
6594
+ ...receipt.item.role === void 0 ? {} : { role: receipt.item.role },
6595
+ ...receipt.item.score === void 0 ? {} : { score: receipt.item.score }
6596
+ }
6597
+ );
6598
+ }
6599
+ },
6600
+ recordTrajectorySelections(sectionId, receipts) {
6601
+ for (const receipt of receipts) {
6602
+ appendRelativeSelection(
6603
+ sectionId,
6604
+ "trajectory-line",
6605
+ receipt.lineStart,
6606
+ receipt.lineEnd,
6607
+ receipt.lineageStatus,
6608
+ {
6609
+ archiveRowIds: [
6610
+ ...receipt.actionArchiveRowIds,
6611
+ ...receipt.observationArchiveRowIds
6612
+ ]
6613
+ }
6614
+ );
6615
+ }
6616
+ },
6617
+ recordSummarySelections(sectionId, receipts) {
6618
+ for (const receipt of receipts) {
6619
+ appendRelativeSelection(
6620
+ sectionId,
6621
+ "lcm-summary",
6622
+ receipt.entryStart,
6623
+ receipt.entryEnd,
6624
+ "exact",
6625
+ {
6626
+ summary: {
6627
+ id: receipt.id,
6628
+ depth: receipt.depth,
6629
+ msgStart: receipt.msgStart,
6630
+ msgEnd: receipt.msgEnd
6631
+ }
6632
+ }
6633
+ );
6634
+ }
6635
+ },
6636
+ recordRawRow(sectionId, range, row) {
6637
+ const archiveRowId = lcmArchiveRowId(row);
6638
+ appendRelativeSelection(
6639
+ sectionId,
6640
+ "raw-row",
6641
+ range.start,
6642
+ range.end,
6643
+ archiveRowId === void 0 ? "unavailable" : "exact",
6644
+ {
6645
+ ...archiveRowId === void 0 ? {} : { archiveRowIds: [archiveRowId] },
6646
+ turnIndex: row.turn_index,
6647
+ role: row.role
6648
+ }
6649
+ );
6650
+ },
6651
+ recordLcmCandidate(candidate) {
6652
+ lcmCandidates.push({ ...candidate });
6653
+ },
6654
+ recordCoreCapture(snapshot) {
6655
+ coreCapture = snapshot ? projectBenchCoreCapture(snapshot) : void 0;
6656
+ },
6657
+ finalize(returnedChars) {
6658
+ const normalizedReturnedChars = Math.max(0, Math.min(returnedChars, composedChars));
6659
+ return {
6660
+ schemaVersion: 1,
6661
+ sensitivity: {
6662
+ classification: "restricted",
6663
+ contentEncoding: "sha256+length",
6664
+ containsGold: false
6665
+ },
6666
+ sections: sections.map((section) => {
6667
+ const visible = visibleRange(
6668
+ section.composedStart,
6669
+ section.composedEnd,
6670
+ normalizedReturnedChars
6671
+ );
6672
+ return {
6673
+ ...section,
6674
+ ...visible,
6675
+ visibleChars: visible.visibleEnd - visible.visibleStart
6676
+ };
6677
+ }),
6678
+ selections: pendingSelections.map((selection) => ({
6679
+ ...selection,
6680
+ ...visibleRange(
6681
+ selection.composedStart,
6682
+ selection.composedEnd,
6683
+ normalizedReturnedChars
6684
+ )
6685
+ })),
6686
+ lcmCandidates: lcmCandidates.map((candidate) => ({ ...candidate })),
6687
+ ...coreCapture === void 0 ? {} : { coreCapture },
6688
+ budget: {
6689
+ requestedChars,
6690
+ composedChars,
6691
+ returnedChars: normalizedReturnedChars,
6692
+ truncated: normalizedReturnedChars < composedChars
6693
+ }
6694
+ };
6695
+ }
6696
+ };
6697
+ }
6698
+
6699
+ // src/recall-budget.ts
6700
+ var DEFAULT_BENCH_RECALL_BUDGET_CHARS = 24e3;
6701
+ var MAX_COMBINED_RECALL_BUDGET_CHARS = 36e3;
6702
+ function benchmarkRecallBudgetForSessionCount(sessionCount) {
6703
+ if (!Number.isInteger(sessionCount) || sessionCount <= 0) {
6704
+ return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
6705
+ }
6706
+ if (sessionCount === 1) {
6707
+ return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
6708
+ }
6709
+ return Math.floor(MAX_COMBINED_RECALL_BUDGET_CHARS / sessionCount);
6710
+ }
6711
+
6712
+ // src/adapters/remnic-adapter.ts
6713
+ var DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE = 0.34;
6714
+ var ANSWER_SUPPORT_STOP_WORDS = /* @__PURE__ */ new Set([
6715
+ "about",
6716
+ "after",
6717
+ "again",
6718
+ "also",
6719
+ "answer",
6720
+ "before",
6721
+ "being",
6722
+ "could",
6723
+ "does",
6724
+ "from",
6725
+ "have",
6726
+ "information",
6727
+ "into",
6728
+ "just",
6729
+ "know",
6730
+ "memory",
6731
+ "might",
6732
+ "please",
6733
+ "question",
6734
+ "recall",
6735
+ "remember",
6736
+ "should",
6737
+ "that",
6738
+ "their",
6739
+ "there",
6740
+ "these",
6741
+ "they",
6742
+ "this",
6743
+ "those",
6744
+ "user",
6745
+ "using",
6746
+ "what",
6747
+ "when",
6748
+ "where",
6749
+ "which",
6750
+ "while",
6751
+ "with",
6752
+ "would",
6753
+ "your"
6754
+ ]);
6755
+ function resolveAnswerSupportMinCoverage(config) {
6756
+ const raw = config?.answerSupportMinCoverage;
6757
+ if (raw === void 0) return DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE;
6758
+ const parsed = typeof raw === "number" ? raw : Number(raw);
6759
+ if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) {
6760
+ throw new Error("answerSupportMinCoverage must be a finite number greater than 0 and at most 1.");
6761
+ }
6762
+ return parsed;
6763
+ }
6764
+ function resolveSkipExtractionLcmFirst(config) {
6765
+ const raw = config?.skipExtractionLcmFirst;
6766
+ if (raw === void 0) return true;
6767
+ if (typeof raw === "boolean") return raw;
6768
+ if (typeof raw === "string") {
6769
+ const normalized = raw.trim().toLowerCase();
6770
+ if (["true", "1", "yes", "on"].includes(normalized)) return true;
6771
+ if (["false", "0", "no", "off"].includes(normalized)) return false;
6772
+ }
6773
+ throw new Error(
6774
+ "skipExtractionLcmFirst must be a boolean or one of true/false, 1/0, yes/no, on/off."
6775
+ );
6776
+ }
6777
+ function shouldIncludeCoreRecallForReplay(options) {
6778
+ return options.useCoreMemoryPipeline && (options.replayExtractionMode !== "skip" || !options.skipExtractionLcmFirst);
6779
+ }
6780
+ function normalizeSupportToken(value) {
6781
+ if (value.length > 5 && value.endsWith("ing")) return value.slice(0, -3);
6782
+ if (value.length > 4 && value.endsWith("ed")) {
6783
+ const base = value.slice(0, -2);
6784
+ return /[vs]$/.test(base) ? `${base}e` : base;
6785
+ }
6786
+ if (value.length > 4 && value.endsWith("es")) return value.slice(0, -2);
6787
+ if (value.length > 3 && value.endsWith("s")) return value.slice(0, -1);
6788
+ return value;
6789
+ }
6790
+ function supportTerms(value) {
6791
+ return [...new Set(
6792
+ (value.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}_-]{2,}/gu) ?? []).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term)).map(normalizeSupportToken).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term) && !/^\d+$/.test(term))
6793
+ )];
6794
+ }
6795
+ function exactContextEvidenceLines(recalledText) {
6796
+ return recalledText.split(/\r?\n/).map((line) => line.trim()).filter((line) => {
6797
+ if (!line || /^#{1,6}\s/.test(line)) return false;
6798
+ return !/^(?:answer guidance:|distinct user-stated targets found:|no (?:direct|historically valid)|these direct temporal statements|this is the most recent|use this list|when answering)/i.test(line);
6799
+ });
6800
+ }
6801
+ function assessRemnicRecallSupport(request, supportThreshold = DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE) {
6802
+ if (request.recalledText.trim().length === 0) {
6803
+ return { status: "empty", reason: "exact responder context is empty", evidenceCount: 0 };
6804
+ }
6805
+ const queryTerms = supportTerms(request.query);
6806
+ if (queryTerms.length < 2) {
6807
+ return {
6808
+ status: "unavailable",
6809
+ reason: "query has fewer than two distinctive terms for conservative support scoring"
6810
+ };
6811
+ }
6812
+ const evidenceLines = exactContextEvidenceLines(request.recalledText);
6813
+ const evidenceTermSets = evidenceLines.map((line) => new Set(supportTerms(line)));
6814
+ const matchedTerms = queryTerms.filter(
5966
6815
  (term) => evidenceTermSets.some((terms) => terms.has(term))
5967
6816
  );
5968
6817
  const evidenceCount = evidenceTermSets.filter(
@@ -6859,6 +7708,14 @@ function pruneBenchEntityStructuredFacts(structuredSections, targetSources, prot
6859
7708
  return { changed, structuredSections: nextSections };
6860
7709
  }
6861
7710
  async function clearBenchCoreEntitiesForSession(orchestrator, sessionId) {
7711
+ const baseDir = orchestrator.storage.dir;
7712
+ await withRawEntityPageMutation(
7713
+ baseDir,
7714
+ path5.join(baseDir, "entities", ".bench-session-cleanup.md"),
7715
+ async () => clearBenchCoreEntitiesForSessionUnlocked(orchestrator, sessionId)
7716
+ );
7717
+ }
7718
+ async function clearBenchCoreEntitiesForSessionUnlocked(orchestrator, sessionId) {
6862
7719
  const storage = orchestrator.storage;
6863
7720
  const entityStorage = storage;
6864
7721
  const entitySchemas = entityStorage.entitySchemas;
@@ -7328,7 +8185,7 @@ function createAdapterFactory(mode) {
7328
8185
  throw error;
7329
8186
  }
7330
8187
  },
7331
- async [composeRecall](sessionId, query, budgetChars, recallOptions = {}, control, traceRecorder) {
8188
+ async [composeRecall](sessionId, query, budgetChars, recallOptions = {}, control, traceRecorder, attributionSink) {
7332
8189
  throwIfBenchPhaseAborted(control, "recall");
7333
8190
  const waitForRecall = (promise) => withBenchPhaseAbort(promise, control, "recall");
7334
8191
  sessionId = normalizeBenchSessionId(sessionId);
@@ -7480,8 +8337,15 @@ function createAdapterFactory(mode) {
7480
8337
  control,
7481
8338
  "recall",
7482
8339
  { waitForCompletionOnAbort: true }
7483
- ).then((capture) => {
8340
+ ).then(async (capture) => {
7484
8341
  traceRecorder.recordCoreCapture(capture.snapshot);
8342
+ attributionSink?.(
8343
+ capture.snapshot ? await captureRecallAttribution(
8344
+ state.orchestrator,
8345
+ sessionId,
8346
+ capture.snapshot
8347
+ ).catch(() => void 0) : void 0
8348
+ );
7485
8349
  return capture.result;
7486
8350
  }) : await waitForRecall(
7487
8351
  state.orchestrator.recall(query, sessionId, coreOptions)
@@ -7784,15 +8648,32 @@ ${coreRecall.trim()}`;
7784
8648
  async recallWithTrace(sessionId, query, budgetChars, recallOptions = {}, control) {
7785
8649
  const budget = budgetChars ?? DEFAULT_BENCH_RECALL_BUDGET_CHARS;
7786
8650
  const traceRecorder = createBenchRecallTraceRecorder(Math.max(0, budget));
8651
+ let attribution;
7787
8652
  const text = await adapter[composeRecall](
7788
8653
  sessionId,
7789
8654
  query,
7790
8655
  budgetChars,
7791
8656
  recallOptions,
7792
8657
  control,
7793
- traceRecorder
8658
+ traceRecorder,
8659
+ (captured) => {
8660
+ attribution = captured;
8661
+ }
7794
8662
  );
7795
- return { text, trace: traceRecorder.finalize(text.length) };
8663
+ return {
8664
+ text,
8665
+ trace: traceRecorder.finalize(text.length),
8666
+ ...attribution ? { attribution } : {}
8667
+ };
8668
+ },
8669
+ async captureAttributionWitness(request) {
8670
+ return captureTaskAttributionWitness({
8671
+ orchestrator: state.orchestrator,
8672
+ qmdCollection: state.orchestrator.config.qmdCollection,
8673
+ qmdIndex: state.qmdSandbox.indexName,
8674
+ goldMemories: request.goldMemories,
8675
+ retrievals: request.retrievals
8676
+ });
7796
8677
  },
7797
8678
  async assessRecallSupport(request, control) {
7798
8679
  throwIfBenchPhaseAborted(control, "assessRecallSupport");
@@ -21704,8 +22585,15 @@ function appendCompletedTask(ctx, tasks, pendingPairedAnswerReplays, task) {
21704
22585
  }
21705
22586
  async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate, pendingPairedAnswerReplays) {
21706
22587
  const trialId = trial.taskId ?? trial.question.slice(0, 60);
22588
+ const attributionCapture = {};
21707
22589
  try {
21708
- return await executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerReplays);
22590
+ return await executeTrial(
22591
+ ctx,
22592
+ trial,
22593
+ answerSupportGate,
22594
+ pendingPairedAnswerReplays,
22595
+ attributionCapture
22596
+ );
21709
22597
  } catch (err) {
21710
22598
  const blocked = findBenchmarkRunBlockedError(err);
21711
22599
  if (blocked) {
@@ -21722,6 +22610,7 @@ async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate,
21722
22610
  latencyMs: 0,
21723
22611
  tokens: { input: 0, output: 0 },
21724
22612
  ...trial.goldMemories ? { goldMemories: trial.goldMemories } : {},
22613
+ ...attributionCapture.witness ? { attributionWitness: attributionCapture.witness } : {},
21725
22614
  details: {
21726
22615
  // Preserve the trial's category so a failed trial is still attributed
21727
22616
  // to its per-category bucket (computeCategoryAggregates), keeping the
@@ -21872,13 +22761,38 @@ function resolveResponderIdentity(responder) {
21872
22761
  const trimmed = typeof raw === "string" ? raw.trim() : "";
21873
22762
  return trimmed.length > 0 ? trimmed : null;
21874
22763
  }
21875
- async function executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerReplays) {
22764
+ async function executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerReplays, attributionCapture) {
21876
22765
  const { result: recallResult, durationMs } = await timed(async () => {
21877
22766
  const recallBudget = benchmarkRecallBudgetForSessionCount(trial.recallSessionIds.length);
22767
+ const witnessEnabled = (trial.goldMemories?.length ?? 0) > 0 && typeof ctx.options.system.recallWithTrace === "function" && typeof ctx.options.system.captureAttributionWitness === "function";
21878
22768
  const recalledSessions = await Promise.all(
21879
- trial.recallSessionIds.map((sessionId) => ctx.options.system.recall(sessionId, trial.question, recallBudget))
22769
+ trial.recallSessionIds.map(async (sessionId) => {
22770
+ if (!witnessEnabled) {
22771
+ return {
22772
+ text: await ctx.options.system.recall(sessionId, trial.question, recallBudget)
22773
+ };
22774
+ }
22775
+ const traced = await ctx.options.system.recallWithTrace(
22776
+ sessionId,
22777
+ trial.question,
22778
+ recallBudget
22779
+ );
22780
+ const attribution = traced.attribution ?? {
22781
+ sessionId,
22782
+ appliedCap: null,
22783
+ atCapMemoryIds: null,
22784
+ headroomMemoryIds: null
22785
+ };
22786
+ return { text: traced.text, attribution };
22787
+ })
21880
22788
  );
21881
- const rawRecalledText = recalledSessions.filter(Boolean).join("\n\n");
22789
+ if (witnessEnabled) {
22790
+ attributionCapture.witness = await ctx.options.system.captureAttributionWitness({
22791
+ goldMemories: trial.goldMemories,
22792
+ retrievals: recalledSessions.map((session) => session.attribution)
22793
+ }).catch(() => void 0);
22794
+ }
22795
+ const rawRecalledText = recalledSessions.map((session) => session.text).filter(Boolean).join("\n\n");
21882
22796
  const recalledText2 = trial.recallTextTransform ? trial.recallTextTransform({
21883
22797
  question: trial.question,
21884
22798
  recalledText: rawRecalledText
@@ -22007,6 +22921,7 @@ async function executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerRe
22007
22921
  output: answered.tokens.output + judgeResult.tokens.output
22008
22922
  },
22009
22923
  ...trial.goldMemories ? { goldMemories: trial.goldMemories } : {},
22924
+ ...attributionCapture.witness ? { attributionWitness: attributionCapture.witness } : {},
22010
22925
  details
22011
22926
  };
22012
22927
  if (answerReplayKey && currentProfile === "baseline" && answered.fallbackReason === void 0) {
@@ -45797,771 +46712,268 @@ async function runCodingGraphBenchmark(config = {}) {
45797
46712
  throw new Error(
45798
46713
  `search_graph failed: ${searchRes.result.code}`
45799
46714
  );
45800
- }
45801
- searchSamples.push(searchRes.ms);
45802
- }
45803
- const deadCode = timeSync(() => store.deadCode());
45804
- if (!deadCode.result.ok) {
45805
- throw new Error(`dead_code failed: ${deadCode.result.code}`);
45806
- }
45807
- await store.drain();
45808
- let dbBytes = statSync(dbPath).size;
45809
- try {
45810
- dbBytes += statSync(dbPath + "-wal").size;
45811
- } catch {
45812
- }
45813
- const kloc = Math.max(1, repo.approximateLoc / 1e3);
45814
- const dbBytesPerKloc = dbBytes / kloc;
45815
- const modifiedSamples = [];
45816
- for (let i = 0; i < iterations; i++) {
45817
- const fileIdx = i % storeFiles.length;
45818
- const original = storeFiles[fileIdx];
45819
- if (!original) continue;
45820
- const ownSyms = new Set(original.symbols.map((sym) => sym.qualifiedName));
45821
- const churnName = `mod.benchChurnSymbol${i}`;
45822
- const representativeEdges = (original.edges ?? []).filter((e) => !ownSyms.has(e.dstQualifiedName)).slice(0, 2).map((e) => ({ ...e, srcQualifiedName: churnName }));
45823
- const modified = {
45824
- ...original,
45825
- contentHash: `${original.contentHash}-mod-${i}`,
45826
- symbols: [
45827
- {
45828
- qualifiedName: churnName,
45829
- name: `benchChurnSymbol${i}`,
45830
- kind: "function",
45831
- span: { startByte: 0, endByte: 0 }
45832
- }
45833
- ],
45834
- edges: representativeEdges
45835
- };
45836
- const modResult = await timeAsync(
45837
- () => store.upsertFileBatch([modified])
45838
- );
45839
- if (!modResult.result.ok) {
45840
- throw new Error(`modified update failed: ${modResult.result.code}`);
45841
- }
45842
- modifiedSamples.push(modResult.ms);
45843
- const restoreResult = await store.upsertFileBatch(storeFiles);
45844
- if (!restoreResult.ok) {
45845
- throw new Error(`restore failed: ${restoreResult.code}`);
45846
- }
45847
- sampleRss();
45848
- }
45849
- sampleRss();
45850
- const peakRssBytes = peakRss;
45851
- return {
45852
- schemaVersion: CODING_GRAPH_BENCH_SCHEMA_VERSION,
45853
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
45854
- machine: captureMachineFingerprint(),
45855
- fixture: {
45856
- config: fixtureConfig,
45857
- approximateLoc: repo.approximateLoc,
45858
- fileCount: repo.files.length,
45859
- symbolCount: repo.files.reduce((sum, f) => sum + f.symbols.length, 0),
45860
- edgeCount: repo.files.reduce((sum, f) => sum + f.edges.length, 0)
45861
- },
45862
- fullIndexMs: { ms: fullIndexMsValue },
45863
- fullIndexLocsPerSecond: Math.round(locsPerSecond),
45864
- incrementalUpdate: computeMicroMetric(incrementalSamples),
45865
- incrementalModifiedUpdate: computeMicroMetric(
45866
- modifiedSamples.length > 0 ? modifiedSamples : [0]
45867
- ),
45868
- tracePath: computeMicroMetric(traceSamples.length > 0 ? traceSamples : [0]),
45869
- searchGraph: computeMicroMetric(searchSamples),
45870
- deadCodeMs: { ms: deadCode.ms },
45871
- dbBytesPerKloc: Math.round(dbBytesPerKloc),
45872
- peakRssBytes,
45873
- dbBytes,
45874
- graphNodeCount,
45875
- graphEdgeCount
45876
- };
45877
- } finally {
45878
- await store.close();
45879
- }
45880
- } finally {
45881
- await rm15(dir, { recursive: true, force: true });
45882
- }
45883
- }
45884
-
45885
- // src/coding-graph/regression.ts
45886
- var METRIC_DIRECTION = {
45887
- fullIndexMs: "lower-is-better",
45888
- fullIndexLocsPerSecond: "higher-is-better",
45889
- incrementalUpdateP95Ms: "lower-is-better",
45890
- incrementalUpdateP50Ms: "lower-is-better",
45891
- incrementalModifiedUpdateP95Ms: "lower-is-better",
45892
- incrementalModifiedUpdateP50Ms: "lower-is-better",
45893
- tracePathP95Ms: "lower-is-better",
45894
- searchGraphP95Ms: "lower-is-better",
45895
- deadCodeMs: "lower-is-better",
45896
- dbBytesPerKloc: "lower-is-better"
45897
- };
45898
- function extractMetrics(report) {
45899
- return {
45900
- fullIndexMs: report.fullIndexMs.ms,
45901
- fullIndexLocsPerSecond: report.fullIndexLocsPerSecond,
45902
- incrementalUpdateP50Ms: report.incrementalUpdate.p50,
45903
- incrementalUpdateP95Ms: report.incrementalUpdate.p95,
45904
- incrementalModifiedUpdateP50Ms: report.incrementalModifiedUpdate.p50,
45905
- incrementalModifiedUpdateP95Ms: report.incrementalModifiedUpdate.p95,
45906
- tracePathP95Ms: report.tracePath.p95,
45907
- searchGraphP95Ms: report.searchGraph.p95,
45908
- deadCodeMs: report.deadCodeMs.ms,
45909
- dbBytesPerKloc: report.dbBytesPerKloc
45910
- };
45911
- }
45912
- var NODE_MAJOR_CACHE = /* @__PURE__ */ new WeakMap();
45913
- function nodeMajor(fp) {
45914
- const cached = NODE_MAJOR_CACHE.get(fp);
45915
- if (cached !== void 0) return cached;
45916
- const major = fp.nodeVersion.replace(/^v/, "").split(".")[0] ?? fp.nodeVersion;
45917
- NODE_MAJOR_CACHE.set(fp, major);
45918
- return major;
45919
- }
45920
- function compareMachineFingerprints(report, baseline) {
45921
- const differing = [];
45922
- if (report.arch !== baseline.arch) differing.push("arch");
45923
- if (report.platform !== baseline.platform) differing.push("platform");
45924
- if (nodeMajor(report) !== nodeMajor(baseline)) differing.push("nodeVersion(major)");
45925
- if (report.cpuModel !== null && baseline.cpuModel !== null && report.cpuModel !== baseline.cpuModel) {
45926
- differing.push("cpuModel");
45927
- }
45928
- if (report.cpuCores !== baseline.cpuCores) differing.push("cpuCores");
45929
- return { differingFields: differing };
45930
- }
45931
- function invalidFingerprintFields(fp) {
45932
- const bad = [];
45933
- if (typeof fp.arch !== "string") bad.push("arch");
45934
- if (typeof fp.platform !== "string") bad.push("platform");
45935
- if (typeof fp.nodeVersion !== "string") bad.push("nodeVersion");
45936
- if (typeof fp.cpuCores !== "number" || !Number.isFinite(fp.cpuCores)) bad.push("cpuCores");
45937
- if (fp.cpuModel !== null && typeof fp.cpuModel !== "string") bad.push("cpuModel");
45938
- return bad;
45939
- }
45940
- function checkCodingGraphRegression(report, baseline, tolerancePercent = DEFAULT_TOLERANCE_PERCENT) {
45941
- const reportFixture = report.fixture.config;
45942
- const baselineFixture = baseline.fixtureConfig;
45943
- const mismatchedKeys = Object.keys(baselineFixture).filter((key) => reportFixture[key] !== baselineFixture[key]);
45944
- if (mismatchedKeys.length > 0) {
45945
- const diffs = mismatchedKeys.map(
45946
- (key) => `${key}: report=${reportFixture[key]} baseline=${baselineFixture[key]}`
45947
- ).join(", ");
45948
- return {
45949
- passed: false,
45950
- regressions: [],
45951
- summary: `Fixture mismatch (${diffs}). Metrics are not comparable across different fixtures.`
45952
- };
45953
- }
45954
- if (report.schemaVersion !== baseline.schemaVersion) {
45955
- return {
45956
- passed: false,
45957
- regressions: [],
45958
- summary: `Schema-version mismatch: report=${report.schemaVersion} baseline=${baseline.schemaVersion}. Metrics are not comparable across schema versions \u2014 regenerate the baseline (#1688).`
45959
- };
45960
- }
45961
- const metricChecks = [
45962
- ["incrementalUpdate.p50", report.incrementalUpdate?.p50],
45963
- ["incrementalUpdate.p95", report.incrementalUpdate?.p95],
45964
- ["incrementalModifiedUpdate.p50", report.incrementalModifiedUpdate?.p50],
45965
- ["incrementalModifiedUpdate.p95", report.incrementalModifiedUpdate?.p95],
45966
- ["tracePath.p95", report.tracePath?.p95],
45967
- ["searchGraph.p95", report.searchGraph?.p95],
45968
- ["fullIndexMs.ms", report.fullIndexMs?.ms],
45969
- ["fullIndexLocsPerSecond", report.fullIndexLocsPerSecond],
45970
- ["deadCodeMs.ms", report.deadCodeMs?.ms],
45971
- ["dbBytesPerKloc", report.dbBytesPerKloc]
45972
- ];
45973
- const missingFields = metricChecks.filter(([, v]) => typeof v !== "number" || !Number.isFinite(v)).map(([k]) => k);
45974
- if (missingFields.length > 0) {
45975
- return {
45976
- passed: false,
45977
- regressions: [],
45978
- summary: "Report claims schemaVersion " + report.schemaVersion + " but is missing required metric field(s): " + missingFields.join(", ") + " \u2014 the report is incomplete or corrupt. Regenerate it (#1688)."
45979
- };
45980
- }
45981
- const badBaselineMetrics = Object.keys(METRIC_DIRECTION).filter((key) => {
45982
- const v = baseline.metrics[key];
45983
- return v != null && (typeof v !== "number" || !Number.isFinite(v));
45984
- });
45985
- if (badBaselineMetrics.length > 0) {
45986
- return {
45987
- passed: false,
45988
- regressions: [],
45989
- summary: "Baseline has a non-numeric required metric field(s): " + badBaselineMetrics.join(", ") + " \u2014 the baseline is corrupt. Regenerate it (#1688)."
45990
- };
45991
- }
45992
- const reportFpBad = report.machine == null ? ["<missing>"] : invalidFingerprintFields(report.machine);
45993
- const baselineFpBad = baseline.machine == null ? ["<missing>"] : invalidFingerprintFields(baseline.machine);
45994
- if (reportFpBad.length > 0 || baselineFpBad.length > 0) {
45995
- const which = [];
45996
- if (reportFpBad.length > 0) which.push("report (" + reportFpBad.join(", ") + ")");
45997
- if (baselineFpBad.length > 0) which.push("baseline (" + baselineFpBad.join(", ") + ")");
45998
- return {
45999
- passed: false,
46000
- regressions: [],
46001
- summary: "Report or baseline machine fingerprint is missing or has an invalid field type(s): " + which.join("; ") + " \u2014 the artifact is incomplete or corrupt. Regenerate it (#1688)."
46002
- };
46003
- }
46004
- const mismatch = compareMachineFingerprints(report.machine, baseline.machine);
46005
- if (mismatch.differingFields.length > 0) {
46006
- return {
46007
- passed: true,
46008
- skipped: true,
46009
- regressions: [],
46010
- summary: "Machine-fingerprint mismatch \u2014 comparison skipped to avoid a false-positive hardware-variance failure. Differing fields: " + mismatch.differingFields.join(", ") + ". Regenerate the baseline on this machine for a real comparison (#1688).",
46011
- machineMismatch: {
46012
- report: report.machine,
46013
- baseline: baseline.machine,
46014
- differingFields: mismatch.differingFields
46015
- }
46016
- };
46017
- }
46018
- const measured = extractMetrics(report);
46019
- const baselineMetrics = baseline.metrics;
46020
- const regressions = [];
46021
- for (const key of Object.keys(METRIC_DIRECTION)) {
46022
- const baseVal = baselineMetrics[key];
46023
- const measVal = measured[key];
46024
- if (baseVal == null || measVal == null) continue;
46025
- if (baseVal === 0) continue;
46026
- const direction = METRIC_DIRECTION[key];
46027
- const ratio2 = measVal / baseVal;
46028
- const percentChange2 = direction === "lower-is-better" ? (ratio2 - 1) * 100 : (1 - ratio2) * 100;
46029
- const regressed = percentChange2 > tolerancePercent;
46030
- if (regressed) {
46031
- regressions.push({
46032
- key,
46033
- baseline: baseVal,
46034
- measured: measVal,
46035
- percentChange: Math.round(percentChange2 * 10) / 10,
46036
- direction,
46037
- tolerancePercent,
46038
- regressed: true
46039
- });
46040
- }
46041
- }
46042
- const passed = regressions.length === 0;
46043
- const summary = passed ? "All metrics within tolerance." : `${regressions.length} metric(s) regressed beyond ${tolerancePercent}% tolerance:
46044
- ` + regressions.map(
46045
- (r) => ` ${r.key}: ${r.baseline} \u2192 ${r.measured} (${r.percentChange > 0 ? "+" : ""}${r.percentChange}% vs baseline)`
46046
- ).join("\n");
46047
- return { passed, regressions, summary };
46048
- }
46049
- function buildBaselineFromReport(report, note) {
46050
- return {
46051
- schemaVersion: report.schemaVersion,
46052
- machine: report.machine,
46053
- fixtureConfig: report.fixture.config,
46054
- metrics: extractMetrics(report),
46055
- createdAt: report.timestamp,
46056
- note
46057
- };
46058
- }
46059
-
46060
- // src/attribution.ts
46061
- var DEFAULT_STOPWORDS = /* @__PURE__ */ new Set([
46062
- "a",
46063
- "an",
46064
- "the",
46065
- "in",
46066
- "on",
46067
- "at",
46068
- "to",
46069
- "for",
46070
- "of",
46071
- "with",
46072
- "by",
46073
- "from",
46074
- "up",
46075
- "about",
46076
- "into",
46077
- "through",
46078
- "during",
46079
- "before",
46080
- "after",
46081
- "above",
46082
- "below",
46083
- "and",
46084
- "or",
46085
- "but",
46086
- "if",
46087
- "then",
46088
- "else",
46089
- "when",
46090
- "where",
46091
- "why",
46092
- "how",
46093
- "all",
46094
- "any",
46095
- "both",
46096
- "each",
46097
- "few",
46098
- "more",
46099
- "most",
46100
- "other",
46101
- "some",
46102
- "such",
46103
- "no",
46104
- "nor",
46105
- "not",
46106
- "only",
46107
- "own",
46108
- "same",
46109
- "so",
46110
- "than",
46111
- "too",
46112
- "very",
46113
- "this",
46114
- "that",
46115
- "these",
46116
- "those",
46117
- "it",
46118
- "its",
46119
- "is",
46120
- "are",
46121
- "was",
46122
- "were",
46123
- "be",
46124
- "been",
46125
- "being",
46126
- "have",
46127
- "has",
46128
- "had",
46129
- "do",
46130
- "does",
46131
- "did"
46132
- ]);
46133
- function extractContentWords(text) {
46134
- const cleaned = text.toLowerCase().replace(/[^a-z0-9\s]/g, " ");
46135
- const tokens = cleaned.split(/\s+/).filter(Boolean);
46136
- return tokens.filter((t) => !DEFAULT_STOPWORDS.has(t));
46137
- }
46138
- function lexicalSimilarity(a, b) {
46139
- const goldWords = extractContentWords(a);
46140
- if (goldWords.length === 0) {
46141
- return 0;
46142
- }
46143
- const candWords = new Set(extractContentWords(b));
46144
- let matchCount = 0;
46145
- for (const word of goldWords) {
46146
- if (candWords.has(word)) {
46147
- matchCount++;
46148
- }
46149
- }
46150
- return matchCount / goldWords.length;
46151
- }
46152
- var CLASS_RANK = {
46153
- extraction_miss: 1,
46154
- index_miss: 2,
46155
- retrieval_miss: 3,
46156
- use_miss: 4,
46157
- unattributed: 5
46158
- };
46159
- var RETRIEVAL_STAGE_RANK = {
46160
- cap: 1,
46161
- rank: 2,
46162
- filter: 3,
46163
- unknown: 4
46164
- };
46165
- function computeOverallLabel(golds) {
46166
- if (golds.length === 0) {
46167
- return { class: "unattributed", reason: "no gold memories" };
46168
- }
46169
- let bestGold = golds[0];
46170
- for (let i = 1; i < golds.length; i++) {
46171
- const current = golds[i];
46172
- const bestRank = CLASS_RANK[bestGold.label.class];
46173
- const currRank = CLASS_RANK[current.label.class];
46174
- if (currRank < bestRank) {
46175
- bestGold = current;
46176
- } else if (currRank === bestRank && current.label.class === "retrieval_miss") {
46177
- const bestRetRank = RETRIEVAL_STAGE_RANK[bestGold.label.retrievalStage ?? "unknown"];
46178
- const currRetRank = RETRIEVAL_STAGE_RANK[current.label.retrievalStage ?? "unknown"];
46179
- if (currRetRank < bestRetRank) {
46180
- bestGold = current;
46181
- }
46182
- }
46183
- }
46184
- return { ...bestGold.label };
46185
- }
46186
- function isDiagnosticScore(name) {
46187
- return name.endsWith("_agreement") || name.includes("_id_leak") || name === "search_hits";
46188
- }
46189
- function isTaskFailed(task) {
46190
- if (!task.scores || Object.keys(task.scores).length === 0) {
46191
- return true;
46192
- }
46193
- if ("overall" in task.scores && typeof task.scores.overall === "number") {
46194
- return task.scores.overall < 1;
46195
- }
46196
- const primaryScores = Object.entries(task.scores).filter(([name, score]) => typeof score === "number" && !isDiagnosticScore(name)).map(([, score]) => score);
46197
- return primaryScores.length === 0 || Math.min(...primaryScores) < 1;
46198
- }
46199
- function withMemoizedListMemories(env) {
46200
- let cache = null;
46201
- return {
46202
- ...env,
46203
- listMemories() {
46204
- if (!cache) {
46205
- cache = env.listMemories();
46206
- }
46207
- return cache;
46208
- }
46209
- };
46210
- }
46211
- async function attributeGoldMemory(goldStatement, question, env, options = {}, recalledText) {
46212
- if (options.threshold !== void 0) {
46213
- if (typeof options.threshold !== "number" || !Number.isFinite(options.threshold) || options.threshold < 0 || options.threshold > 1) {
46214
- throw new RangeError("attribution threshold must be a finite number between 0 and 1");
46215
- }
46216
- }
46217
- const threshold = options.threshold ?? 0.6;
46218
- const simFn = options.similarity ?? lexicalSimilarity;
46219
- const goldInRecalledText = typeof recalledText === "string" && simFn(goldStatement, recalledText) >= threshold;
46220
- const stages = {
46221
- extraction: { status: "unavailable" },
46222
- index: { status: "unavailable" },
46223
- retrieval: { status: "unavailable" },
46224
- use: { status: "unavailable" }
46225
- };
46226
- let memories = [];
46227
- let extractionRan = false;
46228
- let extractionErrorDetail;
46229
- if (typeof env.listMemories === "function") {
46230
- try {
46231
- memories = await env.listMemories();
46232
- extractionRan = true;
46233
- } catch {
46234
- extractionRan = false;
46235
- extractionErrorDetail = "listMemories failed";
46236
- }
46237
- }
46238
- let bestSim = -1;
46239
- let matchedMem = null;
46240
- if (extractionRan) {
46241
- if (memories.length === 0) {
46242
- if (goldInRecalledText) {
46243
- const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
46244
- stages.extraction = { status: "pass", detail: impliedDetail };
46245
- stages.index = { status: "pass", detail: impliedDetail };
46246
- stages.retrieval = { status: "pass", detail: impliedDetail };
46247
- stages.use = {
46248
- status: "fail",
46249
- detail: "Gold memory present in context but answer was incorrect"
46250
- };
46251
- return {
46252
- goldMemory: goldStatement,
46253
- label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
46254
- stages
46255
- };
46256
- }
46257
- const detail = "store contains no memories";
46258
- stages.extraction = { status: "fail", detail };
46259
- stages.index = { status: "unavailable", detail: "not reached" };
46260
- stages.retrieval = { status: "unavailable", detail: "not reached" };
46261
- stages.use = { status: "unavailable", detail: "not reached" };
46262
- return {
46263
- goldMemory: goldStatement,
46264
- label: { class: "extraction_miss", reason: detail },
46265
- stages
46266
- };
46267
- }
46268
- for (const mem of memories) {
46269
- const sim = simFn(goldStatement, mem.content);
46270
- if (sim > bestSim) {
46271
- bestSim = sim;
46272
- matchedMem = mem;
46273
- }
46274
- }
46275
- if (bestSim < threshold || !matchedMem) {
46276
- if (goldInRecalledText) {
46277
- const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
46278
- stages.extraction = { status: "pass", detail: impliedDetail };
46279
- stages.index = { status: "pass", detail: impliedDetail };
46280
- stages.retrieval = { status: "pass", detail: impliedDetail };
46281
- stages.use = {
46282
- status: "fail",
46283
- detail: "Gold memory present in context but answer was incorrect"
46284
- };
46285
- return {
46286
- goldMemory: goldStatement,
46287
- label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
46288
- stages
46289
- };
46715
+ }
46716
+ searchSamples.push(searchRes.ms);
46290
46717
  }
46291
- const detail = `Best similarity ${bestSim >= 0 ? bestSim.toFixed(3) : 0} below threshold ${threshold}`;
46292
- stages.extraction = { status: "fail", detail };
46293
- stages.index = { status: "unavailable", detail: "not reached" };
46294
- stages.retrieval = { status: "unavailable", detail: "not reached" };
46295
- stages.use = { status: "unavailable", detail: "not reached" };
46296
- return {
46297
- goldMemory: goldStatement,
46298
- label: { class: "extraction_miss", reason: detail },
46299
- stages
46300
- };
46301
- }
46302
- stages.extraction = {
46303
- status: "pass",
46304
- detail: `Matched memory ${matchedMem.id} (sim ${bestSim.toFixed(3)})`
46305
- };
46306
- } else {
46307
- stages.extraction = {
46308
- status: "unavailable",
46309
- detail: extractionErrorDetail ?? "listMemories unavailable"
46310
- };
46311
- }
46312
- const matchedMemoryId = matchedMem ? matchedMem.id : void 0;
46313
- const recallLimit = env.recallLimit;
46314
- const replayLimit = env.replayLimit ?? Math.max(25, recallLimit * 5);
46315
- let indexCheckPassed = false;
46316
- let indexCheckFailed = false;
46317
- if (typeof env.oracleSearch === "function" && extractionRan) {
46318
- try {
46319
- const oracleResults = await env.oracleSearch(goldStatement, replayLimit);
46320
- const idMatched = matchedMemoryId ? oracleResults.some((r) => r.id === matchedMemoryId) : false;
46321
- if (idMatched) {
46322
- indexCheckPassed = true;
46323
- } else {
46324
- const memMap = new Map(memories.map((m) => [m.id, m]));
46325
- indexCheckPassed = oracleResults.some((r) => {
46326
- const mem = memMap.get(r.id);
46327
- return mem ? simFn(goldStatement, mem.content) >= threshold : false;
46328
- });
46718
+ const deadCode = timeSync(() => store.deadCode());
46719
+ if (!deadCode.result.ok) {
46720
+ throw new Error(`dead_code failed: ${deadCode.result.code}`);
46329
46721
  }
46330
- if (indexCheckPassed) {
46331
- stages.index = { status: "pass", detail: "Found in oracle search" };
46332
- } else {
46333
- indexCheckFailed = true;
46334
- stages.index = { status: "fail", detail: "Not found in oracle search" };
46722
+ await store.drain();
46723
+ let dbBytes = statSync(dbPath).size;
46724
+ try {
46725
+ dbBytes += statSync(dbPath + "-wal").size;
46726
+ } catch {
46335
46727
  }
46336
- } catch {
46337
- stages.index = { status: "unavailable", detail: "oracleSearch threw error" };
46338
- }
46339
- } else if (typeof env.oracleSearch === "function") {
46340
- stages.index = {
46341
- status: "unavailable",
46342
- detail: "extraction check unavailable; oracle result would be ambiguous"
46343
- };
46344
- } else {
46345
- stages.index = { status: "unavailable", detail: "index check unavailable" };
46346
- }
46347
- let retrievalCheckPassed = false;
46348
- let retrievalStageMiss = void 0;
46349
- if (typeof env.recall === "function") {
46350
- try {
46351
- const recallResults = await env.recall(question, recallLimit);
46352
- const isGoldInRecall = recallResults.some(
46353
- (m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
46354
- );
46355
- if (isGoldInRecall) {
46356
- retrievalCheckPassed = true;
46357
- stages.retrieval = { status: "pass", detail: `Recalled within recallLimit ${recallLimit}` };
46358
- } else {
46359
- const replayResults = await env.recall(question, replayLimit);
46360
- const replayIndex = replayResults.findIndex(
46361
- (m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
46728
+ const kloc = Math.max(1, repo.approximateLoc / 1e3);
46729
+ const dbBytesPerKloc = dbBytes / kloc;
46730
+ const modifiedSamples = [];
46731
+ for (let i = 0; i < iterations; i++) {
46732
+ const fileIdx = i % storeFiles.length;
46733
+ const original = storeFiles[fileIdx];
46734
+ if (!original) continue;
46735
+ const ownSyms = new Set(original.symbols.map((sym) => sym.qualifiedName));
46736
+ const churnName = `mod.benchChurnSymbol${i}`;
46737
+ const representativeEdges = (original.edges ?? []).filter((e) => !ownSyms.has(e.dstQualifiedName)).slice(0, 2).map((e) => ({ ...e, srcQualifiedName: churnName }));
46738
+ const modified = {
46739
+ ...original,
46740
+ contentHash: `${original.contentHash}-mod-${i}`,
46741
+ symbols: [
46742
+ {
46743
+ qualifiedName: churnName,
46744
+ name: `benchChurnSymbol${i}`,
46745
+ kind: "function",
46746
+ span: { startByte: 0, endByte: 0 }
46747
+ }
46748
+ ],
46749
+ edges: representativeEdges
46750
+ };
46751
+ const modResult = await timeAsync(
46752
+ () => store.upsertFileBatch([modified])
46362
46753
  );
46363
- if (replayIndex >= 0) {
46364
- const rank = replayIndex + 1;
46365
- retrievalStageMiss = "cap";
46366
- stages.retrieval = {
46367
- status: "fail",
46368
- detail: `Rank ${rank} exceeds recallLimit ${recallLimit}`
46369
- };
46370
- } else {
46371
- retrievalStageMiss = "unknown";
46372
- stages.retrieval = {
46373
- status: "fail",
46374
- detail: `absent from recall at replayLimit ${replayLimit}; filter vs rank indistinguishable without candidate-stage evidence`
46375
- };
46754
+ if (!modResult.result.ok) {
46755
+ throw new Error(`modified update failed: ${modResult.result.code}`);
46756
+ }
46757
+ modifiedSamples.push(modResult.ms);
46758
+ const restoreResult = await store.upsertFileBatch(storeFiles);
46759
+ if (!restoreResult.ok) {
46760
+ throw new Error(`restore failed: ${restoreResult.code}`);
46376
46761
  }
46762
+ sampleRss();
46377
46763
  }
46378
- } catch {
46379
- stages.retrieval = { status: "unavailable", detail: "recall threw error" };
46764
+ sampleRss();
46765
+ const peakRssBytes = peakRss;
46766
+ return {
46767
+ schemaVersion: CODING_GRAPH_BENCH_SCHEMA_VERSION,
46768
+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),
46769
+ machine: captureMachineFingerprint(),
46770
+ fixture: {
46771
+ config: fixtureConfig,
46772
+ approximateLoc: repo.approximateLoc,
46773
+ fileCount: repo.files.length,
46774
+ symbolCount: repo.files.reduce((sum, f) => sum + f.symbols.length, 0),
46775
+ edgeCount: repo.files.reduce((sum, f) => sum + f.edges.length, 0)
46776
+ },
46777
+ fullIndexMs: { ms: fullIndexMsValue },
46778
+ fullIndexLocsPerSecond: Math.round(locsPerSecond),
46779
+ incrementalUpdate: computeMicroMetric(incrementalSamples),
46780
+ incrementalModifiedUpdate: computeMicroMetric(
46781
+ modifiedSamples.length > 0 ? modifiedSamples : [0]
46782
+ ),
46783
+ tracePath: computeMicroMetric(traceSamples.length > 0 ? traceSamples : [0]),
46784
+ searchGraph: computeMicroMetric(searchSamples),
46785
+ deadCodeMs: { ms: deadCode.ms },
46786
+ dbBytesPerKloc: Math.round(dbBytesPerKloc),
46787
+ peakRssBytes,
46788
+ dbBytes,
46789
+ graphNodeCount,
46790
+ graphEdgeCount
46791
+ };
46792
+ } finally {
46793
+ await store.close();
46380
46794
  }
46381
- } else {
46382
- stages.retrieval = { status: "unavailable", detail: "retrieval check unavailable" };
46383
- }
46384
- if (!retrievalCheckPassed && goldInRecalledText) {
46385
- retrievalCheckPassed = true;
46386
- stages.retrieval = { status: "pass", detail: "Found in recalledText context" };
46795
+ } finally {
46796
+ await rm15(dir, { recursive: true, force: true });
46387
46797
  }
46388
- if (retrievalCheckPassed) {
46389
- if (indexCheckFailed) {
46390
- stages.index = { status: "pass", detail: "implied pass from retrieval (oracle query missed)" };
46391
- indexCheckPassed = true;
46392
- } else if (stages.index.status === "unavailable") {
46393
- stages.index = { status: "pass", detail: "implied pass from retrieval" };
46394
- indexCheckPassed = true;
46395
- }
46798
+ }
46799
+
46800
+ // src/coding-graph/regression.ts
46801
+ var METRIC_DIRECTION = {
46802
+ fullIndexMs: "lower-is-better",
46803
+ fullIndexLocsPerSecond: "higher-is-better",
46804
+ incrementalUpdateP95Ms: "lower-is-better",
46805
+ incrementalUpdateP50Ms: "lower-is-better",
46806
+ incrementalModifiedUpdateP95Ms: "lower-is-better",
46807
+ incrementalModifiedUpdateP50Ms: "lower-is-better",
46808
+ tracePathP95Ms: "lower-is-better",
46809
+ searchGraphP95Ms: "lower-is-better",
46810
+ deadCodeMs: "lower-is-better",
46811
+ dbBytesPerKloc: "lower-is-better"
46812
+ };
46813
+ function extractMetrics(report) {
46814
+ return {
46815
+ fullIndexMs: report.fullIndexMs.ms,
46816
+ fullIndexLocsPerSecond: report.fullIndexLocsPerSecond,
46817
+ incrementalUpdateP50Ms: report.incrementalUpdate.p50,
46818
+ incrementalUpdateP95Ms: report.incrementalUpdate.p95,
46819
+ incrementalModifiedUpdateP50Ms: report.incrementalModifiedUpdate.p50,
46820
+ incrementalModifiedUpdateP95Ms: report.incrementalModifiedUpdate.p95,
46821
+ tracePathP95Ms: report.tracePath.p95,
46822
+ searchGraphP95Ms: report.searchGraph.p95,
46823
+ deadCodeMs: report.deadCodeMs.ms,
46824
+ dbBytesPerKloc: report.dbBytesPerKloc
46825
+ };
46826
+ }
46827
+ var NODE_MAJOR_CACHE = /* @__PURE__ */ new WeakMap();
46828
+ function nodeMajor(fp) {
46829
+ const cached = NODE_MAJOR_CACHE.get(fp);
46830
+ if (cached !== void 0) return cached;
46831
+ const major = fp.nodeVersion.replace(/^v/, "").split(".")[0] ?? fp.nodeVersion;
46832
+ NODE_MAJOR_CACHE.set(fp, major);
46833
+ return major;
46834
+ }
46835
+ function compareMachineFingerprints(report, baseline) {
46836
+ const differing = [];
46837
+ if (report.arch !== baseline.arch) differing.push("arch");
46838
+ if (report.platform !== baseline.platform) differing.push("platform");
46839
+ if (nodeMajor(report) !== nodeMajor(baseline)) differing.push("nodeVersion(major)");
46840
+ if (report.cpuModel !== null && baseline.cpuModel !== null && report.cpuModel !== baseline.cpuModel) {
46841
+ differing.push("cpuModel");
46396
46842
  }
46397
- if (retrievalCheckPassed) {
46398
- stages.use = {
46399
- status: "fail",
46400
- detail: "Gold memory present in context but answer was incorrect"
46843
+ if (report.cpuCores !== baseline.cpuCores) differing.push("cpuCores");
46844
+ return { differingFields: differing };
46845
+ }
46846
+ function invalidFingerprintFields(fp) {
46847
+ const bad = [];
46848
+ if (typeof fp.arch !== "string") bad.push("arch");
46849
+ if (typeof fp.platform !== "string") bad.push("platform");
46850
+ if (typeof fp.nodeVersion !== "string") bad.push("nodeVersion");
46851
+ if (typeof fp.cpuCores !== "number" || !Number.isFinite(fp.cpuCores)) bad.push("cpuCores");
46852
+ if (fp.cpuModel !== null && typeof fp.cpuModel !== "string") bad.push("cpuModel");
46853
+ return bad;
46854
+ }
46855
+ function checkCodingGraphRegression(report, baseline, tolerancePercent = DEFAULT_TOLERANCE_PERCENT) {
46856
+ const reportFixture = report.fixture.config;
46857
+ const baselineFixture = baseline.fixtureConfig;
46858
+ const mismatchedKeys = Object.keys(baselineFixture).filter((key) => reportFixture[key] !== baselineFixture[key]);
46859
+ if (mismatchedKeys.length > 0) {
46860
+ const diffs = mismatchedKeys.map(
46861
+ (key) => `${key}: report=${reportFixture[key]} baseline=${baselineFixture[key]}`
46862
+ ).join(", ");
46863
+ return {
46864
+ passed: false,
46865
+ regressions: [],
46866
+ summary: `Fixture mismatch (${diffs}). Metrics are not comparable across different fixtures.`
46401
46867
  };
46868
+ }
46869
+ if (report.schemaVersion !== baseline.schemaVersion) {
46402
46870
  return {
46403
- goldMemory: goldStatement,
46404
- label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
46405
- stages
46871
+ passed: false,
46872
+ regressions: [],
46873
+ summary: `Schema-version mismatch: report=${report.schemaVersion} baseline=${baseline.schemaVersion}. Metrics are not comparable across schema versions \u2014 regenerate the baseline (#1688).`
46406
46874
  };
46407
46875
  }
46408
- if (stages.extraction.status === "pass" && indexCheckFailed) {
46409
- stages.use = { status: "unavailable", detail: "not reached" };
46876
+ const metricChecks = [
46877
+ ["incrementalUpdate.p50", report.incrementalUpdate?.p50],
46878
+ ["incrementalUpdate.p95", report.incrementalUpdate?.p95],
46879
+ ["incrementalModifiedUpdate.p50", report.incrementalModifiedUpdate?.p50],
46880
+ ["incrementalModifiedUpdate.p95", report.incrementalModifiedUpdate?.p95],
46881
+ ["tracePath.p95", report.tracePath?.p95],
46882
+ ["searchGraph.p95", report.searchGraph?.p95],
46883
+ ["fullIndexMs.ms", report.fullIndexMs?.ms],
46884
+ ["fullIndexLocsPerSecond", report.fullIndexLocsPerSecond],
46885
+ ["deadCodeMs.ms", report.deadCodeMs?.ms],
46886
+ ["dbBytesPerKloc", report.dbBytesPerKloc]
46887
+ ];
46888
+ const missingFields = metricChecks.filter(([, v]) => typeof v !== "number" || !Number.isFinite(v)).map(([k]) => k);
46889
+ if (missingFields.length > 0) {
46410
46890
  return {
46411
- goldMemory: goldStatement,
46412
- label: { class: "index_miss", reason: "Gold statement missing from search index" },
46413
- stages
46891
+ passed: false,
46892
+ regressions: [],
46893
+ summary: "Report claims schemaVersion " + report.schemaVersion + " but is missing required metric field(s): " + missingFields.join(", ") + " \u2014 the report is incomplete or corrupt. Regenerate it (#1688)."
46414
46894
  };
46415
46895
  }
46416
- if (stages.extraction.status === "pass" && stages.index.status === "pass" && stages.retrieval.status === "fail") {
46417
- stages.use = { status: "unavailable", detail: "not reached" };
46896
+ const badBaselineMetrics = Object.keys(METRIC_DIRECTION).filter((key) => {
46897
+ const v = baseline.metrics[key];
46898
+ return v != null && (typeof v !== "number" || !Number.isFinite(v));
46899
+ });
46900
+ if (badBaselineMetrics.length > 0) {
46418
46901
  return {
46419
- goldMemory: goldStatement,
46420
- label: {
46421
- class: "retrieval_miss",
46422
- retrievalStage: retrievalStageMiss ?? "unknown",
46423
- reason: stages.retrieval.detail
46424
- },
46425
- stages
46902
+ passed: false,
46903
+ regressions: [],
46904
+ summary: "Baseline has a non-numeric required metric field(s): " + badBaselineMetrics.join(", ") + " \u2014 the baseline is corrupt. Regenerate it (#1688)."
46426
46905
  };
46427
46906
  }
46428
- const missingReason = stages.extraction.status === "unavailable" ? `extraction check unavailable (${stages.extraction.detail})` : stages.index.status === "unavailable" && stages.retrieval.status === "unavailable" ? "index/retrieval checks unavailable in this attribution environment" : stages.index.status === "unavailable" ? "index check unavailable; a retrieval miss cannot be isolated from an index miss" : "retrieval check unavailable";
46429
- stages.use = { status: "unavailable", detail: "not reached" };
46430
- return {
46431
- goldMemory: goldStatement,
46432
- label: { class: "unattributed", reason: missingReason },
46433
- stages
46434
- };
46435
- }
46436
- async function attributeTask(task, env, options = {}) {
46437
- const golds = task.goldMemories;
46438
- if (!golds || golds.length === 0) {
46439
- return null;
46907
+ const reportFpBad = report.machine == null ? ["<missing>"] : invalidFingerprintFields(report.machine);
46908
+ const baselineFpBad = baseline.machine == null ? ["<missing>"] : invalidFingerprintFields(baseline.machine);
46909
+ if (reportFpBad.length > 0 || baselineFpBad.length > 0) {
46910
+ const which = [];
46911
+ if (reportFpBad.length > 0) which.push("report (" + reportFpBad.join(", ") + ")");
46912
+ if (baselineFpBad.length > 0) which.push("baseline (" + baselineFpBad.join(", ") + ")");
46913
+ return {
46914
+ passed: false,
46915
+ regressions: [],
46916
+ summary: "Report or baseline machine fingerprint is missing or has an invalid field type(s): " + which.join("; ") + " \u2014 the artifact is incomplete or corrupt. Regenerate it (#1688)."
46917
+ };
46440
46918
  }
46441
- const memoizedEnv = withMemoizedListMemories(env);
46442
- const recalledText = typeof task.details?.recalledText === "string" ? task.details.recalledText : void 0;
46443
- const goldAttributions = [];
46444
- for (const gold of golds) {
46445
- const attr = await attributeGoldMemory(gold, task.question, memoizedEnv, options, recalledText);
46446
- goldAttributions.push(attr);
46919
+ const mismatch = compareMachineFingerprints(report.machine, baseline.machine);
46920
+ if (mismatch.differingFields.length > 0) {
46921
+ return {
46922
+ passed: true,
46923
+ skipped: true,
46924
+ regressions: [],
46925
+ summary: "Machine-fingerprint mismatch \u2014 comparison skipped to avoid a false-positive hardware-variance failure. Differing fields: " + mismatch.differingFields.join(", ") + ". Regenerate the baseline on this machine for a real comparison (#1688).",
46926
+ machineMismatch: {
46927
+ report: report.machine,
46928
+ baseline: baseline.machine,
46929
+ differingFields: mismatch.differingFields
46930
+ }
46931
+ };
46447
46932
  }
46448
- const overall = computeOverallLabel(goldAttributions);
46449
- return {
46450
- taskId: task.taskId,
46451
- question: task.question,
46452
- golds: goldAttributions,
46453
- overall
46454
- };
46455
- }
46456
- async function attributeRun(result, env, options = {}) {
46457
- const runId = result.meta?.runId ?? result.meta?.id ?? "unknown-run";
46458
- const memoizedEnv = withMemoizedListMemories(env);
46459
- const totals = {
46460
- extraction_miss: 0,
46461
- index_miss: 0,
46462
- retrieval_miss: 0,
46463
- use_miss: 0,
46464
- unattributed: 0
46465
- };
46466
- const retrievalStages = {
46467
- filter: 0,
46468
- cap: 0,
46469
- rank: 0,
46470
- unknown: 0
46471
- };
46472
- const items = [];
46473
- const skippedTasks = [];
46474
- for (const task of result.results.tasks) {
46475
- if (task.details?.benchmarkFailure && typeof task.details.benchmarkFailure === "object") {
46476
- skippedTasks.push({
46477
- taskId: task.taskId,
46478
- reason: "trial execution failure (not an answer failure)"
46479
- });
46480
- continue;
46481
- }
46482
- if (!task.goldMemories || task.goldMemories.length === 0) {
46483
- skippedTasks.push({
46484
- taskId: task.taskId,
46485
- reason: "No goldMemories specified"
46486
- });
46487
- continue;
46488
- }
46489
- if (!isTaskFailed(task)) {
46490
- skippedTasks.push({
46491
- taskId: task.taskId,
46492
- reason: "Task passed (score >= 1)"
46933
+ const measured = extractMetrics(report);
46934
+ const baselineMetrics = baseline.metrics;
46935
+ const regressions = [];
46936
+ for (const key of Object.keys(METRIC_DIRECTION)) {
46937
+ const baseVal = baselineMetrics[key];
46938
+ const measVal = measured[key];
46939
+ if (baseVal == null || measVal == null) continue;
46940
+ if (baseVal === 0) continue;
46941
+ const direction = METRIC_DIRECTION[key];
46942
+ const ratio2 = measVal / baseVal;
46943
+ const percentChange2 = direction === "lower-is-better" ? (ratio2 - 1) * 100 : (1 - ratio2) * 100;
46944
+ const regressed = percentChange2 > tolerancePercent;
46945
+ if (regressed) {
46946
+ regressions.push({
46947
+ key,
46948
+ baseline: baseVal,
46949
+ measured: measVal,
46950
+ percentChange: Math.round(percentChange2 * 10) / 10,
46951
+ direction,
46952
+ tolerancePercent,
46953
+ regressed: true
46493
46954
  });
46494
- continue;
46495
- }
46496
- const taskAttr = await attributeTask(task, memoizedEnv, options);
46497
- if (taskAttr) {
46498
- items.push(taskAttr);
46499
- }
46500
- }
46501
- items.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
46502
- skippedTasks.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
46503
- for (const item of items) {
46504
- totals[item.overall.class]++;
46505
- if (item.overall.class === "retrieval_miss") {
46506
- const stage = item.overall.retrievalStage ?? "unknown";
46507
- retrievalStages[stage]++;
46508
46955
  }
46509
46956
  }
46957
+ const passed = regressions.length === 0;
46958
+ const summary = passed ? "All metrics within tolerance." : `${regressions.length} metric(s) regressed beyond ${tolerancePercent}% tolerance:
46959
+ ` + regressions.map(
46960
+ (r) => ` ${r.key}: ${r.baseline} \u2192 ${r.measured} (${r.percentChange > 0 ? "+" : ""}${r.percentChange}% vs baseline)`
46961
+ ).join("\n");
46962
+ return { passed, regressions, summary };
46963
+ }
46964
+ function buildBaselineFromReport(report, note) {
46510
46965
  return {
46511
- runId,
46512
- totals,
46513
- retrievalStages,
46514
- attributedTasks: items.length,
46515
- skippedTasks,
46516
- items
46966
+ schemaVersion: report.schemaVersion,
46967
+ machine: report.machine,
46968
+ fixtureConfig: report.fixture.config,
46969
+ metrics: extractMetrics(report),
46970
+ createdAt: report.timestamp,
46971
+ note
46517
46972
  };
46518
46973
  }
46519
- function renderAttributionReportTable(report) {
46520
- const lines = [];
46521
- lines.push(`Attribution Report (Run: ${report.runId})`);
46522
- lines.push(`Failed-task predicate: minimum primary answer score < 1 (scores.overall or non-diagnostic scores)`);
46523
- lines.push(`Attributed tasks: ${report.attributedTasks}, Skipped tasks: ${report.skippedTasks.length}`);
46524
- lines.push("");
46525
- lines.push("Totals by Class:");
46526
- lines.push(` extraction_miss: ${report.totals.extraction_miss}`);
46527
- lines.push(` index_miss: ${report.totals.index_miss}`);
46528
- lines.push(` retrieval_miss: ${report.totals.retrieval_miss}`);
46529
- lines.push(` use_miss: ${report.totals.use_miss}`);
46530
- lines.push(` unattributed: ${report.totals.unattributed}`);
46531
- lines.push("");
46532
- lines.push("Retrieval Miss Stages:");
46533
- lines.push(` filter: ${report.retrievalStages.filter}`);
46534
- lines.push(` cap: ${report.retrievalStages.cap}`);
46535
- lines.push(` rank: ${report.retrievalStages.rank}`);
46536
- lines.push(` unknown: ${report.retrievalStages.unknown}`);
46537
- lines.push("");
46538
- lines.push("Task Attributions:");
46539
- if (report.items.length === 0) {
46540
- lines.push(" (none)");
46541
- } else {
46542
- for (const item of report.items) {
46543
- const stageStr = item.overall.retrievalStage ? ` (${item.overall.retrievalStage})` : "";
46544
- const labelStr = `${item.overall.class}${stageStr}`;
46545
- const reasonStr = item.overall.reason ? ` - ${item.overall.reason}` : "";
46546
- lines.push(` ${item.taskId.padEnd(20)} ${labelStr.padEnd(24)}${reasonStr}`);
46547
- }
46548
- }
46549
- if (report.skippedTasks.length > 0) {
46550
- lines.push("");
46551
- lines.push("Skipped Tasks:");
46552
- for (const skipped of report.skippedTasks) {
46553
- lines.push(` ${skipped.taskId.padEnd(20)} ${skipped.reason}`);
46554
- }
46555
- }
46556
- return `${lines.join("\n")}
46557
- `;
46558
- }
46559
- function serializeAttributionReport(report) {
46560
- return `${JSON.stringify(report, null, 2)}
46561
- `;
46562
- }
46563
46974
 
46564
46975
  // src/attribute-cli.ts
46976
+ import { QmdClient } from "@remnic/core";
46565
46977
  import { lstat as lstat6, readdir as readdir8, readFile as readFile25 } from "fs/promises";
46566
46978
  import path40 from "path";
46567
46979
  function parseFrontmatter2(fileContent) {
@@ -46658,6 +47070,53 @@ async function scanMemoryDir(dirPath) {
46658
47070
  }
46659
47071
  return memories;
46660
47072
  }
47073
+ async function resolveQmdMemory(memoryDir, collection, resultPath) {
47074
+ const root = path40.resolve(memoryDir);
47075
+ const candidates = /* @__PURE__ */ new Set();
47076
+ const addCandidate = (candidate) => {
47077
+ const resolved = path40.resolve(candidate);
47078
+ const relative = path40.relative(root, resolved);
47079
+ if (relative !== ".." && !relative.startsWith(`..${path40.sep}`) && !path40.isAbsolute(relative)) {
47080
+ candidates.add(resolved);
47081
+ }
47082
+ };
47083
+ const addRelative = (relativePath) => {
47084
+ const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
47085
+ if (!normalized) return;
47086
+ addCandidate(path40.join(root, normalized));
47087
+ if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
47088
+ addCandidate(path40.join(root, "facts", normalized));
47089
+ }
47090
+ };
47091
+ if (path40.isAbsolute(resultPath)) {
47092
+ addCandidate(resultPath);
47093
+ } else {
47094
+ addRelative(resultPath);
47095
+ const normalized = resultPath.replace(/\\/g, "/").replace(/^\/+/, "");
47096
+ if (normalized.startsWith(`${collection}/`)) {
47097
+ addRelative(normalized.slice(collection.length + 1));
47098
+ }
47099
+ }
47100
+ let resolvedMemory = null;
47101
+ for (const candidate of candidates) {
47102
+ try {
47103
+ const stats = await lstat6(candidate);
47104
+ if (!stats.isFile() || stats.isSymbolicLink()) continue;
47105
+ const parsed = parseFrontmatter2(await readFile25(candidate, "utf8"));
47106
+ if (!parsed.id || parsed.id.trim().length === 0) {
47107
+ throw new Error("QMD result has no canonical frontmatter id");
47108
+ }
47109
+ const memory = { id: parsed.id, content: parsed.body.trim() };
47110
+ if (resolvedMemory && resolvedMemory.id !== memory.id) {
47111
+ throw new Error("QMD result path resolves to more than one canonical memory id");
47112
+ }
47113
+ resolvedMemory = memory;
47114
+ } catch (error) {
47115
+ if (error instanceof Error && error.message.startsWith("QMD result")) throw error;
47116
+ }
47117
+ }
47118
+ return resolvedMemory;
47119
+ }
46661
47120
  async function runAttributeCliCommand(options) {
46662
47121
  const summary = await resolveBenchmarkResultReference(options.resultsDir, options.runRef);
46663
47122
  if (!summary) {
@@ -46677,6 +47136,12 @@ async function runAttributeCliCommand(options) {
46677
47136
  `
46678
47137
  };
46679
47138
  }
47139
+ if (Boolean(options.qmdPath) !== Boolean(options.collection)) {
47140
+ return {
47141
+ exitCode: 1,
47142
+ output: "Error: --qmd <path> and --collection <name> must be provided together.\n"
47143
+ };
47144
+ }
46680
47145
  let memorySnapshot;
46681
47146
  const listMemoriesFn = async () => {
46682
47147
  if (memorySnapshot) return memorySnapshot;
@@ -46697,14 +47162,64 @@ async function runAttributeCliCommand(options) {
46697
47162
  }
46698
47163
  const recallLimitRaw = result.config?.remnicConfig?.recallLimit;
46699
47164
  const recallLimit = typeof recallLimitRaw === "number" && recallLimitRaw > 0 ? recallLimitRaw : 10;
46700
- const rankMemories2 = async (query, limit) => (await listMemoriesFn()).map((memory) => ({ memory, score: lexicalSimilarity(query, memory.content) })).filter(({ score }) => score > 0).sort((a, b) => b.score - a.score || a.memory.id.localeCompare(b.memory.id)).slice(0, limit).map(({ memory }) => memory);
47165
+ const needsLegacyFallback = result.results.tasks.some(
47166
+ (task) => task.attributionWitness === void 0 && isTaskFailed(task) && Array.isArray(task.goldMemories) && task.goldMemories.length > 0 && !(task.details?.benchmarkFailure && typeof task.details.benchmarkFailure === "object")
47167
+ );
47168
+ if (needsLegacyFallback && options.qmdPath && !options.memoryDir) {
47169
+ return {
47170
+ exitCode: 1,
47171
+ output: "Error: explicit legacy QMD fallback requires --memory-dir <path>.\n"
47172
+ };
47173
+ }
46701
47174
  const env = {
46702
47175
  listMemories: listMemoriesFn,
46703
- oracleSearch: async (query, limit) => (await rankMemories2(query, limit)).map(({ id }) => ({ id })),
46704
- recall: rankMemories2,
46705
47176
  recallLimit
46706
47177
  };
46707
- const report = await attributeRun(result, env, { threshold: options.threshold });
47178
+ let qmdClient;
47179
+ if (needsLegacyFallback && options.qmdPath && options.collection && options.memoryDir) {
47180
+ qmdClient = new QmdClient(options.collection, recallLimit, {
47181
+ qmdPath: options.qmdPath,
47182
+ qmdStrictPath: true
47183
+ });
47184
+ const qmdAvailable = await qmdClient.probe().catch(() => false);
47185
+ const search = async (query, limit) => {
47186
+ if (!qmdAvailable) {
47187
+ throw new Error("QMD unavailable");
47188
+ }
47189
+ const degradations = [];
47190
+ const results = await qmdClient.search(
47191
+ query,
47192
+ options.collection,
47193
+ limit,
47194
+ void 0,
47195
+ { onDegradation: (degradation) => degradations.push(degradation) }
47196
+ );
47197
+ if (degradations.length > 0) {
47198
+ throw new Error("QMD search degraded");
47199
+ }
47200
+ const memories = [];
47201
+ const seenIds = /* @__PURE__ */ new Set();
47202
+ for (const resultItem of results) {
47203
+ const memory = await resolveQmdMemory(options.memoryDir, options.collection, resultItem.path);
47204
+ if (!memory) {
47205
+ throw new Error("QMD result canonical identity unavailable");
47206
+ }
47207
+ if (!seenIds.has(memory.id)) {
47208
+ seenIds.add(memory.id);
47209
+ memories.push(memory);
47210
+ }
47211
+ }
47212
+ return memories;
47213
+ };
47214
+ env.oracleSearch = async (query, limit) => (await search(query, limit)).map(({ id }) => ({ id }));
47215
+ env.recall = search;
47216
+ }
47217
+ let report;
47218
+ try {
47219
+ report = await attributeRun(result, env, { threshold: options.threshold });
47220
+ } finally {
47221
+ await qmdClient?.dispose();
47222
+ }
46708
47223
  const output = options.json ? serializeAttributionReport(report) : renderAttributionReportTable(report);
46709
47224
  return {
46710
47225
  exitCode: 0,