@remnic/bench 9.45.3 → 9.45.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +50 -1
- package/dist/index.js +1586 -1071
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1977,6 +1977,63 @@ function isProviderConfigLike(value) {
|
|
|
1977
1977
|
}
|
|
1978
1978
|
return isObjectRecord(value) && typeof value.provider === "string" && typeof value.model === "string";
|
|
1979
1979
|
}
|
|
1980
|
+
function isNonEmptyString(value) {
|
|
1981
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
1982
|
+
}
|
|
1983
|
+
function isNonNegativeInteger(value) {
|
|
1984
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
1985
|
+
}
|
|
1986
|
+
function isUniqueMemoryIdList(value) {
|
|
1987
|
+
if (value === null) return true;
|
|
1988
|
+
if (!Array.isArray(value)) return false;
|
|
1989
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1990
|
+
for (const id of value) {
|
|
1991
|
+
if (!isNonEmptyString(id) || ids.has(id)) return false;
|
|
1992
|
+
ids.add(id);
|
|
1993
|
+
}
|
|
1994
|
+
return true;
|
|
1995
|
+
}
|
|
1996
|
+
function isTaskAttributionWitnessLike(value, goldMemories) {
|
|
1997
|
+
if (!isObjectRecord(value) || value.schemaVersion !== 1) {
|
|
1998
|
+
return false;
|
|
1999
|
+
}
|
|
2000
|
+
const runtime = value.runtime;
|
|
2001
|
+
if (!isObjectRecord(runtime) || !isNonEmptyString(runtime.qmdCollection) || !isNonEmptyString(runtime.qmdIndex) || !isNonNegativeInteger(runtime.qmdMaxResults) || typeof runtime.attributionThreshold !== "number" || !Number.isFinite(runtime.attributionThreshold) || runtime.attributionThreshold < 0 || runtime.attributionThreshold > 1) {
|
|
2002
|
+
return false;
|
|
2003
|
+
}
|
|
2004
|
+
if (!Array.isArray(value.golds) || goldMemories !== void 0 && value.golds.length !== goldMemories.length) {
|
|
2005
|
+
return false;
|
|
2006
|
+
}
|
|
2007
|
+
for (let index = 0; index < value.golds.length; index += 1) {
|
|
2008
|
+
const gold = value.golds[index];
|
|
2009
|
+
if (!isObjectRecord(gold) || !isNonEmptyString(gold.goldMemory) || goldMemories !== void 0 && gold.goldMemory !== goldMemories[index] || !isUniqueMemoryIdList(gold.storeMemoryIds) || !isUniqueMemoryIdList(gold.oracleMemoryIds)) {
|
|
2010
|
+
return false;
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
if (!Array.isArray(value.retrievals)) {
|
|
2014
|
+
return false;
|
|
2015
|
+
}
|
|
2016
|
+
const sessionIds = /* @__PURE__ */ new Set();
|
|
2017
|
+
for (const retrieval of value.retrievals) {
|
|
2018
|
+
if (!isObjectRecord(retrieval) || !isNonEmptyString(retrieval.sessionId) || sessionIds.has(retrieval.sessionId) || !(retrieval.appliedCap === null || isNonNegativeInteger(retrieval.appliedCap)) || !isUniqueMemoryIdList(retrieval.atCapMemoryIds) || !isUniqueMemoryIdList(retrieval.headroomMemoryIds)) {
|
|
2019
|
+
return false;
|
|
2020
|
+
}
|
|
2021
|
+
sessionIds.add(retrieval.sessionId);
|
|
2022
|
+
const appliedCap = retrieval.appliedCap;
|
|
2023
|
+
const atCapIds = retrieval.atCapMemoryIds;
|
|
2024
|
+
const headroomIds = retrieval.headroomMemoryIds;
|
|
2025
|
+
if (appliedCap === null && (atCapIds !== null || headroomIds !== null) || appliedCap !== null && atCapIds !== null && atCapIds.length > appliedCap || appliedCap !== null && atCapIds !== null && atCapIds.length < appliedCap && headroomIds === null || appliedCap !== null && headroomIds !== null && headroomIds.length > 0 && (atCapIds === null || atCapIds.length !== appliedCap)) {
|
|
2026
|
+
return false;
|
|
2027
|
+
}
|
|
2028
|
+
if (atCapIds !== null && headroomIds !== null) {
|
|
2029
|
+
const atCapSet = new Set(atCapIds);
|
|
2030
|
+
if (headroomIds.some((id) => atCapSet.has(id))) {
|
|
2031
|
+
return false;
|
|
2032
|
+
}
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
return true;
|
|
2036
|
+
}
|
|
1980
2037
|
function isBenchmarkResult(value) {
|
|
1981
2038
|
if (!isObjectRecord(value)) {
|
|
1982
2039
|
return false;
|
|
@@ -2013,6 +2070,9 @@ function isTaskResultLike(value) {
|
|
|
2013
2070
|
if (goldMemories !== void 0 && (!Array.isArray(goldMemories) || !goldMemories.every((item) => typeof item === "string"))) {
|
|
2014
2071
|
return false;
|
|
2015
2072
|
}
|
|
2073
|
+
if (value.attributionWitness !== void 0 && !isTaskAttributionWitnessLike(value.attributionWitness, goldMemories)) {
|
|
2074
|
+
return false;
|
|
2075
|
+
}
|
|
2016
2076
|
return typeof value.taskId === "string" && typeof value.question === "string" && typeof value.expected === "string" && typeof value.actual === "string" && isObjectRecord(value.scores) && Object.values(value.scores).every(isFiniteNumber) && isFiniteNumber(value.latencyMs) && isObjectRecord(tokens) && isFiniteNumber(tokens.input) && isFiniteNumber(tokens.output);
|
|
2017
2077
|
}
|
|
2018
2078
|
function isStoredBenchmarkBaseline(value) {
|
|
@@ -5587,7 +5647,8 @@ import {
|
|
|
5587
5647
|
parseConfig,
|
|
5588
5648
|
parseEntityFile,
|
|
5589
5649
|
serializeEntityFile,
|
|
5590
|
-
StorageManager
|
|
5650
|
+
StorageManager,
|
|
5651
|
+
withRawEntityPageMutation
|
|
5591
5652
|
} from "@remnic/core";
|
|
5592
5653
|
|
|
5593
5654
|
// src/adapters/with-bench-core-memory-source.ts
|
|
@@ -5625,344 +5686,1132 @@ async function withBenchCoreMemorySource(orchestrator, source, task) {
|
|
|
5625
5686
|
}
|
|
5626
5687
|
}
|
|
5627
5688
|
|
|
5628
|
-
// src/
|
|
5629
|
-
|
|
5630
|
-
|
|
5631
|
-
|
|
5632
|
-
|
|
5633
|
-
|
|
5634
|
-
|
|
5635
|
-
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
|
|
5640
|
-
|
|
5641
|
-
|
|
5642
|
-
visibleEnd: Math.max(visibleStart, Math.min(end, returnedChars))
|
|
5643
|
-
};
|
|
5644
|
-
}
|
|
5645
|
-
function projectBenchCoreCapture(snapshot) {
|
|
5646
|
-
return {
|
|
5647
|
-
snapshotId: snapshot.snapshotId,
|
|
5648
|
-
capturedAt: snapshot.capturedAt,
|
|
5649
|
-
...snapshot.traceId === void 0 ? {} : { traceId: snapshot.traceId },
|
|
5650
|
-
budget: { chars: snapshot.budget.chars, used: snapshot.budget.used },
|
|
5651
|
-
filters: snapshot.filters.map(({ name, considered, admitted }) => ({
|
|
5652
|
-
name,
|
|
5653
|
-
considered,
|
|
5654
|
-
admitted
|
|
5655
|
-
})),
|
|
5656
|
-
results: snapshot.results.map((result) => {
|
|
5657
|
-
const scores = result.scoreDecomposition;
|
|
5658
|
-
return {
|
|
5659
|
-
memoryIdRef: {
|
|
5660
|
-
sha256: createHash6("sha256").update(result.memoryId, "utf8").digest("hex"),
|
|
5661
|
-
length: Buffer.byteLength(result.memoryId, "utf8")
|
|
5662
|
-
},
|
|
5663
|
-
servedBy: result.servedBy,
|
|
5664
|
-
scoreDecomposition: {
|
|
5665
|
-
...typeof scores.vector === "number" ? { vector: scores.vector } : {},
|
|
5666
|
-
...typeof scores.bm25 === "number" ? { bm25: scores.bm25 } : {},
|
|
5667
|
-
...typeof scores.importance === "number" ? { importance: scores.importance } : {},
|
|
5668
|
-
...typeof scores.mmrPenalty === "number" ? { mmrPenalty: scores.mmrPenalty } : {},
|
|
5669
|
-
...typeof scores.tierPrior === "number" ? { tierPrior: scores.tierPrior } : {},
|
|
5670
|
-
...typeof scores.reinforcementBoost === "number" ? { reinforcementBoost: scores.reinforcementBoost } : {},
|
|
5671
|
-
final: scores.final
|
|
5672
|
-
},
|
|
5673
|
-
admittedBy: [...result.admittedBy],
|
|
5674
|
-
...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
|
|
5675
|
-
...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
|
|
5676
|
-
...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
|
|
5677
|
-
};
|
|
5678
|
-
})
|
|
5679
|
-
};
|
|
5680
|
-
}
|
|
5681
|
-
function createBenchRecallTraceRecorder(requestedChars) {
|
|
5682
|
-
const sections = [];
|
|
5683
|
-
const pendingSelections = [];
|
|
5684
|
-
const lcmCandidates = [];
|
|
5685
|
-
let composedChars = 0;
|
|
5686
|
-
let coreCapture;
|
|
5687
|
-
const sectionById = (sectionId) => {
|
|
5688
|
-
const section = sections.find((entry) => entry.id === sectionId);
|
|
5689
|
-
if (!section) throw new Error(`Unknown benchmark recall trace section: ${sectionId}`);
|
|
5690
|
-
return section;
|
|
5691
|
-
};
|
|
5692
|
-
const appendRelativeSelection = (sectionId, kind, start, end, lineageStatus, fields = {}) => {
|
|
5693
|
-
const section = sectionById(sectionId);
|
|
5694
|
-
const contentLength = section.contentEnd - section.contentStart;
|
|
5695
|
-
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > contentLength) {
|
|
5696
|
-
throw new Error(
|
|
5697
|
-
`Invalid benchmark recall trace range for ${sectionId}: ${start}..${end}.`
|
|
5698
|
-
);
|
|
5699
|
-
}
|
|
5700
|
-
pendingSelections.push({
|
|
5701
|
-
sectionId,
|
|
5702
|
-
kind,
|
|
5703
|
-
lineageStatus,
|
|
5704
|
-
composedStart: section.contentStart + start,
|
|
5705
|
-
composedEnd: section.contentStart + end,
|
|
5706
|
-
...fields
|
|
5707
|
-
});
|
|
5708
|
-
};
|
|
5709
|
-
return {
|
|
5710
|
-
appendSection(id, source, renderedLength) {
|
|
5711
|
-
if (!Number.isSafeInteger(renderedLength) || renderedLength < 0) {
|
|
5712
|
-
throw new Error("Benchmark recall trace section length must be a non-negative integer.");
|
|
5713
|
-
}
|
|
5714
|
-
if (sections.some((section) => section.id === id)) {
|
|
5715
|
-
throw new Error(`Duplicate benchmark recall trace section: ${id}`);
|
|
5716
|
-
}
|
|
5717
|
-
const separatorStart = composedChars;
|
|
5718
|
-
const contentStart = composedChars + (sections.length === 0 ? 0 : 2);
|
|
5719
|
-
const contentEnd = contentStart + renderedLength;
|
|
5720
|
-
sections.push({
|
|
5721
|
-
id,
|
|
5722
|
-
source,
|
|
5723
|
-
separatorStart,
|
|
5724
|
-
contentStart,
|
|
5725
|
-
contentEnd,
|
|
5726
|
-
composedStart: separatorStart,
|
|
5727
|
-
composedEnd: contentEnd,
|
|
5728
|
-
visibleStart: 0,
|
|
5729
|
-
visibleEnd: 0,
|
|
5730
|
-
visibleChars: 0
|
|
5731
|
-
});
|
|
5732
|
-
composedChars = contentEnd;
|
|
5733
|
-
},
|
|
5734
|
-
recordEvidenceSelections(sectionId, receipts) {
|
|
5735
|
-
for (const receipt of receipts) {
|
|
5736
|
-
appendRelativeSelection(
|
|
5737
|
-
sectionId,
|
|
5738
|
-
"evidence-block",
|
|
5739
|
-
receipt.blockStart,
|
|
5740
|
-
receipt.blockEnd,
|
|
5741
|
-
receipt.item.archiveRowId === void 0 ? "unavailable" : "exact",
|
|
5742
|
-
{
|
|
5743
|
-
...receipt.item.archiveRowId === void 0 ? {} : { archiveRowIds: [receipt.item.archiveRowId] },
|
|
5744
|
-
...receipt.item.turnIndex === void 0 ? {} : { turnIndex: receipt.item.turnIndex },
|
|
5745
|
-
...receipt.item.role === void 0 ? {} : { role: receipt.item.role },
|
|
5746
|
-
...receipt.item.score === void 0 ? {} : { score: receipt.item.score }
|
|
5747
|
-
}
|
|
5748
|
-
);
|
|
5749
|
-
}
|
|
5750
|
-
},
|
|
5751
|
-
recordTrajectorySelections(sectionId, receipts) {
|
|
5752
|
-
for (const receipt of receipts) {
|
|
5753
|
-
appendRelativeSelection(
|
|
5754
|
-
sectionId,
|
|
5755
|
-
"trajectory-line",
|
|
5756
|
-
receipt.lineStart,
|
|
5757
|
-
receipt.lineEnd,
|
|
5758
|
-
receipt.lineageStatus,
|
|
5759
|
-
{
|
|
5760
|
-
archiveRowIds: [
|
|
5761
|
-
...receipt.actionArchiveRowIds,
|
|
5762
|
-
...receipt.observationArchiveRowIds
|
|
5763
|
-
]
|
|
5764
|
-
}
|
|
5765
|
-
);
|
|
5766
|
-
}
|
|
5767
|
-
},
|
|
5768
|
-
recordSummarySelections(sectionId, receipts) {
|
|
5769
|
-
for (const receipt of receipts) {
|
|
5770
|
-
appendRelativeSelection(
|
|
5771
|
-
sectionId,
|
|
5772
|
-
"lcm-summary",
|
|
5773
|
-
receipt.entryStart,
|
|
5774
|
-
receipt.entryEnd,
|
|
5775
|
-
"exact",
|
|
5776
|
-
{
|
|
5777
|
-
summary: {
|
|
5778
|
-
id: receipt.id,
|
|
5779
|
-
depth: receipt.depth,
|
|
5780
|
-
msgStart: receipt.msgStart,
|
|
5781
|
-
msgEnd: receipt.msgEnd
|
|
5782
|
-
}
|
|
5783
|
-
}
|
|
5784
|
-
);
|
|
5785
|
-
}
|
|
5786
|
-
},
|
|
5787
|
-
recordRawRow(sectionId, range, row) {
|
|
5788
|
-
const archiveRowId = lcmArchiveRowId(row);
|
|
5789
|
-
appendRelativeSelection(
|
|
5790
|
-
sectionId,
|
|
5791
|
-
"raw-row",
|
|
5792
|
-
range.start,
|
|
5793
|
-
range.end,
|
|
5794
|
-
archiveRowId === void 0 ? "unavailable" : "exact",
|
|
5795
|
-
{
|
|
5796
|
-
...archiveRowId === void 0 ? {} : { archiveRowIds: [archiveRowId] },
|
|
5797
|
-
turnIndex: row.turn_index,
|
|
5798
|
-
role: row.role
|
|
5799
|
-
}
|
|
5800
|
-
);
|
|
5801
|
-
},
|
|
5802
|
-
recordLcmCandidate(candidate) {
|
|
5803
|
-
lcmCandidates.push({ ...candidate });
|
|
5804
|
-
},
|
|
5805
|
-
recordCoreCapture(snapshot) {
|
|
5806
|
-
coreCapture = snapshot ? projectBenchCoreCapture(snapshot) : void 0;
|
|
5807
|
-
},
|
|
5808
|
-
finalize(returnedChars) {
|
|
5809
|
-
const normalizedReturnedChars = Math.max(0, Math.min(returnedChars, composedChars));
|
|
5810
|
-
return {
|
|
5811
|
-
schemaVersion: 1,
|
|
5812
|
-
sensitivity: {
|
|
5813
|
-
classification: "restricted",
|
|
5814
|
-
contentEncoding: "sha256+length",
|
|
5815
|
-
containsGold: false
|
|
5816
|
-
},
|
|
5817
|
-
sections: sections.map((section) => {
|
|
5818
|
-
const visible = visibleRange(
|
|
5819
|
-
section.composedStart,
|
|
5820
|
-
section.composedEnd,
|
|
5821
|
-
normalizedReturnedChars
|
|
5822
|
-
);
|
|
5823
|
-
return {
|
|
5824
|
-
...section,
|
|
5825
|
-
...visible,
|
|
5826
|
-
visibleChars: visible.visibleEnd - visible.visibleStart
|
|
5827
|
-
};
|
|
5828
|
-
}),
|
|
5829
|
-
selections: pendingSelections.map((selection) => ({
|
|
5830
|
-
...selection,
|
|
5831
|
-
...visibleRange(
|
|
5832
|
-
selection.composedStart,
|
|
5833
|
-
selection.composedEnd,
|
|
5834
|
-
normalizedReturnedChars
|
|
5835
|
-
)
|
|
5836
|
-
})),
|
|
5837
|
-
lcmCandidates: lcmCandidates.map((candidate) => ({ ...candidate })),
|
|
5838
|
-
...coreCapture === void 0 ? {} : { coreCapture },
|
|
5839
|
-
budget: {
|
|
5840
|
-
requestedChars,
|
|
5841
|
-
composedChars,
|
|
5842
|
-
returnedChars: normalizedReturnedChars,
|
|
5843
|
-
truncated: normalizedReturnedChars < composedChars
|
|
5844
|
-
}
|
|
5845
|
-
};
|
|
5846
|
-
}
|
|
5847
|
-
};
|
|
5848
|
-
}
|
|
5849
|
-
|
|
5850
|
-
// src/recall-budget.ts
|
|
5851
|
-
var DEFAULT_BENCH_RECALL_BUDGET_CHARS = 24e3;
|
|
5852
|
-
var MAX_COMBINED_RECALL_BUDGET_CHARS = 36e3;
|
|
5853
|
-
function benchmarkRecallBudgetForSessionCount(sessionCount) {
|
|
5854
|
-
if (!Number.isInteger(sessionCount) || sessionCount <= 0) {
|
|
5855
|
-
return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
|
|
5856
|
-
}
|
|
5857
|
-
if (sessionCount === 1) {
|
|
5858
|
-
return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
|
|
5859
|
-
}
|
|
5860
|
-
return Math.floor(MAX_COMBINED_RECALL_BUDGET_CHARS / sessionCount);
|
|
5861
|
-
}
|
|
5862
|
-
|
|
5863
|
-
// src/adapters/remnic-adapter.ts
|
|
5864
|
-
var DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE = 0.34;
|
|
5865
|
-
var ANSWER_SUPPORT_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
5866
|
-
"about",
|
|
5867
|
-
"after",
|
|
5868
|
-
"again",
|
|
5869
|
-
"also",
|
|
5870
|
-
"answer",
|
|
5871
|
-
"before",
|
|
5872
|
-
"being",
|
|
5873
|
-
"could",
|
|
5874
|
-
"does",
|
|
5689
|
+
// src/attribution.ts
|
|
5690
|
+
var DEFAULT_ATTRIBUTION_THRESHOLD = 0.6;
|
|
5691
|
+
var DEFAULT_STOPWORDS = /* @__PURE__ */ new Set([
|
|
5692
|
+
"a",
|
|
5693
|
+
"an",
|
|
5694
|
+
"the",
|
|
5695
|
+
"in",
|
|
5696
|
+
"on",
|
|
5697
|
+
"at",
|
|
5698
|
+
"to",
|
|
5699
|
+
"for",
|
|
5700
|
+
"of",
|
|
5701
|
+
"with",
|
|
5702
|
+
"by",
|
|
5875
5703
|
"from",
|
|
5876
|
-
"
|
|
5877
|
-
"
|
|
5704
|
+
"up",
|
|
5705
|
+
"about",
|
|
5878
5706
|
"into",
|
|
5879
|
-
"
|
|
5880
|
-
"
|
|
5881
|
-
"
|
|
5882
|
-
"
|
|
5883
|
-
"
|
|
5884
|
-
"
|
|
5885
|
-
"
|
|
5886
|
-
"
|
|
5887
|
-
"
|
|
5707
|
+
"through",
|
|
5708
|
+
"during",
|
|
5709
|
+
"before",
|
|
5710
|
+
"after",
|
|
5711
|
+
"above",
|
|
5712
|
+
"below",
|
|
5713
|
+
"and",
|
|
5714
|
+
"or",
|
|
5715
|
+
"but",
|
|
5716
|
+
"if",
|
|
5717
|
+
"then",
|
|
5718
|
+
"else",
|
|
5719
|
+
"when",
|
|
5720
|
+
"where",
|
|
5721
|
+
"why",
|
|
5722
|
+
"how",
|
|
5723
|
+
"all",
|
|
5724
|
+
"any",
|
|
5725
|
+
"both",
|
|
5726
|
+
"each",
|
|
5727
|
+
"few",
|
|
5728
|
+
"more",
|
|
5729
|
+
"most",
|
|
5730
|
+
"other",
|
|
5731
|
+
"some",
|
|
5732
|
+
"such",
|
|
5733
|
+
"no",
|
|
5734
|
+
"nor",
|
|
5735
|
+
"not",
|
|
5736
|
+
"only",
|
|
5737
|
+
"own",
|
|
5738
|
+
"same",
|
|
5739
|
+
"so",
|
|
5740
|
+
"than",
|
|
5741
|
+
"too",
|
|
5742
|
+
"very",
|
|
5743
|
+
"this",
|
|
5888
5744
|
"that",
|
|
5889
|
-
"their",
|
|
5890
|
-
"there",
|
|
5891
5745
|
"these",
|
|
5892
|
-
"they",
|
|
5893
|
-
"this",
|
|
5894
5746
|
"those",
|
|
5895
|
-
"
|
|
5896
|
-
"
|
|
5897
|
-
"
|
|
5898
|
-
"
|
|
5899
|
-
"
|
|
5900
|
-
"
|
|
5901
|
-
"
|
|
5902
|
-
"
|
|
5903
|
-
"
|
|
5904
|
-
"
|
|
5747
|
+
"it",
|
|
5748
|
+
"its",
|
|
5749
|
+
"is",
|
|
5750
|
+
"are",
|
|
5751
|
+
"was",
|
|
5752
|
+
"were",
|
|
5753
|
+
"be",
|
|
5754
|
+
"been",
|
|
5755
|
+
"being",
|
|
5756
|
+
"have",
|
|
5757
|
+
"has",
|
|
5758
|
+
"had",
|
|
5759
|
+
"do",
|
|
5760
|
+
"does",
|
|
5761
|
+
"did"
|
|
5905
5762
|
]);
|
|
5906
|
-
function
|
|
5907
|
-
const
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5763
|
+
function extractContentWords(text) {
|
|
5764
|
+
const cleaned = text.toLowerCase().replace(/[^a-z0-9\s]/g, " ");
|
|
5765
|
+
const tokens = cleaned.split(/\s+/).filter(Boolean);
|
|
5766
|
+
return tokens.filter((t) => !DEFAULT_STOPWORDS.has(t));
|
|
5767
|
+
}
|
|
5768
|
+
function lexicalSimilarity(a, b) {
|
|
5769
|
+
const goldWords = extractContentWords(a);
|
|
5770
|
+
if (goldWords.length === 0) {
|
|
5771
|
+
return 0;
|
|
5912
5772
|
}
|
|
5913
|
-
|
|
5773
|
+
const candWords = new Set(extractContentWords(b));
|
|
5774
|
+
let matchCount = 0;
|
|
5775
|
+
for (const word of goldWords) {
|
|
5776
|
+
if (candWords.has(word)) {
|
|
5777
|
+
matchCount++;
|
|
5778
|
+
}
|
|
5779
|
+
}
|
|
5780
|
+
return matchCount / goldWords.length;
|
|
5914
5781
|
}
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5920
|
-
|
|
5921
|
-
|
|
5922
|
-
|
|
5782
|
+
var CLASS_RANK = {
|
|
5783
|
+
extraction_miss: 1,
|
|
5784
|
+
index_miss: 2,
|
|
5785
|
+
retrieval_miss: 3,
|
|
5786
|
+
use_miss: 4,
|
|
5787
|
+
unattributed: 5
|
|
5788
|
+
};
|
|
5789
|
+
var RETRIEVAL_STAGE_RANK = {
|
|
5790
|
+
cap: 1,
|
|
5791
|
+
rank: 2,
|
|
5792
|
+
filter: 3,
|
|
5793
|
+
unknown: 4
|
|
5794
|
+
};
|
|
5795
|
+
function computeOverallLabel(golds) {
|
|
5796
|
+
if (golds.length === 0) {
|
|
5797
|
+
return { class: "unattributed", reason: "no gold memories" };
|
|
5923
5798
|
}
|
|
5924
|
-
|
|
5925
|
-
|
|
5926
|
-
|
|
5799
|
+
let bestGold = golds[0];
|
|
5800
|
+
for (let i = 1; i < golds.length; i++) {
|
|
5801
|
+
const current = golds[i];
|
|
5802
|
+
const bestRank = CLASS_RANK[bestGold.label.class];
|
|
5803
|
+
const currRank = CLASS_RANK[current.label.class];
|
|
5804
|
+
if (currRank < bestRank) {
|
|
5805
|
+
bestGold = current;
|
|
5806
|
+
} else if (currRank === bestRank && current.label.class === "retrieval_miss") {
|
|
5807
|
+
const bestRetRank = RETRIEVAL_STAGE_RANK[bestGold.label.retrievalStage ?? "unknown"];
|
|
5808
|
+
const currRetRank = RETRIEVAL_STAGE_RANK[current.label.retrievalStage ?? "unknown"];
|
|
5809
|
+
if (currRetRank < bestRetRank) {
|
|
5810
|
+
bestGold = current;
|
|
5811
|
+
}
|
|
5812
|
+
}
|
|
5813
|
+
}
|
|
5814
|
+
return { ...bestGold.label };
|
|
5927
5815
|
}
|
|
5928
|
-
function
|
|
5929
|
-
return
|
|
5816
|
+
function isDiagnosticScore(name) {
|
|
5817
|
+
return name.endsWith("_agreement") || name.includes("_id_leak") || name === "search_hits";
|
|
5930
5818
|
}
|
|
5931
|
-
function
|
|
5932
|
-
if (
|
|
5933
|
-
|
|
5934
|
-
const base = value.slice(0, -2);
|
|
5935
|
-
return /[vs]$/.test(base) ? `${base}e` : base;
|
|
5819
|
+
function isTaskFailed(task) {
|
|
5820
|
+
if (!task.scores || Object.keys(task.scores).length === 0) {
|
|
5821
|
+
return true;
|
|
5936
5822
|
}
|
|
5937
|
-
if (
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
return [...new Set(
|
|
5943
|
-
(value.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}_-]{2,}/gu) ?? []).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term)).map(normalizeSupportToken).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term) && !/^\d+$/.test(term))
|
|
5944
|
-
)];
|
|
5823
|
+
if ("overall" in task.scores && typeof task.scores.overall === "number") {
|
|
5824
|
+
return task.scores.overall < 1;
|
|
5825
|
+
}
|
|
5826
|
+
const primaryScores = Object.entries(task.scores).filter(([name, score]) => typeof score === "number" && !isDiagnosticScore(name)).map(([, score]) => score);
|
|
5827
|
+
return primaryScores.length === 0 || Math.min(...primaryScores) < 1;
|
|
5945
5828
|
}
|
|
5946
|
-
function
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5829
|
+
function withMemoizedListMemories(env) {
|
|
5830
|
+
let cache = null;
|
|
5831
|
+
return {
|
|
5832
|
+
...env,
|
|
5833
|
+
listMemories() {
|
|
5834
|
+
if (!cache) {
|
|
5835
|
+
cache = env.listMemories();
|
|
5836
|
+
}
|
|
5837
|
+
return cache;
|
|
5838
|
+
}
|
|
5839
|
+
};
|
|
5951
5840
|
}
|
|
5952
|
-
function
|
|
5953
|
-
if (
|
|
5954
|
-
|
|
5841
|
+
async function attributeGoldMemory(goldStatement, question, env, options = {}, recalledText) {
|
|
5842
|
+
if (options.threshold !== void 0) {
|
|
5843
|
+
if (typeof options.threshold !== "number" || !Number.isFinite(options.threshold) || options.threshold < 0 || options.threshold > 1) {
|
|
5844
|
+
throw new RangeError("attribution threshold must be a finite number between 0 and 1");
|
|
5845
|
+
}
|
|
5955
5846
|
}
|
|
5956
|
-
const
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
}
|
|
5847
|
+
const threshold = options.threshold ?? DEFAULT_ATTRIBUTION_THRESHOLD;
|
|
5848
|
+
const simFn = options.similarity ?? lexicalSimilarity;
|
|
5849
|
+
const goldInRecalledText = typeof recalledText === "string" && simFn(goldStatement, recalledText) >= threshold;
|
|
5850
|
+
const stages = {
|
|
5851
|
+
extraction: { status: "unavailable" },
|
|
5852
|
+
index: { status: "unavailable" },
|
|
5853
|
+
retrieval: { status: "unavailable" },
|
|
5854
|
+
use: { status: "unavailable" }
|
|
5855
|
+
};
|
|
5856
|
+
let memories = [];
|
|
5857
|
+
let extractionRan = false;
|
|
5858
|
+
let extractionErrorDetail;
|
|
5859
|
+
if (typeof env.listMemories === "function") {
|
|
5860
|
+
try {
|
|
5861
|
+
memories = await env.listMemories();
|
|
5862
|
+
extractionRan = true;
|
|
5863
|
+
} catch {
|
|
5864
|
+
extractionRan = false;
|
|
5865
|
+
extractionErrorDetail = "listMemories failed";
|
|
5866
|
+
}
|
|
5962
5867
|
}
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
|
|
5868
|
+
let bestSim = -1;
|
|
5869
|
+
let matchedMem = null;
|
|
5870
|
+
if (extractionRan) {
|
|
5871
|
+
if (memories.length === 0) {
|
|
5872
|
+
if (goldInRecalledText) {
|
|
5873
|
+
const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
|
|
5874
|
+
stages.extraction = { status: "pass", detail: impliedDetail };
|
|
5875
|
+
stages.index = { status: "pass", detail: impliedDetail };
|
|
5876
|
+
stages.retrieval = { status: "pass", detail: impliedDetail };
|
|
5877
|
+
stages.use = {
|
|
5878
|
+
status: "fail",
|
|
5879
|
+
detail: "Gold memory present in context but answer was incorrect"
|
|
5880
|
+
};
|
|
5881
|
+
return {
|
|
5882
|
+
goldMemory: goldStatement,
|
|
5883
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
5884
|
+
stages
|
|
5885
|
+
};
|
|
5886
|
+
}
|
|
5887
|
+
const detail = "store contains no memories";
|
|
5888
|
+
stages.extraction = { status: "fail", detail };
|
|
5889
|
+
stages.index = { status: "unavailable", detail: "not reached" };
|
|
5890
|
+
stages.retrieval = { status: "unavailable", detail: "not reached" };
|
|
5891
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
5892
|
+
return {
|
|
5893
|
+
goldMemory: goldStatement,
|
|
5894
|
+
label: { class: "extraction_miss", reason: detail },
|
|
5895
|
+
stages
|
|
5896
|
+
};
|
|
5897
|
+
}
|
|
5898
|
+
for (const mem of memories) {
|
|
5899
|
+
const sim = simFn(goldStatement, mem.content);
|
|
5900
|
+
if (sim > bestSim) {
|
|
5901
|
+
bestSim = sim;
|
|
5902
|
+
matchedMem = mem;
|
|
5903
|
+
}
|
|
5904
|
+
}
|
|
5905
|
+
if (bestSim < threshold || !matchedMem) {
|
|
5906
|
+
if (goldInRecalledText) {
|
|
5907
|
+
const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
|
|
5908
|
+
stages.extraction = { status: "pass", detail: impliedDetail };
|
|
5909
|
+
stages.index = { status: "pass", detail: impliedDetail };
|
|
5910
|
+
stages.retrieval = { status: "pass", detail: impliedDetail };
|
|
5911
|
+
stages.use = {
|
|
5912
|
+
status: "fail",
|
|
5913
|
+
detail: "Gold memory present in context but answer was incorrect"
|
|
5914
|
+
};
|
|
5915
|
+
return {
|
|
5916
|
+
goldMemory: goldStatement,
|
|
5917
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
5918
|
+
stages
|
|
5919
|
+
};
|
|
5920
|
+
}
|
|
5921
|
+
const detail = `Best similarity ${bestSim >= 0 ? bestSim.toFixed(3) : 0} below threshold ${threshold}`;
|
|
5922
|
+
stages.extraction = { status: "fail", detail };
|
|
5923
|
+
stages.index = { status: "unavailable", detail: "not reached" };
|
|
5924
|
+
stages.retrieval = { status: "unavailable", detail: "not reached" };
|
|
5925
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
5926
|
+
return {
|
|
5927
|
+
goldMemory: goldStatement,
|
|
5928
|
+
label: { class: "extraction_miss", reason: detail },
|
|
5929
|
+
stages
|
|
5930
|
+
};
|
|
5931
|
+
}
|
|
5932
|
+
stages.extraction = {
|
|
5933
|
+
status: "pass",
|
|
5934
|
+
detail: `Matched memory ${matchedMem.id} (sim ${bestSim.toFixed(3)})`
|
|
5935
|
+
};
|
|
5936
|
+
} else {
|
|
5937
|
+
stages.extraction = {
|
|
5938
|
+
status: "unavailable",
|
|
5939
|
+
detail: extractionErrorDetail ?? "listMemories unavailable"
|
|
5940
|
+
};
|
|
5941
|
+
}
|
|
5942
|
+
const matchedMemoryId = matchedMem ? matchedMem.id : void 0;
|
|
5943
|
+
const recallLimit = env.recallLimit;
|
|
5944
|
+
const replayLimit = env.replayLimit ?? Math.max(25, recallLimit * 5);
|
|
5945
|
+
let indexCheckPassed = false;
|
|
5946
|
+
let indexCheckFailed = false;
|
|
5947
|
+
if (typeof env.oracleSearch === "function" && extractionRan) {
|
|
5948
|
+
try {
|
|
5949
|
+
const oracleResults = await env.oracleSearch(goldStatement, replayLimit);
|
|
5950
|
+
const idMatched = matchedMemoryId ? oracleResults.some((r) => r.id === matchedMemoryId) : false;
|
|
5951
|
+
if (idMatched) {
|
|
5952
|
+
indexCheckPassed = true;
|
|
5953
|
+
} else {
|
|
5954
|
+
const memMap = new Map(memories.map((m) => [m.id, m]));
|
|
5955
|
+
indexCheckPassed = oracleResults.some((r) => {
|
|
5956
|
+
const mem = memMap.get(r.id);
|
|
5957
|
+
return mem ? simFn(goldStatement, mem.content) >= threshold : false;
|
|
5958
|
+
});
|
|
5959
|
+
}
|
|
5960
|
+
if (indexCheckPassed) {
|
|
5961
|
+
stages.index = { status: "pass", detail: "Found in oracle search" };
|
|
5962
|
+
} else {
|
|
5963
|
+
indexCheckFailed = true;
|
|
5964
|
+
stages.index = { status: "fail", detail: "Not found in oracle search" };
|
|
5965
|
+
}
|
|
5966
|
+
} catch {
|
|
5967
|
+
stages.index = { status: "unavailable", detail: "oracleSearch threw error" };
|
|
5968
|
+
}
|
|
5969
|
+
} else if (typeof env.oracleSearch === "function") {
|
|
5970
|
+
stages.index = {
|
|
5971
|
+
status: "unavailable",
|
|
5972
|
+
detail: "extraction check unavailable; oracle result would be ambiguous"
|
|
5973
|
+
};
|
|
5974
|
+
} else {
|
|
5975
|
+
stages.index = { status: "unavailable", detail: "index check unavailable" };
|
|
5976
|
+
}
|
|
5977
|
+
let retrievalCheckPassed = false;
|
|
5978
|
+
let retrievalStageMiss = void 0;
|
|
5979
|
+
if (typeof env.recall === "function") {
|
|
5980
|
+
try {
|
|
5981
|
+
const recallResults = await env.recall(question, recallLimit);
|
|
5982
|
+
const isGoldInRecall = recallResults.some(
|
|
5983
|
+
(m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
|
|
5984
|
+
);
|
|
5985
|
+
if (isGoldInRecall) {
|
|
5986
|
+
retrievalCheckPassed = true;
|
|
5987
|
+
stages.retrieval = { status: "pass", detail: `Recalled within recallLimit ${recallLimit}` };
|
|
5988
|
+
} else {
|
|
5989
|
+
const replayResults = await env.recall(question, replayLimit);
|
|
5990
|
+
const replayIndex = replayResults.findIndex(
|
|
5991
|
+
(m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
|
|
5992
|
+
);
|
|
5993
|
+
if (replayIndex >= 0) {
|
|
5994
|
+
const rank = replayIndex + 1;
|
|
5995
|
+
retrievalStageMiss = "cap";
|
|
5996
|
+
stages.retrieval = {
|
|
5997
|
+
status: "fail",
|
|
5998
|
+
detail: `Rank ${rank} exceeds recallLimit ${recallLimit}`
|
|
5999
|
+
};
|
|
6000
|
+
} else {
|
|
6001
|
+
retrievalStageMiss = "unknown";
|
|
6002
|
+
stages.retrieval = {
|
|
6003
|
+
status: "fail",
|
|
6004
|
+
detail: `absent from recall at replayLimit ${replayLimit}; filter vs rank indistinguishable without candidate-stage evidence`
|
|
6005
|
+
};
|
|
6006
|
+
}
|
|
6007
|
+
}
|
|
6008
|
+
} catch {
|
|
6009
|
+
stages.retrieval = { status: "unavailable", detail: "recall threw error" };
|
|
6010
|
+
}
|
|
6011
|
+
} else {
|
|
6012
|
+
stages.retrieval = { status: "unavailable", detail: "retrieval check unavailable" };
|
|
6013
|
+
}
|
|
6014
|
+
if (!retrievalCheckPassed && goldInRecalledText) {
|
|
6015
|
+
retrievalCheckPassed = true;
|
|
6016
|
+
stages.retrieval = { status: "pass", detail: "Found in recalledText context" };
|
|
6017
|
+
}
|
|
6018
|
+
if (retrievalCheckPassed) {
|
|
6019
|
+
if (indexCheckFailed) {
|
|
6020
|
+
stages.index = { status: "pass", detail: "implied pass from retrieval (oracle query missed)" };
|
|
6021
|
+
indexCheckPassed = true;
|
|
6022
|
+
} else if (stages.index.status === "unavailable") {
|
|
6023
|
+
stages.index = { status: "pass", detail: "implied pass from retrieval" };
|
|
6024
|
+
indexCheckPassed = true;
|
|
6025
|
+
}
|
|
6026
|
+
}
|
|
6027
|
+
if (retrievalCheckPassed) {
|
|
6028
|
+
stages.use = {
|
|
6029
|
+
status: "fail",
|
|
6030
|
+
detail: "Gold memory present in context but answer was incorrect"
|
|
6031
|
+
};
|
|
6032
|
+
return {
|
|
6033
|
+
goldMemory: goldStatement,
|
|
6034
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
6035
|
+
stages
|
|
6036
|
+
};
|
|
6037
|
+
}
|
|
6038
|
+
if (stages.extraction.status === "pass" && indexCheckFailed) {
|
|
6039
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6040
|
+
return {
|
|
6041
|
+
goldMemory: goldStatement,
|
|
6042
|
+
label: { class: "index_miss", reason: "Gold statement missing from search index" },
|
|
6043
|
+
stages
|
|
6044
|
+
};
|
|
6045
|
+
}
|
|
6046
|
+
if (stages.extraction.status === "pass" && stages.index.status === "pass" && stages.retrieval.status === "fail") {
|
|
6047
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6048
|
+
return {
|
|
6049
|
+
goldMemory: goldStatement,
|
|
6050
|
+
label: {
|
|
6051
|
+
class: "retrieval_miss",
|
|
6052
|
+
retrievalStage: retrievalStageMiss ?? "unknown",
|
|
6053
|
+
reason: stages.retrieval.detail
|
|
6054
|
+
},
|
|
6055
|
+
stages
|
|
6056
|
+
};
|
|
6057
|
+
}
|
|
6058
|
+
const missingReason = stages.extraction.status === "unavailable" ? `extraction check unavailable (${stages.extraction.detail})` : stages.index.status === "unavailable" && stages.retrieval.status === "unavailable" ? "index/retrieval checks unavailable in this attribution environment" : stages.index.status === "unavailable" ? "index check unavailable; a retrieval miss cannot be isolated from an index miss" : "retrieval check unavailable";
|
|
6059
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6060
|
+
return {
|
|
6061
|
+
goldMemory: goldStatement,
|
|
6062
|
+
label: { class: "unattributed", reason: missingReason },
|
|
6063
|
+
stages
|
|
6064
|
+
};
|
|
6065
|
+
}
|
|
6066
|
+
function attributeGoldMemoryFromWitness(goldStatement, goldWitness, retrievals, witnessThreshold, options, recalledText) {
|
|
6067
|
+
const threshold = options.threshold ?? DEFAULT_ATTRIBUTION_THRESHOLD;
|
|
6068
|
+
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
|
|
6069
|
+
throw new RangeError("attribution threshold must be a finite number between 0 and 1");
|
|
6070
|
+
}
|
|
6071
|
+
const similarity = options.similarity ?? lexicalSimilarity;
|
|
6072
|
+
const stages = {
|
|
6073
|
+
extraction: { status: "unavailable" },
|
|
6074
|
+
index: { status: "unavailable" },
|
|
6075
|
+
retrieval: { status: "unavailable" },
|
|
6076
|
+
use: { status: "unavailable" }
|
|
6077
|
+
};
|
|
6078
|
+
if (typeof recalledText === "string" && similarity(goldStatement, recalledText) >= threshold) {
|
|
6079
|
+
const detail = "implied pass from recalled context";
|
|
6080
|
+
stages.extraction = { status: "pass", detail };
|
|
6081
|
+
stages.index = { status: "pass", detail };
|
|
6082
|
+
stages.retrieval = { status: "pass", detail: "Found in recalledText context" };
|
|
6083
|
+
stages.use = { status: "fail", detail: "Gold memory present in context but answer was incorrect" };
|
|
6084
|
+
return {
|
|
6085
|
+
goldMemory: goldStatement,
|
|
6086
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
6087
|
+
stages
|
|
6088
|
+
};
|
|
6089
|
+
}
|
|
6090
|
+
if (options.similarity || threshold !== witnessThreshold) {
|
|
6091
|
+
const detail = "stored extraction witness uses a different similarity policy";
|
|
6092
|
+
stages.extraction = { status: "unavailable", detail };
|
|
6093
|
+
stages.index = { status: "unavailable", detail: "extraction check unavailable" };
|
|
6094
|
+
stages.retrieval = { status: "unavailable", detail: "extraction check unavailable" };
|
|
6095
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6096
|
+
return {
|
|
6097
|
+
goldMemory: goldStatement,
|
|
6098
|
+
label: { class: "unattributed", reason: detail },
|
|
6099
|
+
stages
|
|
6100
|
+
};
|
|
6101
|
+
}
|
|
6102
|
+
const storeIds = goldWitness.storeMemoryIds;
|
|
6103
|
+
if (storeIds === null) {
|
|
6104
|
+
stages.extraction = { status: "unavailable", detail: "stored extraction witness unavailable" };
|
|
6105
|
+
stages.index = { status: "unavailable", detail: "extraction check unavailable" };
|
|
6106
|
+
stages.retrieval = { status: "unavailable", detail: "extraction check unavailable" };
|
|
6107
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6108
|
+
return {
|
|
6109
|
+
goldMemory: goldStatement,
|
|
6110
|
+
label: { class: "unattributed", reason: "extraction check unavailable (stored witness)" },
|
|
6111
|
+
stages
|
|
6112
|
+
};
|
|
6113
|
+
}
|
|
6114
|
+
if (storeIds.length === 0) {
|
|
6115
|
+
const detail = "stored witness found no matching memory";
|
|
6116
|
+
stages.extraction = { status: "fail", detail };
|
|
6117
|
+
stages.index = { status: "unavailable", detail: "not reached" };
|
|
6118
|
+
stages.retrieval = { status: "unavailable", detail: "not reached" };
|
|
6119
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6120
|
+
return {
|
|
6121
|
+
goldMemory: goldStatement,
|
|
6122
|
+
label: { class: "extraction_miss", reason: detail },
|
|
6123
|
+
stages
|
|
6124
|
+
};
|
|
6125
|
+
}
|
|
6126
|
+
const storeIdSet = new Set(storeIds);
|
|
6127
|
+
stages.extraction = {
|
|
6128
|
+
status: "pass",
|
|
6129
|
+
detail: `Stored witness matched ${storeIds.length} memory id${storeIds.length === 1 ? "" : "s"}`
|
|
6130
|
+
};
|
|
6131
|
+
const oracleIds = goldWitness.oracleMemoryIds;
|
|
6132
|
+
let indexPassed = false;
|
|
6133
|
+
let indexFailed = false;
|
|
6134
|
+
if (oracleIds === null) {
|
|
6135
|
+
stages.index = { status: "unavailable", detail: "stored oracle witness unavailable" };
|
|
6136
|
+
} else if (oracleIds.some((id) => storeIdSet.has(id))) {
|
|
6137
|
+
indexPassed = true;
|
|
6138
|
+
stages.index = { status: "pass", detail: "Found in stored oracle witness" };
|
|
6139
|
+
} else {
|
|
6140
|
+
indexFailed = true;
|
|
6141
|
+
stages.index = { status: "fail", detail: "Not found in stored oracle witness" };
|
|
6142
|
+
}
|
|
6143
|
+
let appliedHit;
|
|
6144
|
+
let atCapUnavailable = retrievals.length === 0;
|
|
6145
|
+
for (const retrieval of retrievals) {
|
|
6146
|
+
if (retrieval.atCapMemoryIds === null) {
|
|
6147
|
+
atCapUnavailable = true;
|
|
6148
|
+
continue;
|
|
6149
|
+
}
|
|
6150
|
+
const index = retrieval.atCapMemoryIds.findIndex((id) => storeIdSet.has(id));
|
|
6151
|
+
if (index >= 0) {
|
|
6152
|
+
appliedHit = { sessionId: retrieval.sessionId, rank: index + 1 };
|
|
6153
|
+
break;
|
|
6154
|
+
}
|
|
6155
|
+
}
|
|
6156
|
+
let headroomHit;
|
|
6157
|
+
if (!appliedHit) {
|
|
6158
|
+
for (const retrieval of retrievals) {
|
|
6159
|
+
if (retrieval.headroomMemoryIds === null || retrieval.appliedCap === null) {
|
|
6160
|
+
continue;
|
|
6161
|
+
}
|
|
6162
|
+
const index = retrieval.headroomMemoryIds.findIndex((id) => storeIdSet.has(id));
|
|
6163
|
+
if (index >= 0) {
|
|
6164
|
+
headroomHit = {
|
|
6165
|
+
sessionId: retrieval.sessionId,
|
|
6166
|
+
rank: retrieval.appliedCap + index + 1,
|
|
6167
|
+
appliedCap: retrieval.appliedCap
|
|
6168
|
+
};
|
|
6169
|
+
break;
|
|
6170
|
+
}
|
|
6171
|
+
}
|
|
6172
|
+
}
|
|
6173
|
+
if (appliedHit) {
|
|
6174
|
+
stages.retrieval = {
|
|
6175
|
+
status: "pass",
|
|
6176
|
+
detail: `Found at rank ${appliedHit.rank} in stored session ${appliedHit.sessionId}`
|
|
6177
|
+
};
|
|
6178
|
+
if (!indexPassed) {
|
|
6179
|
+
indexPassed = true;
|
|
6180
|
+
indexFailed = false;
|
|
6181
|
+
stages.index = { status: "pass", detail: "implied pass from retrieval" };
|
|
6182
|
+
}
|
|
6183
|
+
stages.use = { status: "fail", detail: "Gold memory present in context but answer was incorrect" };
|
|
6184
|
+
return {
|
|
6185
|
+
goldMemory: goldStatement,
|
|
6186
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
6187
|
+
stages
|
|
6188
|
+
};
|
|
6189
|
+
}
|
|
6190
|
+
let retrievalStageMiss = "unknown";
|
|
6191
|
+
if (atCapUnavailable) {
|
|
6192
|
+
stages.retrieval = { status: "unavailable", detail: "stored at-cap witness unavailable" };
|
|
6193
|
+
} else if (headroomHit) {
|
|
6194
|
+
retrievalStageMiss = "cap";
|
|
6195
|
+
stages.retrieval = {
|
|
6196
|
+
status: "fail",
|
|
6197
|
+
detail: `Headroom rank ${headroomHit.rank} exceeds applied cap ${headroomHit.appliedCap} in session ${headroomHit.sessionId}`
|
|
6198
|
+
};
|
|
6199
|
+
if (!indexPassed) {
|
|
6200
|
+
indexPassed = true;
|
|
6201
|
+
indexFailed = false;
|
|
6202
|
+
stages.index = { status: "pass", detail: "implied pass from retrieval headroom" };
|
|
6203
|
+
}
|
|
6204
|
+
} else {
|
|
6205
|
+
stages.retrieval = { status: "fail", detail: "absent from every stored at-cap retrieval witness" };
|
|
6206
|
+
}
|
|
6207
|
+
if (indexFailed && stages.retrieval.status === "fail") {
|
|
6208
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6209
|
+
return {
|
|
6210
|
+
goldMemory: goldStatement,
|
|
6211
|
+
label: { class: "index_miss", reason: "Gold statement missing from search index" },
|
|
6212
|
+
stages
|
|
6213
|
+
};
|
|
6214
|
+
}
|
|
6215
|
+
if (indexPassed && stages.retrieval.status === "fail") {
|
|
6216
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6217
|
+
return {
|
|
6218
|
+
goldMemory: goldStatement,
|
|
6219
|
+
label: {
|
|
6220
|
+
class: "retrieval_miss",
|
|
6221
|
+
retrievalStage: retrievalStageMiss,
|
|
6222
|
+
reason: stages.retrieval.detail
|
|
6223
|
+
},
|
|
6224
|
+
stages
|
|
6225
|
+
};
|
|
6226
|
+
}
|
|
6227
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
6228
|
+
const reason = stages.index.status === "unavailable" ? "index check unavailable; a retrieval miss cannot be isolated from an index miss" : "retrieval check unavailable";
|
|
6229
|
+
return {
|
|
6230
|
+
goldMemory: goldStatement,
|
|
6231
|
+
label: { class: "unattributed", reason },
|
|
6232
|
+
stages
|
|
6233
|
+
};
|
|
6234
|
+
}
|
|
6235
|
+
async function attributeTask(task, env, options = {}) {
|
|
6236
|
+
const golds = task.goldMemories ?? task.attributionWitness?.golds.map((gold) => gold.goldMemory);
|
|
6237
|
+
if (!golds || golds.length === 0) {
|
|
6238
|
+
return null;
|
|
6239
|
+
}
|
|
6240
|
+
const recalledText = typeof task.details?.recalledText === "string" ? task.details.recalledText : void 0;
|
|
6241
|
+
if (task.attributionWitness) {
|
|
6242
|
+
const goldAttributions2 = [];
|
|
6243
|
+
for (let index = 0; index < golds.length; index += 1) {
|
|
6244
|
+
const gold = golds[index];
|
|
6245
|
+
const goldWitness = task.attributionWitness.golds[index];
|
|
6246
|
+
if (!goldWitness || goldWitness.goldMemory !== gold) {
|
|
6247
|
+
throw new Error("attribution witness golds must match task goldMemories in length and order");
|
|
6248
|
+
}
|
|
6249
|
+
goldAttributions2.push(attributeGoldMemoryFromWitness(
|
|
6250
|
+
gold,
|
|
6251
|
+
goldWitness,
|
|
6252
|
+
task.attributionWitness.retrievals,
|
|
6253
|
+
task.attributionWitness.runtime.attributionThreshold,
|
|
6254
|
+
options,
|
|
6255
|
+
recalledText
|
|
6256
|
+
));
|
|
6257
|
+
}
|
|
6258
|
+
if (task.attributionWitness.golds.length !== golds.length) {
|
|
6259
|
+
throw new Error("attribution witness golds must match task goldMemories in length and order");
|
|
6260
|
+
}
|
|
6261
|
+
return {
|
|
6262
|
+
taskId: task.taskId,
|
|
6263
|
+
question: task.question,
|
|
6264
|
+
golds: goldAttributions2,
|
|
6265
|
+
overall: computeOverallLabel(goldAttributions2)
|
|
6266
|
+
};
|
|
6267
|
+
}
|
|
6268
|
+
const memoizedEnv = withMemoizedListMemories(env);
|
|
6269
|
+
const goldAttributions = [];
|
|
6270
|
+
for (const gold of golds) {
|
|
6271
|
+
const attr = await attributeGoldMemory(gold, task.question, memoizedEnv, options, recalledText);
|
|
6272
|
+
goldAttributions.push(attr);
|
|
6273
|
+
}
|
|
6274
|
+
const overall = computeOverallLabel(goldAttributions);
|
|
6275
|
+
return {
|
|
6276
|
+
taskId: task.taskId,
|
|
6277
|
+
question: task.question,
|
|
6278
|
+
golds: goldAttributions,
|
|
6279
|
+
overall
|
|
6280
|
+
};
|
|
6281
|
+
}
|
|
6282
|
+
async function attributeRun(result, env, options = {}) {
|
|
6283
|
+
const runId = result.meta?.runId ?? result.meta?.id ?? "unknown-run";
|
|
6284
|
+
let memoizedEnv;
|
|
6285
|
+
const totals = {
|
|
6286
|
+
extraction_miss: 0,
|
|
6287
|
+
index_miss: 0,
|
|
6288
|
+
retrieval_miss: 0,
|
|
6289
|
+
use_miss: 0,
|
|
6290
|
+
unattributed: 0
|
|
6291
|
+
};
|
|
6292
|
+
const retrievalStages = {
|
|
6293
|
+
filter: 0,
|
|
6294
|
+
cap: 0,
|
|
6295
|
+
rank: 0,
|
|
6296
|
+
unknown: 0
|
|
6297
|
+
};
|
|
6298
|
+
const items = [];
|
|
6299
|
+
const skippedTasks = [];
|
|
6300
|
+
for (const task of result.results.tasks) {
|
|
6301
|
+
if (task.details?.benchmarkFailure && typeof task.details.benchmarkFailure === "object") {
|
|
6302
|
+
skippedTasks.push({
|
|
6303
|
+
taskId: task.taskId,
|
|
6304
|
+
reason: "trial execution failure (not an answer failure)"
|
|
6305
|
+
});
|
|
6306
|
+
continue;
|
|
6307
|
+
}
|
|
6308
|
+
const goldCount = task.goldMemories?.length ?? task.attributionWitness?.golds.length ?? 0;
|
|
6309
|
+
if (goldCount === 0) {
|
|
6310
|
+
skippedTasks.push({
|
|
6311
|
+
taskId: task.taskId,
|
|
6312
|
+
reason: "No goldMemories specified"
|
|
6313
|
+
});
|
|
6314
|
+
continue;
|
|
6315
|
+
}
|
|
6316
|
+
if (!isTaskFailed(task)) {
|
|
6317
|
+
skippedTasks.push({
|
|
6318
|
+
taskId: task.taskId,
|
|
6319
|
+
reason: "Task passed (score >= 1)"
|
|
6320
|
+
});
|
|
6321
|
+
continue;
|
|
6322
|
+
}
|
|
6323
|
+
const taskEnv = task.attributionWitness ? env : memoizedEnv ??= withMemoizedListMemories(env);
|
|
6324
|
+
const taskAttr = await attributeTask(task, taskEnv, options);
|
|
6325
|
+
if (taskAttr) {
|
|
6326
|
+
items.push(taskAttr);
|
|
6327
|
+
}
|
|
6328
|
+
}
|
|
6329
|
+
items.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
|
|
6330
|
+
skippedTasks.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
|
|
6331
|
+
for (const item of items) {
|
|
6332
|
+
totals[item.overall.class]++;
|
|
6333
|
+
if (item.overall.class === "retrieval_miss") {
|
|
6334
|
+
const stage = item.overall.retrievalStage ?? "unknown";
|
|
6335
|
+
retrievalStages[stage]++;
|
|
6336
|
+
}
|
|
6337
|
+
}
|
|
6338
|
+
return {
|
|
6339
|
+
runId,
|
|
6340
|
+
totals,
|
|
6341
|
+
retrievalStages,
|
|
6342
|
+
attributedTasks: items.length,
|
|
6343
|
+
skippedTasks,
|
|
6344
|
+
items
|
|
6345
|
+
};
|
|
6346
|
+
}
|
|
6347
|
+
function renderAttributionReportTable(report) {
|
|
6348
|
+
const lines = [];
|
|
6349
|
+
lines.push(`Attribution Report (Run: ${report.runId})`);
|
|
6350
|
+
lines.push(`Failed-task predicate: minimum primary answer score < 1 (scores.overall or non-diagnostic scores)`);
|
|
6351
|
+
lines.push(`Attributed tasks: ${report.attributedTasks}, Skipped tasks: ${report.skippedTasks.length}`);
|
|
6352
|
+
lines.push("");
|
|
6353
|
+
lines.push("Totals by Class:");
|
|
6354
|
+
lines.push(` extraction_miss: ${report.totals.extraction_miss}`);
|
|
6355
|
+
lines.push(` index_miss: ${report.totals.index_miss}`);
|
|
6356
|
+
lines.push(` retrieval_miss: ${report.totals.retrieval_miss}`);
|
|
6357
|
+
lines.push(` use_miss: ${report.totals.use_miss}`);
|
|
6358
|
+
lines.push(` unattributed: ${report.totals.unattributed}`);
|
|
6359
|
+
lines.push("");
|
|
6360
|
+
lines.push("Retrieval Miss Stages:");
|
|
6361
|
+
lines.push(` filter: ${report.retrievalStages.filter}`);
|
|
6362
|
+
lines.push(` cap: ${report.retrievalStages.cap}`);
|
|
6363
|
+
lines.push(` rank: ${report.retrievalStages.rank}`);
|
|
6364
|
+
lines.push(` unknown: ${report.retrievalStages.unknown}`);
|
|
6365
|
+
lines.push("");
|
|
6366
|
+
lines.push("Task Attributions:");
|
|
6367
|
+
if (report.items.length === 0) {
|
|
6368
|
+
lines.push(" (none)");
|
|
6369
|
+
} else {
|
|
6370
|
+
for (const item of report.items) {
|
|
6371
|
+
const stageStr = item.overall.retrievalStage ? ` (${item.overall.retrievalStage})` : "";
|
|
6372
|
+
const labelStr = `${item.overall.class}${stageStr}`;
|
|
6373
|
+
const reasonStr = item.overall.reason ? ` - ${item.overall.reason}` : "";
|
|
6374
|
+
lines.push(` ${item.taskId.padEnd(20)} ${labelStr.padEnd(24)}${reasonStr}`);
|
|
6375
|
+
}
|
|
6376
|
+
}
|
|
6377
|
+
if (report.skippedTasks.length > 0) {
|
|
6378
|
+
lines.push("");
|
|
6379
|
+
lines.push("Skipped Tasks:");
|
|
6380
|
+
for (const skipped of report.skippedTasks) {
|
|
6381
|
+
lines.push(` ${skipped.taskId.padEnd(20)} ${skipped.reason}`);
|
|
6382
|
+
}
|
|
6383
|
+
}
|
|
6384
|
+
return `${lines.join("\n")}
|
|
6385
|
+
`;
|
|
6386
|
+
}
|
|
6387
|
+
function serializeAttributionReport(report) {
|
|
6388
|
+
return `${JSON.stringify(report, null, 2)}
|
|
6389
|
+
`;
|
|
6390
|
+
}
|
|
6391
|
+
|
|
6392
|
+
// src/adapters/attribution-witness.ts
|
|
6393
|
+
async function readCoreMemories(orchestrator) {
|
|
6394
|
+
return [
|
|
6395
|
+
...await orchestrator.storage.readAllMemories(),
|
|
6396
|
+
...await orchestrator.storage.readAllColdMemories()
|
|
6397
|
+
];
|
|
6398
|
+
}
|
|
6399
|
+
async function canonicalizeMemoryIds(orchestrator, results) {
|
|
6400
|
+
const memories = await Promise.all(
|
|
6401
|
+
results.map((result) => orchestrator.qmdResultResolver.readQmdResultMemory(
|
|
6402
|
+
result.path,
|
|
6403
|
+
orchestrator.storage,
|
|
6404
|
+
[],
|
|
6405
|
+
result.namespace
|
|
6406
|
+
))
|
|
6407
|
+
);
|
|
6408
|
+
if (memories.some((memory) => memory === null)) return null;
|
|
6409
|
+
return memories.map((memory) => memory.frontmatter.id);
|
|
6410
|
+
}
|
|
6411
|
+
async function captureRecallAttribution(orchestrator, sessionId, snapshot) {
|
|
6412
|
+
const [atCapMemoryIds, headroomMemoryIds] = await Promise.all([
|
|
6413
|
+
canonicalizeMemoryIds(orchestrator, snapshot.appliedResults),
|
|
6414
|
+
canonicalizeMemoryIds(orchestrator, snapshot.headroomResults)
|
|
6415
|
+
]);
|
|
6416
|
+
if (atCapMemoryIds === null || headroomMemoryIds === null) return void 0;
|
|
6417
|
+
return {
|
|
6418
|
+
sessionId,
|
|
6419
|
+
appliedCap: snapshot.appliedResultLimit,
|
|
6420
|
+
atCapMemoryIds,
|
|
6421
|
+
headroomMemoryIds
|
|
6422
|
+
};
|
|
6423
|
+
}
|
|
6424
|
+
async function captureTaskAttributionWitness(options) {
|
|
6425
|
+
const qmdMaxResults = options.orchestrator.config.qmdMaxResults;
|
|
6426
|
+
let storedMemories;
|
|
6427
|
+
try {
|
|
6428
|
+
storedMemories = await readCoreMemories(options.orchestrator);
|
|
6429
|
+
} catch {
|
|
6430
|
+
storedMemories = null;
|
|
6431
|
+
}
|
|
6432
|
+
const golds = await Promise.all(
|
|
6433
|
+
options.goldMemories.map(async (goldMemory) => {
|
|
6434
|
+
const storeMemoryIds = storedMemories === null ? null : [
|
|
6435
|
+
...new Set(
|
|
6436
|
+
storedMemories.filter((memory) => lexicalSimilarity(goldMemory, memory.content) >= 0.6).map((memory) => memory.frontmatter.id)
|
|
6437
|
+
)
|
|
6438
|
+
];
|
|
6439
|
+
let oracleMemoryIds = null;
|
|
6440
|
+
if (qmdMaxResults > 0) {
|
|
6441
|
+
try {
|
|
6442
|
+
const oracleResults = await options.orchestrator.searchAcrossNamespaces({
|
|
6443
|
+
query: goldMemory,
|
|
6444
|
+
maxResults: qmdMaxResults,
|
|
6445
|
+
mode: "search"
|
|
6446
|
+
});
|
|
6447
|
+
const canonicalIds = await canonicalizeMemoryIds(
|
|
6448
|
+
options.orchestrator,
|
|
6449
|
+
oracleResults
|
|
6450
|
+
);
|
|
6451
|
+
oracleMemoryIds = canonicalIds === null ? null : [...new Set(canonicalIds)];
|
|
6452
|
+
} catch {
|
|
6453
|
+
oracleMemoryIds = null;
|
|
6454
|
+
}
|
|
6455
|
+
}
|
|
6456
|
+
return { goldMemory, storeMemoryIds, oracleMemoryIds };
|
|
6457
|
+
})
|
|
6458
|
+
);
|
|
6459
|
+
return {
|
|
6460
|
+
schemaVersion: 1,
|
|
6461
|
+
runtime: {
|
|
6462
|
+
qmdCollection: options.qmdCollection,
|
|
6463
|
+
qmdIndex: options.qmdIndex,
|
|
6464
|
+
qmdMaxResults,
|
|
6465
|
+
attributionThreshold: DEFAULT_ATTRIBUTION_THRESHOLD
|
|
6466
|
+
},
|
|
6467
|
+
golds,
|
|
6468
|
+
retrievals: options.retrievals.map((retrieval) => ({
|
|
6469
|
+
sessionId: retrieval.sessionId,
|
|
6470
|
+
appliedCap: retrieval.appliedCap,
|
|
6471
|
+
atCapMemoryIds: retrieval.atCapMemoryIds === null ? null : [...retrieval.atCapMemoryIds],
|
|
6472
|
+
headroomMemoryIds: retrieval.headroomMemoryIds === null ? null : [...retrieval.headroomMemoryIds]
|
|
6473
|
+
}))
|
|
6474
|
+
};
|
|
6475
|
+
}
|
|
6476
|
+
|
|
6477
|
+
// src/adapters/remnic-adapter.ts
|
|
6478
|
+
import {
|
|
6479
|
+
lcmEvidenceIdentity
|
|
6480
|
+
} from "@remnic/core/lcm";
|
|
6481
|
+
|
|
6482
|
+
// src/adapters/remnic-recall-trace.ts
|
|
6483
|
+
import { createHash as createHash6 } from "crypto";
|
|
6484
|
+
import {
|
|
6485
|
+
lcmArchiveRowId
|
|
6486
|
+
} from "@remnic/core/lcm";
|
|
6487
|
+
function visibleRange(start, end, returnedChars) {
|
|
6488
|
+
const visibleStart = Math.min(start, returnedChars);
|
|
6489
|
+
return {
|
|
6490
|
+
visibleStart,
|
|
6491
|
+
visibleEnd: Math.max(visibleStart, Math.min(end, returnedChars))
|
|
6492
|
+
};
|
|
6493
|
+
}
|
|
6494
|
+
function projectBenchCoreCapture(snapshot) {
|
|
6495
|
+
return {
|
|
6496
|
+
snapshotId: snapshot.snapshotId,
|
|
6497
|
+
capturedAt: snapshot.capturedAt,
|
|
6498
|
+
...snapshot.traceId === void 0 ? {} : { traceId: snapshot.traceId },
|
|
6499
|
+
budget: { chars: snapshot.budget.chars, used: snapshot.budget.used },
|
|
6500
|
+
filters: snapshot.filters.map(({ name, considered, admitted }) => ({
|
|
6501
|
+
name,
|
|
6502
|
+
considered,
|
|
6503
|
+
admitted
|
|
6504
|
+
})),
|
|
6505
|
+
results: snapshot.results.map((result) => {
|
|
6506
|
+
const scores = result.scoreDecomposition;
|
|
6507
|
+
return {
|
|
6508
|
+
memoryIdRef: {
|
|
6509
|
+
sha256: createHash6("sha256").update(result.memoryId, "utf8").digest("hex"),
|
|
6510
|
+
length: Buffer.byteLength(result.memoryId, "utf8")
|
|
6511
|
+
},
|
|
6512
|
+
servedBy: result.servedBy,
|
|
6513
|
+
scoreDecomposition: {
|
|
6514
|
+
...typeof scores.vector === "number" ? { vector: scores.vector } : {},
|
|
6515
|
+
...typeof scores.bm25 === "number" ? { bm25: scores.bm25 } : {},
|
|
6516
|
+
...typeof scores.importance === "number" ? { importance: scores.importance } : {},
|
|
6517
|
+
...typeof scores.mmrPenalty === "number" ? { mmrPenalty: scores.mmrPenalty } : {},
|
|
6518
|
+
...typeof scores.tierPrior === "number" ? { tierPrior: scores.tierPrior } : {},
|
|
6519
|
+
...typeof scores.reinforcementBoost === "number" ? { reinforcementBoost: scores.reinforcementBoost } : {},
|
|
6520
|
+
final: scores.final
|
|
6521
|
+
},
|
|
6522
|
+
admittedBy: [...result.admittedBy],
|
|
6523
|
+
...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
|
|
6524
|
+
...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
|
|
6525
|
+
...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
|
|
6526
|
+
};
|
|
6527
|
+
})
|
|
6528
|
+
};
|
|
6529
|
+
}
|
|
6530
|
+
function createBenchRecallTraceRecorder(requestedChars) {
|
|
6531
|
+
const sections = [];
|
|
6532
|
+
const pendingSelections = [];
|
|
6533
|
+
const lcmCandidates = [];
|
|
6534
|
+
let composedChars = 0;
|
|
6535
|
+
let coreCapture;
|
|
6536
|
+
const sectionById = (sectionId) => {
|
|
6537
|
+
const section = sections.find((entry) => entry.id === sectionId);
|
|
6538
|
+
if (!section) throw new Error(`Unknown benchmark recall trace section: ${sectionId}`);
|
|
6539
|
+
return section;
|
|
6540
|
+
};
|
|
6541
|
+
const appendRelativeSelection = (sectionId, kind, start, end, lineageStatus, fields = {}) => {
|
|
6542
|
+
const section = sectionById(sectionId);
|
|
6543
|
+
const contentLength = section.contentEnd - section.contentStart;
|
|
6544
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > contentLength) {
|
|
6545
|
+
throw new Error(
|
|
6546
|
+
`Invalid benchmark recall trace range for ${sectionId}: ${start}..${end}.`
|
|
6547
|
+
);
|
|
6548
|
+
}
|
|
6549
|
+
pendingSelections.push({
|
|
6550
|
+
sectionId,
|
|
6551
|
+
kind,
|
|
6552
|
+
lineageStatus,
|
|
6553
|
+
composedStart: section.contentStart + start,
|
|
6554
|
+
composedEnd: section.contentStart + end,
|
|
6555
|
+
...fields
|
|
6556
|
+
});
|
|
6557
|
+
};
|
|
6558
|
+
return {
|
|
6559
|
+
appendSection(id, source, renderedLength) {
|
|
6560
|
+
if (!Number.isSafeInteger(renderedLength) || renderedLength < 0) {
|
|
6561
|
+
throw new Error("Benchmark recall trace section length must be a non-negative integer.");
|
|
6562
|
+
}
|
|
6563
|
+
if (sections.some((section) => section.id === id)) {
|
|
6564
|
+
throw new Error(`Duplicate benchmark recall trace section: ${id}`);
|
|
6565
|
+
}
|
|
6566
|
+
const separatorStart = composedChars;
|
|
6567
|
+
const contentStart = composedChars + (sections.length === 0 ? 0 : 2);
|
|
6568
|
+
const contentEnd = contentStart + renderedLength;
|
|
6569
|
+
sections.push({
|
|
6570
|
+
id,
|
|
6571
|
+
source,
|
|
6572
|
+
separatorStart,
|
|
6573
|
+
contentStart,
|
|
6574
|
+
contentEnd,
|
|
6575
|
+
composedStart: separatorStart,
|
|
6576
|
+
composedEnd: contentEnd,
|
|
6577
|
+
visibleStart: 0,
|
|
6578
|
+
visibleEnd: 0,
|
|
6579
|
+
visibleChars: 0
|
|
6580
|
+
});
|
|
6581
|
+
composedChars = contentEnd;
|
|
6582
|
+
},
|
|
6583
|
+
recordEvidenceSelections(sectionId, receipts) {
|
|
6584
|
+
for (const receipt of receipts) {
|
|
6585
|
+
appendRelativeSelection(
|
|
6586
|
+
sectionId,
|
|
6587
|
+
"evidence-block",
|
|
6588
|
+
receipt.blockStart,
|
|
6589
|
+
receipt.blockEnd,
|
|
6590
|
+
receipt.item.archiveRowId === void 0 ? "unavailable" : "exact",
|
|
6591
|
+
{
|
|
6592
|
+
...receipt.item.archiveRowId === void 0 ? {} : { archiveRowIds: [receipt.item.archiveRowId] },
|
|
6593
|
+
...receipt.item.turnIndex === void 0 ? {} : { turnIndex: receipt.item.turnIndex },
|
|
6594
|
+
...receipt.item.role === void 0 ? {} : { role: receipt.item.role },
|
|
6595
|
+
...receipt.item.score === void 0 ? {} : { score: receipt.item.score }
|
|
6596
|
+
}
|
|
6597
|
+
);
|
|
6598
|
+
}
|
|
6599
|
+
},
|
|
6600
|
+
recordTrajectorySelections(sectionId, receipts) {
|
|
6601
|
+
for (const receipt of receipts) {
|
|
6602
|
+
appendRelativeSelection(
|
|
6603
|
+
sectionId,
|
|
6604
|
+
"trajectory-line",
|
|
6605
|
+
receipt.lineStart,
|
|
6606
|
+
receipt.lineEnd,
|
|
6607
|
+
receipt.lineageStatus,
|
|
6608
|
+
{
|
|
6609
|
+
archiveRowIds: [
|
|
6610
|
+
...receipt.actionArchiveRowIds,
|
|
6611
|
+
...receipt.observationArchiveRowIds
|
|
6612
|
+
]
|
|
6613
|
+
}
|
|
6614
|
+
);
|
|
6615
|
+
}
|
|
6616
|
+
},
|
|
6617
|
+
recordSummarySelections(sectionId, receipts) {
|
|
6618
|
+
for (const receipt of receipts) {
|
|
6619
|
+
appendRelativeSelection(
|
|
6620
|
+
sectionId,
|
|
6621
|
+
"lcm-summary",
|
|
6622
|
+
receipt.entryStart,
|
|
6623
|
+
receipt.entryEnd,
|
|
6624
|
+
"exact",
|
|
6625
|
+
{
|
|
6626
|
+
summary: {
|
|
6627
|
+
id: receipt.id,
|
|
6628
|
+
depth: receipt.depth,
|
|
6629
|
+
msgStart: receipt.msgStart,
|
|
6630
|
+
msgEnd: receipt.msgEnd
|
|
6631
|
+
}
|
|
6632
|
+
}
|
|
6633
|
+
);
|
|
6634
|
+
}
|
|
6635
|
+
},
|
|
6636
|
+
recordRawRow(sectionId, range, row) {
|
|
6637
|
+
const archiveRowId = lcmArchiveRowId(row);
|
|
6638
|
+
appendRelativeSelection(
|
|
6639
|
+
sectionId,
|
|
6640
|
+
"raw-row",
|
|
6641
|
+
range.start,
|
|
6642
|
+
range.end,
|
|
6643
|
+
archiveRowId === void 0 ? "unavailable" : "exact",
|
|
6644
|
+
{
|
|
6645
|
+
...archiveRowId === void 0 ? {} : { archiveRowIds: [archiveRowId] },
|
|
6646
|
+
turnIndex: row.turn_index,
|
|
6647
|
+
role: row.role
|
|
6648
|
+
}
|
|
6649
|
+
);
|
|
6650
|
+
},
|
|
6651
|
+
recordLcmCandidate(candidate) {
|
|
6652
|
+
lcmCandidates.push({ ...candidate });
|
|
6653
|
+
},
|
|
6654
|
+
recordCoreCapture(snapshot) {
|
|
6655
|
+
coreCapture = snapshot ? projectBenchCoreCapture(snapshot) : void 0;
|
|
6656
|
+
},
|
|
6657
|
+
finalize(returnedChars) {
|
|
6658
|
+
const normalizedReturnedChars = Math.max(0, Math.min(returnedChars, composedChars));
|
|
6659
|
+
return {
|
|
6660
|
+
schemaVersion: 1,
|
|
6661
|
+
sensitivity: {
|
|
6662
|
+
classification: "restricted",
|
|
6663
|
+
contentEncoding: "sha256+length",
|
|
6664
|
+
containsGold: false
|
|
6665
|
+
},
|
|
6666
|
+
sections: sections.map((section) => {
|
|
6667
|
+
const visible = visibleRange(
|
|
6668
|
+
section.composedStart,
|
|
6669
|
+
section.composedEnd,
|
|
6670
|
+
normalizedReturnedChars
|
|
6671
|
+
);
|
|
6672
|
+
return {
|
|
6673
|
+
...section,
|
|
6674
|
+
...visible,
|
|
6675
|
+
visibleChars: visible.visibleEnd - visible.visibleStart
|
|
6676
|
+
};
|
|
6677
|
+
}),
|
|
6678
|
+
selections: pendingSelections.map((selection) => ({
|
|
6679
|
+
...selection,
|
|
6680
|
+
...visibleRange(
|
|
6681
|
+
selection.composedStart,
|
|
6682
|
+
selection.composedEnd,
|
|
6683
|
+
normalizedReturnedChars
|
|
6684
|
+
)
|
|
6685
|
+
})),
|
|
6686
|
+
lcmCandidates: lcmCandidates.map((candidate) => ({ ...candidate })),
|
|
6687
|
+
...coreCapture === void 0 ? {} : { coreCapture },
|
|
6688
|
+
budget: {
|
|
6689
|
+
requestedChars,
|
|
6690
|
+
composedChars,
|
|
6691
|
+
returnedChars: normalizedReturnedChars,
|
|
6692
|
+
truncated: normalizedReturnedChars < composedChars
|
|
6693
|
+
}
|
|
6694
|
+
};
|
|
6695
|
+
}
|
|
6696
|
+
};
|
|
6697
|
+
}
|
|
6698
|
+
|
|
6699
|
+
// src/recall-budget.ts
|
|
6700
|
+
var DEFAULT_BENCH_RECALL_BUDGET_CHARS = 24e3;
|
|
6701
|
+
var MAX_COMBINED_RECALL_BUDGET_CHARS = 36e3;
|
|
6702
|
+
function benchmarkRecallBudgetForSessionCount(sessionCount) {
|
|
6703
|
+
if (!Number.isInteger(sessionCount) || sessionCount <= 0) {
|
|
6704
|
+
return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
|
|
6705
|
+
}
|
|
6706
|
+
if (sessionCount === 1) {
|
|
6707
|
+
return DEFAULT_BENCH_RECALL_BUDGET_CHARS;
|
|
6708
|
+
}
|
|
6709
|
+
return Math.floor(MAX_COMBINED_RECALL_BUDGET_CHARS / sessionCount);
|
|
6710
|
+
}
|
|
6711
|
+
|
|
6712
|
+
// src/adapters/remnic-adapter.ts
|
|
6713
|
+
var DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE = 0.34;
|
|
6714
|
+
var ANSWER_SUPPORT_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
6715
|
+
"about",
|
|
6716
|
+
"after",
|
|
6717
|
+
"again",
|
|
6718
|
+
"also",
|
|
6719
|
+
"answer",
|
|
6720
|
+
"before",
|
|
6721
|
+
"being",
|
|
6722
|
+
"could",
|
|
6723
|
+
"does",
|
|
6724
|
+
"from",
|
|
6725
|
+
"have",
|
|
6726
|
+
"information",
|
|
6727
|
+
"into",
|
|
6728
|
+
"just",
|
|
6729
|
+
"know",
|
|
6730
|
+
"memory",
|
|
6731
|
+
"might",
|
|
6732
|
+
"please",
|
|
6733
|
+
"question",
|
|
6734
|
+
"recall",
|
|
6735
|
+
"remember",
|
|
6736
|
+
"should",
|
|
6737
|
+
"that",
|
|
6738
|
+
"their",
|
|
6739
|
+
"there",
|
|
6740
|
+
"these",
|
|
6741
|
+
"they",
|
|
6742
|
+
"this",
|
|
6743
|
+
"those",
|
|
6744
|
+
"user",
|
|
6745
|
+
"using",
|
|
6746
|
+
"what",
|
|
6747
|
+
"when",
|
|
6748
|
+
"where",
|
|
6749
|
+
"which",
|
|
6750
|
+
"while",
|
|
6751
|
+
"with",
|
|
6752
|
+
"would",
|
|
6753
|
+
"your"
|
|
6754
|
+
]);
|
|
6755
|
+
function resolveAnswerSupportMinCoverage(config) {
|
|
6756
|
+
const raw = config?.answerSupportMinCoverage;
|
|
6757
|
+
if (raw === void 0) return DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE;
|
|
6758
|
+
const parsed = typeof raw === "number" ? raw : Number(raw);
|
|
6759
|
+
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) {
|
|
6760
|
+
throw new Error("answerSupportMinCoverage must be a finite number greater than 0 and at most 1.");
|
|
6761
|
+
}
|
|
6762
|
+
return parsed;
|
|
6763
|
+
}
|
|
6764
|
+
function resolveSkipExtractionLcmFirst(config) {
|
|
6765
|
+
const raw = config?.skipExtractionLcmFirst;
|
|
6766
|
+
if (raw === void 0) return true;
|
|
6767
|
+
if (typeof raw === "boolean") return raw;
|
|
6768
|
+
if (typeof raw === "string") {
|
|
6769
|
+
const normalized = raw.trim().toLowerCase();
|
|
6770
|
+
if (["true", "1", "yes", "on"].includes(normalized)) return true;
|
|
6771
|
+
if (["false", "0", "no", "off"].includes(normalized)) return false;
|
|
6772
|
+
}
|
|
6773
|
+
throw new Error(
|
|
6774
|
+
"skipExtractionLcmFirst must be a boolean or one of true/false, 1/0, yes/no, on/off."
|
|
6775
|
+
);
|
|
6776
|
+
}
|
|
6777
|
+
function shouldIncludeCoreRecallForReplay(options) {
|
|
6778
|
+
return options.useCoreMemoryPipeline && (options.replayExtractionMode !== "skip" || !options.skipExtractionLcmFirst);
|
|
6779
|
+
}
|
|
6780
|
+
function normalizeSupportToken(value) {
|
|
6781
|
+
if (value.length > 5 && value.endsWith("ing")) return value.slice(0, -3);
|
|
6782
|
+
if (value.length > 4 && value.endsWith("ed")) {
|
|
6783
|
+
const base = value.slice(0, -2);
|
|
6784
|
+
return /[vs]$/.test(base) ? `${base}e` : base;
|
|
6785
|
+
}
|
|
6786
|
+
if (value.length > 4 && value.endsWith("es")) return value.slice(0, -2);
|
|
6787
|
+
if (value.length > 3 && value.endsWith("s")) return value.slice(0, -1);
|
|
6788
|
+
return value;
|
|
6789
|
+
}
|
|
6790
|
+
function supportTerms(value) {
|
|
6791
|
+
return [...new Set(
|
|
6792
|
+
(value.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}_-]{2,}/gu) ?? []).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term)).map(normalizeSupportToken).filter((term) => !ANSWER_SUPPORT_STOP_WORDS.has(term) && !/^\d+$/.test(term))
|
|
6793
|
+
)];
|
|
6794
|
+
}
|
|
6795
|
+
function exactContextEvidenceLines(recalledText) {
|
|
6796
|
+
return recalledText.split(/\r?\n/).map((line) => line.trim()).filter((line) => {
|
|
6797
|
+
if (!line || /^#{1,6}\s/.test(line)) return false;
|
|
6798
|
+
return !/^(?:answer guidance:|distinct user-stated targets found:|no (?:direct|historically valid)|these direct temporal statements|this is the most recent|use this list|when answering)/i.test(line);
|
|
6799
|
+
});
|
|
6800
|
+
}
|
|
6801
|
+
function assessRemnicRecallSupport(request, supportThreshold = DEFAULT_ANSWER_SUPPORT_MIN_COVERAGE) {
|
|
6802
|
+
if (request.recalledText.trim().length === 0) {
|
|
6803
|
+
return { status: "empty", reason: "exact responder context is empty", evidenceCount: 0 };
|
|
6804
|
+
}
|
|
6805
|
+
const queryTerms = supportTerms(request.query);
|
|
6806
|
+
if (queryTerms.length < 2) {
|
|
6807
|
+
return {
|
|
6808
|
+
status: "unavailable",
|
|
6809
|
+
reason: "query has fewer than two distinctive terms for conservative support scoring"
|
|
6810
|
+
};
|
|
6811
|
+
}
|
|
6812
|
+
const evidenceLines = exactContextEvidenceLines(request.recalledText);
|
|
6813
|
+
const evidenceTermSets = evidenceLines.map((line) => new Set(supportTerms(line)));
|
|
6814
|
+
const matchedTerms = queryTerms.filter(
|
|
5966
6815
|
(term) => evidenceTermSets.some((terms) => terms.has(term))
|
|
5967
6816
|
);
|
|
5968
6817
|
const evidenceCount = evidenceTermSets.filter(
|
|
@@ -6859,6 +7708,14 @@ function pruneBenchEntityStructuredFacts(structuredSections, targetSources, prot
|
|
|
6859
7708
|
return { changed, structuredSections: nextSections };
|
|
6860
7709
|
}
|
|
6861
7710
|
async function clearBenchCoreEntitiesForSession(orchestrator, sessionId) {
|
|
7711
|
+
const baseDir = orchestrator.storage.dir;
|
|
7712
|
+
await withRawEntityPageMutation(
|
|
7713
|
+
baseDir,
|
|
7714
|
+
path5.join(baseDir, "entities", ".bench-session-cleanup.md"),
|
|
7715
|
+
async () => clearBenchCoreEntitiesForSessionUnlocked(orchestrator, sessionId)
|
|
7716
|
+
);
|
|
7717
|
+
}
|
|
7718
|
+
async function clearBenchCoreEntitiesForSessionUnlocked(orchestrator, sessionId) {
|
|
6862
7719
|
const storage = orchestrator.storage;
|
|
6863
7720
|
const entityStorage = storage;
|
|
6864
7721
|
const entitySchemas = entityStorage.entitySchemas;
|
|
@@ -7328,7 +8185,7 @@ function createAdapterFactory(mode) {
|
|
|
7328
8185
|
throw error;
|
|
7329
8186
|
}
|
|
7330
8187
|
},
|
|
7331
|
-
async [composeRecall](sessionId, query, budgetChars, recallOptions = {}, control, traceRecorder) {
|
|
8188
|
+
async [composeRecall](sessionId, query, budgetChars, recallOptions = {}, control, traceRecorder, attributionSink) {
|
|
7332
8189
|
throwIfBenchPhaseAborted(control, "recall");
|
|
7333
8190
|
const waitForRecall = (promise) => withBenchPhaseAbort(promise, control, "recall");
|
|
7334
8191
|
sessionId = normalizeBenchSessionId(sessionId);
|
|
@@ -7480,8 +8337,15 @@ function createAdapterFactory(mode) {
|
|
|
7480
8337
|
control,
|
|
7481
8338
|
"recall",
|
|
7482
8339
|
{ waitForCompletionOnAbort: true }
|
|
7483
|
-
).then((capture) => {
|
|
8340
|
+
).then(async (capture) => {
|
|
7484
8341
|
traceRecorder.recordCoreCapture(capture.snapshot);
|
|
8342
|
+
attributionSink?.(
|
|
8343
|
+
capture.snapshot ? await captureRecallAttribution(
|
|
8344
|
+
state.orchestrator,
|
|
8345
|
+
sessionId,
|
|
8346
|
+
capture.snapshot
|
|
8347
|
+
).catch(() => void 0) : void 0
|
|
8348
|
+
);
|
|
7485
8349
|
return capture.result;
|
|
7486
8350
|
}) : await waitForRecall(
|
|
7487
8351
|
state.orchestrator.recall(query, sessionId, coreOptions)
|
|
@@ -7784,15 +8648,32 @@ ${coreRecall.trim()}`;
|
|
|
7784
8648
|
async recallWithTrace(sessionId, query, budgetChars, recallOptions = {}, control) {
|
|
7785
8649
|
const budget = budgetChars ?? DEFAULT_BENCH_RECALL_BUDGET_CHARS;
|
|
7786
8650
|
const traceRecorder = createBenchRecallTraceRecorder(Math.max(0, budget));
|
|
8651
|
+
let attribution;
|
|
7787
8652
|
const text = await adapter[composeRecall](
|
|
7788
8653
|
sessionId,
|
|
7789
8654
|
query,
|
|
7790
8655
|
budgetChars,
|
|
7791
8656
|
recallOptions,
|
|
7792
8657
|
control,
|
|
7793
|
-
traceRecorder
|
|
8658
|
+
traceRecorder,
|
|
8659
|
+
(captured) => {
|
|
8660
|
+
attribution = captured;
|
|
8661
|
+
}
|
|
7794
8662
|
);
|
|
7795
|
-
return {
|
|
8663
|
+
return {
|
|
8664
|
+
text,
|
|
8665
|
+
trace: traceRecorder.finalize(text.length),
|
|
8666
|
+
...attribution ? { attribution } : {}
|
|
8667
|
+
};
|
|
8668
|
+
},
|
|
8669
|
+
async captureAttributionWitness(request) {
|
|
8670
|
+
return captureTaskAttributionWitness({
|
|
8671
|
+
orchestrator: state.orchestrator,
|
|
8672
|
+
qmdCollection: state.orchestrator.config.qmdCollection,
|
|
8673
|
+
qmdIndex: state.qmdSandbox.indexName,
|
|
8674
|
+
goldMemories: request.goldMemories,
|
|
8675
|
+
retrievals: request.retrievals
|
|
8676
|
+
});
|
|
7796
8677
|
},
|
|
7797
8678
|
async assessRecallSupport(request, control) {
|
|
7798
8679
|
throwIfBenchPhaseAborted(control, "assessRecallSupport");
|
|
@@ -21704,8 +22585,15 @@ function appendCompletedTask(ctx, tasks, pendingPairedAnswerReplays, task) {
|
|
|
21704
22585
|
}
|
|
21705
22586
|
async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate, pendingPairedAnswerReplays) {
|
|
21706
22587
|
const trialId = trial.taskId ?? trial.question.slice(0, 60);
|
|
22588
|
+
const attributionCapture = {};
|
|
21707
22589
|
try {
|
|
21708
|
-
return await executeTrial(
|
|
22590
|
+
return await executeTrial(
|
|
22591
|
+
ctx,
|
|
22592
|
+
trial,
|
|
22593
|
+
answerSupportGate,
|
|
22594
|
+
pendingPairedAnswerReplays,
|
|
22595
|
+
attributionCapture
|
|
22596
|
+
);
|
|
21709
22597
|
} catch (err) {
|
|
21710
22598
|
const blocked = findBenchmarkRunBlockedError(err);
|
|
21711
22599
|
if (blocked) {
|
|
@@ -21722,6 +22610,7 @@ async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate,
|
|
|
21722
22610
|
latencyMs: 0,
|
|
21723
22611
|
tokens: { input: 0, output: 0 },
|
|
21724
22612
|
...trial.goldMemories ? { goldMemories: trial.goldMemories } : {},
|
|
22613
|
+
...attributionCapture.witness ? { attributionWitness: attributionCapture.witness } : {},
|
|
21725
22614
|
details: {
|
|
21726
22615
|
// Preserve the trial's category so a failed trial is still attributed
|
|
21727
22616
|
// to its per-category bucket (computeCategoryAggregates), keeping the
|
|
@@ -21872,13 +22761,38 @@ function resolveResponderIdentity(responder) {
|
|
|
21872
22761
|
const trimmed = typeof raw === "string" ? raw.trim() : "";
|
|
21873
22762
|
return trimmed.length > 0 ? trimmed : null;
|
|
21874
22763
|
}
|
|
21875
|
-
async function executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerReplays) {
|
|
22764
|
+
async function executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerReplays, attributionCapture) {
|
|
21876
22765
|
const { result: recallResult, durationMs } = await timed(async () => {
|
|
21877
22766
|
const recallBudget = benchmarkRecallBudgetForSessionCount(trial.recallSessionIds.length);
|
|
22767
|
+
const witnessEnabled = (trial.goldMemories?.length ?? 0) > 0 && typeof ctx.options.system.recallWithTrace === "function" && typeof ctx.options.system.captureAttributionWitness === "function";
|
|
21878
22768
|
const recalledSessions = await Promise.all(
|
|
21879
|
-
trial.recallSessionIds.map((sessionId) =>
|
|
22769
|
+
trial.recallSessionIds.map(async (sessionId) => {
|
|
22770
|
+
if (!witnessEnabled) {
|
|
22771
|
+
return {
|
|
22772
|
+
text: await ctx.options.system.recall(sessionId, trial.question, recallBudget)
|
|
22773
|
+
};
|
|
22774
|
+
}
|
|
22775
|
+
const traced = await ctx.options.system.recallWithTrace(
|
|
22776
|
+
sessionId,
|
|
22777
|
+
trial.question,
|
|
22778
|
+
recallBudget
|
|
22779
|
+
);
|
|
22780
|
+
const attribution = traced.attribution ?? {
|
|
22781
|
+
sessionId,
|
|
22782
|
+
appliedCap: null,
|
|
22783
|
+
atCapMemoryIds: null,
|
|
22784
|
+
headroomMemoryIds: null
|
|
22785
|
+
};
|
|
22786
|
+
return { text: traced.text, attribution };
|
|
22787
|
+
})
|
|
21880
22788
|
);
|
|
21881
|
-
|
|
22789
|
+
if (witnessEnabled) {
|
|
22790
|
+
attributionCapture.witness = await ctx.options.system.captureAttributionWitness({
|
|
22791
|
+
goldMemories: trial.goldMemories,
|
|
22792
|
+
retrievals: recalledSessions.map((session) => session.attribution)
|
|
22793
|
+
}).catch(() => void 0);
|
|
22794
|
+
}
|
|
22795
|
+
const rawRecalledText = recalledSessions.map((session) => session.text).filter(Boolean).join("\n\n");
|
|
21882
22796
|
const recalledText2 = trial.recallTextTransform ? trial.recallTextTransform({
|
|
21883
22797
|
question: trial.question,
|
|
21884
22798
|
recalledText: rawRecalledText
|
|
@@ -22007,6 +22921,7 @@ async function executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerRe
|
|
|
22007
22921
|
output: answered.tokens.output + judgeResult.tokens.output
|
|
22008
22922
|
},
|
|
22009
22923
|
...trial.goldMemories ? { goldMemories: trial.goldMemories } : {},
|
|
22924
|
+
...attributionCapture.witness ? { attributionWitness: attributionCapture.witness } : {},
|
|
22010
22925
|
details
|
|
22011
22926
|
};
|
|
22012
22927
|
if (answerReplayKey && currentProfile === "baseline" && answered.fallbackReason === void 0) {
|
|
@@ -45797,771 +46712,268 @@ async function runCodingGraphBenchmark(config = {}) {
|
|
|
45797
46712
|
throw new Error(
|
|
45798
46713
|
`search_graph failed: ${searchRes.result.code}`
|
|
45799
46714
|
);
|
|
45800
|
-
}
|
|
45801
|
-
searchSamples.push(searchRes.ms);
|
|
45802
|
-
}
|
|
45803
|
-
const deadCode = timeSync(() => store.deadCode());
|
|
45804
|
-
if (!deadCode.result.ok) {
|
|
45805
|
-
throw new Error(`dead_code failed: ${deadCode.result.code}`);
|
|
45806
|
-
}
|
|
45807
|
-
await store.drain();
|
|
45808
|
-
let dbBytes = statSync(dbPath).size;
|
|
45809
|
-
try {
|
|
45810
|
-
dbBytes += statSync(dbPath + "-wal").size;
|
|
45811
|
-
} catch {
|
|
45812
|
-
}
|
|
45813
|
-
const kloc = Math.max(1, repo.approximateLoc / 1e3);
|
|
45814
|
-
const dbBytesPerKloc = dbBytes / kloc;
|
|
45815
|
-
const modifiedSamples = [];
|
|
45816
|
-
for (let i = 0; i < iterations; i++) {
|
|
45817
|
-
const fileIdx = i % storeFiles.length;
|
|
45818
|
-
const original = storeFiles[fileIdx];
|
|
45819
|
-
if (!original) continue;
|
|
45820
|
-
const ownSyms = new Set(original.symbols.map((sym) => sym.qualifiedName));
|
|
45821
|
-
const churnName = `mod.benchChurnSymbol${i}`;
|
|
45822
|
-
const representativeEdges = (original.edges ?? []).filter((e) => !ownSyms.has(e.dstQualifiedName)).slice(0, 2).map((e) => ({ ...e, srcQualifiedName: churnName }));
|
|
45823
|
-
const modified = {
|
|
45824
|
-
...original,
|
|
45825
|
-
contentHash: `${original.contentHash}-mod-${i}`,
|
|
45826
|
-
symbols: [
|
|
45827
|
-
{
|
|
45828
|
-
qualifiedName: churnName,
|
|
45829
|
-
name: `benchChurnSymbol${i}`,
|
|
45830
|
-
kind: "function",
|
|
45831
|
-
span: { startByte: 0, endByte: 0 }
|
|
45832
|
-
}
|
|
45833
|
-
],
|
|
45834
|
-
edges: representativeEdges
|
|
45835
|
-
};
|
|
45836
|
-
const modResult = await timeAsync(
|
|
45837
|
-
() => store.upsertFileBatch([modified])
|
|
45838
|
-
);
|
|
45839
|
-
if (!modResult.result.ok) {
|
|
45840
|
-
throw new Error(`modified update failed: ${modResult.result.code}`);
|
|
45841
|
-
}
|
|
45842
|
-
modifiedSamples.push(modResult.ms);
|
|
45843
|
-
const restoreResult = await store.upsertFileBatch(storeFiles);
|
|
45844
|
-
if (!restoreResult.ok) {
|
|
45845
|
-
throw new Error(`restore failed: ${restoreResult.code}`);
|
|
45846
|
-
}
|
|
45847
|
-
sampleRss();
|
|
45848
|
-
}
|
|
45849
|
-
sampleRss();
|
|
45850
|
-
const peakRssBytes = peakRss;
|
|
45851
|
-
return {
|
|
45852
|
-
schemaVersion: CODING_GRAPH_BENCH_SCHEMA_VERSION,
|
|
45853
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
45854
|
-
machine: captureMachineFingerprint(),
|
|
45855
|
-
fixture: {
|
|
45856
|
-
config: fixtureConfig,
|
|
45857
|
-
approximateLoc: repo.approximateLoc,
|
|
45858
|
-
fileCount: repo.files.length,
|
|
45859
|
-
symbolCount: repo.files.reduce((sum, f) => sum + f.symbols.length, 0),
|
|
45860
|
-
edgeCount: repo.files.reduce((sum, f) => sum + f.edges.length, 0)
|
|
45861
|
-
},
|
|
45862
|
-
fullIndexMs: { ms: fullIndexMsValue },
|
|
45863
|
-
fullIndexLocsPerSecond: Math.round(locsPerSecond),
|
|
45864
|
-
incrementalUpdate: computeMicroMetric(incrementalSamples),
|
|
45865
|
-
incrementalModifiedUpdate: computeMicroMetric(
|
|
45866
|
-
modifiedSamples.length > 0 ? modifiedSamples : [0]
|
|
45867
|
-
),
|
|
45868
|
-
tracePath: computeMicroMetric(traceSamples.length > 0 ? traceSamples : [0]),
|
|
45869
|
-
searchGraph: computeMicroMetric(searchSamples),
|
|
45870
|
-
deadCodeMs: { ms: deadCode.ms },
|
|
45871
|
-
dbBytesPerKloc: Math.round(dbBytesPerKloc),
|
|
45872
|
-
peakRssBytes,
|
|
45873
|
-
dbBytes,
|
|
45874
|
-
graphNodeCount,
|
|
45875
|
-
graphEdgeCount
|
|
45876
|
-
};
|
|
45877
|
-
} finally {
|
|
45878
|
-
await store.close();
|
|
45879
|
-
}
|
|
45880
|
-
} finally {
|
|
45881
|
-
await rm15(dir, { recursive: true, force: true });
|
|
45882
|
-
}
|
|
45883
|
-
}
|
|
45884
|
-
|
|
45885
|
-
// src/coding-graph/regression.ts
|
|
45886
|
-
var METRIC_DIRECTION = {
|
|
45887
|
-
fullIndexMs: "lower-is-better",
|
|
45888
|
-
fullIndexLocsPerSecond: "higher-is-better",
|
|
45889
|
-
incrementalUpdateP95Ms: "lower-is-better",
|
|
45890
|
-
incrementalUpdateP50Ms: "lower-is-better",
|
|
45891
|
-
incrementalModifiedUpdateP95Ms: "lower-is-better",
|
|
45892
|
-
incrementalModifiedUpdateP50Ms: "lower-is-better",
|
|
45893
|
-
tracePathP95Ms: "lower-is-better",
|
|
45894
|
-
searchGraphP95Ms: "lower-is-better",
|
|
45895
|
-
deadCodeMs: "lower-is-better",
|
|
45896
|
-
dbBytesPerKloc: "lower-is-better"
|
|
45897
|
-
};
|
|
45898
|
-
function extractMetrics(report) {
|
|
45899
|
-
return {
|
|
45900
|
-
fullIndexMs: report.fullIndexMs.ms,
|
|
45901
|
-
fullIndexLocsPerSecond: report.fullIndexLocsPerSecond,
|
|
45902
|
-
incrementalUpdateP50Ms: report.incrementalUpdate.p50,
|
|
45903
|
-
incrementalUpdateP95Ms: report.incrementalUpdate.p95,
|
|
45904
|
-
incrementalModifiedUpdateP50Ms: report.incrementalModifiedUpdate.p50,
|
|
45905
|
-
incrementalModifiedUpdateP95Ms: report.incrementalModifiedUpdate.p95,
|
|
45906
|
-
tracePathP95Ms: report.tracePath.p95,
|
|
45907
|
-
searchGraphP95Ms: report.searchGraph.p95,
|
|
45908
|
-
deadCodeMs: report.deadCodeMs.ms,
|
|
45909
|
-
dbBytesPerKloc: report.dbBytesPerKloc
|
|
45910
|
-
};
|
|
45911
|
-
}
|
|
45912
|
-
var NODE_MAJOR_CACHE = /* @__PURE__ */ new WeakMap();
|
|
45913
|
-
function nodeMajor(fp) {
|
|
45914
|
-
const cached = NODE_MAJOR_CACHE.get(fp);
|
|
45915
|
-
if (cached !== void 0) return cached;
|
|
45916
|
-
const major = fp.nodeVersion.replace(/^v/, "").split(".")[0] ?? fp.nodeVersion;
|
|
45917
|
-
NODE_MAJOR_CACHE.set(fp, major);
|
|
45918
|
-
return major;
|
|
45919
|
-
}
|
|
45920
|
-
function compareMachineFingerprints(report, baseline) {
|
|
45921
|
-
const differing = [];
|
|
45922
|
-
if (report.arch !== baseline.arch) differing.push("arch");
|
|
45923
|
-
if (report.platform !== baseline.platform) differing.push("platform");
|
|
45924
|
-
if (nodeMajor(report) !== nodeMajor(baseline)) differing.push("nodeVersion(major)");
|
|
45925
|
-
if (report.cpuModel !== null && baseline.cpuModel !== null && report.cpuModel !== baseline.cpuModel) {
|
|
45926
|
-
differing.push("cpuModel");
|
|
45927
|
-
}
|
|
45928
|
-
if (report.cpuCores !== baseline.cpuCores) differing.push("cpuCores");
|
|
45929
|
-
return { differingFields: differing };
|
|
45930
|
-
}
|
|
45931
|
-
function invalidFingerprintFields(fp) {
|
|
45932
|
-
const bad = [];
|
|
45933
|
-
if (typeof fp.arch !== "string") bad.push("arch");
|
|
45934
|
-
if (typeof fp.platform !== "string") bad.push("platform");
|
|
45935
|
-
if (typeof fp.nodeVersion !== "string") bad.push("nodeVersion");
|
|
45936
|
-
if (typeof fp.cpuCores !== "number" || !Number.isFinite(fp.cpuCores)) bad.push("cpuCores");
|
|
45937
|
-
if (fp.cpuModel !== null && typeof fp.cpuModel !== "string") bad.push("cpuModel");
|
|
45938
|
-
return bad;
|
|
45939
|
-
}
|
|
45940
|
-
function checkCodingGraphRegression(report, baseline, tolerancePercent = DEFAULT_TOLERANCE_PERCENT) {
|
|
45941
|
-
const reportFixture = report.fixture.config;
|
|
45942
|
-
const baselineFixture = baseline.fixtureConfig;
|
|
45943
|
-
const mismatchedKeys = Object.keys(baselineFixture).filter((key) => reportFixture[key] !== baselineFixture[key]);
|
|
45944
|
-
if (mismatchedKeys.length > 0) {
|
|
45945
|
-
const diffs = mismatchedKeys.map(
|
|
45946
|
-
(key) => `${key}: report=${reportFixture[key]} baseline=${baselineFixture[key]}`
|
|
45947
|
-
).join(", ");
|
|
45948
|
-
return {
|
|
45949
|
-
passed: false,
|
|
45950
|
-
regressions: [],
|
|
45951
|
-
summary: `Fixture mismatch (${diffs}). Metrics are not comparable across different fixtures.`
|
|
45952
|
-
};
|
|
45953
|
-
}
|
|
45954
|
-
if (report.schemaVersion !== baseline.schemaVersion) {
|
|
45955
|
-
return {
|
|
45956
|
-
passed: false,
|
|
45957
|
-
regressions: [],
|
|
45958
|
-
summary: `Schema-version mismatch: report=${report.schemaVersion} baseline=${baseline.schemaVersion}. Metrics are not comparable across schema versions \u2014 regenerate the baseline (#1688).`
|
|
45959
|
-
};
|
|
45960
|
-
}
|
|
45961
|
-
const metricChecks = [
|
|
45962
|
-
["incrementalUpdate.p50", report.incrementalUpdate?.p50],
|
|
45963
|
-
["incrementalUpdate.p95", report.incrementalUpdate?.p95],
|
|
45964
|
-
["incrementalModifiedUpdate.p50", report.incrementalModifiedUpdate?.p50],
|
|
45965
|
-
["incrementalModifiedUpdate.p95", report.incrementalModifiedUpdate?.p95],
|
|
45966
|
-
["tracePath.p95", report.tracePath?.p95],
|
|
45967
|
-
["searchGraph.p95", report.searchGraph?.p95],
|
|
45968
|
-
["fullIndexMs.ms", report.fullIndexMs?.ms],
|
|
45969
|
-
["fullIndexLocsPerSecond", report.fullIndexLocsPerSecond],
|
|
45970
|
-
["deadCodeMs.ms", report.deadCodeMs?.ms],
|
|
45971
|
-
["dbBytesPerKloc", report.dbBytesPerKloc]
|
|
45972
|
-
];
|
|
45973
|
-
const missingFields = metricChecks.filter(([, v]) => typeof v !== "number" || !Number.isFinite(v)).map(([k]) => k);
|
|
45974
|
-
if (missingFields.length > 0) {
|
|
45975
|
-
return {
|
|
45976
|
-
passed: false,
|
|
45977
|
-
regressions: [],
|
|
45978
|
-
summary: "Report claims schemaVersion " + report.schemaVersion + " but is missing required metric field(s): " + missingFields.join(", ") + " \u2014 the report is incomplete or corrupt. Regenerate it (#1688)."
|
|
45979
|
-
};
|
|
45980
|
-
}
|
|
45981
|
-
const badBaselineMetrics = Object.keys(METRIC_DIRECTION).filter((key) => {
|
|
45982
|
-
const v = baseline.metrics[key];
|
|
45983
|
-
return v != null && (typeof v !== "number" || !Number.isFinite(v));
|
|
45984
|
-
});
|
|
45985
|
-
if (badBaselineMetrics.length > 0) {
|
|
45986
|
-
return {
|
|
45987
|
-
passed: false,
|
|
45988
|
-
regressions: [],
|
|
45989
|
-
summary: "Baseline has a non-numeric required metric field(s): " + badBaselineMetrics.join(", ") + " \u2014 the baseline is corrupt. Regenerate it (#1688)."
|
|
45990
|
-
};
|
|
45991
|
-
}
|
|
45992
|
-
const reportFpBad = report.machine == null ? ["<missing>"] : invalidFingerprintFields(report.machine);
|
|
45993
|
-
const baselineFpBad = baseline.machine == null ? ["<missing>"] : invalidFingerprintFields(baseline.machine);
|
|
45994
|
-
if (reportFpBad.length > 0 || baselineFpBad.length > 0) {
|
|
45995
|
-
const which = [];
|
|
45996
|
-
if (reportFpBad.length > 0) which.push("report (" + reportFpBad.join(", ") + ")");
|
|
45997
|
-
if (baselineFpBad.length > 0) which.push("baseline (" + baselineFpBad.join(", ") + ")");
|
|
45998
|
-
return {
|
|
45999
|
-
passed: false,
|
|
46000
|
-
regressions: [],
|
|
46001
|
-
summary: "Report or baseline machine fingerprint is missing or has an invalid field type(s): " + which.join("; ") + " \u2014 the artifact is incomplete or corrupt. Regenerate it (#1688)."
|
|
46002
|
-
};
|
|
46003
|
-
}
|
|
46004
|
-
const mismatch = compareMachineFingerprints(report.machine, baseline.machine);
|
|
46005
|
-
if (mismatch.differingFields.length > 0) {
|
|
46006
|
-
return {
|
|
46007
|
-
passed: true,
|
|
46008
|
-
skipped: true,
|
|
46009
|
-
regressions: [],
|
|
46010
|
-
summary: "Machine-fingerprint mismatch \u2014 comparison skipped to avoid a false-positive hardware-variance failure. Differing fields: " + mismatch.differingFields.join(", ") + ". Regenerate the baseline on this machine for a real comparison (#1688).",
|
|
46011
|
-
machineMismatch: {
|
|
46012
|
-
report: report.machine,
|
|
46013
|
-
baseline: baseline.machine,
|
|
46014
|
-
differingFields: mismatch.differingFields
|
|
46015
|
-
}
|
|
46016
|
-
};
|
|
46017
|
-
}
|
|
46018
|
-
const measured = extractMetrics(report);
|
|
46019
|
-
const baselineMetrics = baseline.metrics;
|
|
46020
|
-
const regressions = [];
|
|
46021
|
-
for (const key of Object.keys(METRIC_DIRECTION)) {
|
|
46022
|
-
const baseVal = baselineMetrics[key];
|
|
46023
|
-
const measVal = measured[key];
|
|
46024
|
-
if (baseVal == null || measVal == null) continue;
|
|
46025
|
-
if (baseVal === 0) continue;
|
|
46026
|
-
const direction = METRIC_DIRECTION[key];
|
|
46027
|
-
const ratio2 = measVal / baseVal;
|
|
46028
|
-
const percentChange2 = direction === "lower-is-better" ? (ratio2 - 1) * 100 : (1 - ratio2) * 100;
|
|
46029
|
-
const regressed = percentChange2 > tolerancePercent;
|
|
46030
|
-
if (regressed) {
|
|
46031
|
-
regressions.push({
|
|
46032
|
-
key,
|
|
46033
|
-
baseline: baseVal,
|
|
46034
|
-
measured: measVal,
|
|
46035
|
-
percentChange: Math.round(percentChange2 * 10) / 10,
|
|
46036
|
-
direction,
|
|
46037
|
-
tolerancePercent,
|
|
46038
|
-
regressed: true
|
|
46039
|
-
});
|
|
46040
|
-
}
|
|
46041
|
-
}
|
|
46042
|
-
const passed = regressions.length === 0;
|
|
46043
|
-
const summary = passed ? "All metrics within tolerance." : `${regressions.length} metric(s) regressed beyond ${tolerancePercent}% tolerance:
|
|
46044
|
-
` + regressions.map(
|
|
46045
|
-
(r) => ` ${r.key}: ${r.baseline} \u2192 ${r.measured} (${r.percentChange > 0 ? "+" : ""}${r.percentChange}% vs baseline)`
|
|
46046
|
-
).join("\n");
|
|
46047
|
-
return { passed, regressions, summary };
|
|
46048
|
-
}
|
|
46049
|
-
function buildBaselineFromReport(report, note) {
|
|
46050
|
-
return {
|
|
46051
|
-
schemaVersion: report.schemaVersion,
|
|
46052
|
-
machine: report.machine,
|
|
46053
|
-
fixtureConfig: report.fixture.config,
|
|
46054
|
-
metrics: extractMetrics(report),
|
|
46055
|
-
createdAt: report.timestamp,
|
|
46056
|
-
note
|
|
46057
|
-
};
|
|
46058
|
-
}
|
|
46059
|
-
|
|
46060
|
-
// src/attribution.ts
|
|
46061
|
-
var DEFAULT_STOPWORDS = /* @__PURE__ */ new Set([
|
|
46062
|
-
"a",
|
|
46063
|
-
"an",
|
|
46064
|
-
"the",
|
|
46065
|
-
"in",
|
|
46066
|
-
"on",
|
|
46067
|
-
"at",
|
|
46068
|
-
"to",
|
|
46069
|
-
"for",
|
|
46070
|
-
"of",
|
|
46071
|
-
"with",
|
|
46072
|
-
"by",
|
|
46073
|
-
"from",
|
|
46074
|
-
"up",
|
|
46075
|
-
"about",
|
|
46076
|
-
"into",
|
|
46077
|
-
"through",
|
|
46078
|
-
"during",
|
|
46079
|
-
"before",
|
|
46080
|
-
"after",
|
|
46081
|
-
"above",
|
|
46082
|
-
"below",
|
|
46083
|
-
"and",
|
|
46084
|
-
"or",
|
|
46085
|
-
"but",
|
|
46086
|
-
"if",
|
|
46087
|
-
"then",
|
|
46088
|
-
"else",
|
|
46089
|
-
"when",
|
|
46090
|
-
"where",
|
|
46091
|
-
"why",
|
|
46092
|
-
"how",
|
|
46093
|
-
"all",
|
|
46094
|
-
"any",
|
|
46095
|
-
"both",
|
|
46096
|
-
"each",
|
|
46097
|
-
"few",
|
|
46098
|
-
"more",
|
|
46099
|
-
"most",
|
|
46100
|
-
"other",
|
|
46101
|
-
"some",
|
|
46102
|
-
"such",
|
|
46103
|
-
"no",
|
|
46104
|
-
"nor",
|
|
46105
|
-
"not",
|
|
46106
|
-
"only",
|
|
46107
|
-
"own",
|
|
46108
|
-
"same",
|
|
46109
|
-
"so",
|
|
46110
|
-
"than",
|
|
46111
|
-
"too",
|
|
46112
|
-
"very",
|
|
46113
|
-
"this",
|
|
46114
|
-
"that",
|
|
46115
|
-
"these",
|
|
46116
|
-
"those",
|
|
46117
|
-
"it",
|
|
46118
|
-
"its",
|
|
46119
|
-
"is",
|
|
46120
|
-
"are",
|
|
46121
|
-
"was",
|
|
46122
|
-
"were",
|
|
46123
|
-
"be",
|
|
46124
|
-
"been",
|
|
46125
|
-
"being",
|
|
46126
|
-
"have",
|
|
46127
|
-
"has",
|
|
46128
|
-
"had",
|
|
46129
|
-
"do",
|
|
46130
|
-
"does",
|
|
46131
|
-
"did"
|
|
46132
|
-
]);
|
|
46133
|
-
function extractContentWords(text) {
|
|
46134
|
-
const cleaned = text.toLowerCase().replace(/[^a-z0-9\s]/g, " ");
|
|
46135
|
-
const tokens = cleaned.split(/\s+/).filter(Boolean);
|
|
46136
|
-
return tokens.filter((t) => !DEFAULT_STOPWORDS.has(t));
|
|
46137
|
-
}
|
|
46138
|
-
function lexicalSimilarity(a, b) {
|
|
46139
|
-
const goldWords = extractContentWords(a);
|
|
46140
|
-
if (goldWords.length === 0) {
|
|
46141
|
-
return 0;
|
|
46142
|
-
}
|
|
46143
|
-
const candWords = new Set(extractContentWords(b));
|
|
46144
|
-
let matchCount = 0;
|
|
46145
|
-
for (const word of goldWords) {
|
|
46146
|
-
if (candWords.has(word)) {
|
|
46147
|
-
matchCount++;
|
|
46148
|
-
}
|
|
46149
|
-
}
|
|
46150
|
-
return matchCount / goldWords.length;
|
|
46151
|
-
}
|
|
46152
|
-
var CLASS_RANK = {
|
|
46153
|
-
extraction_miss: 1,
|
|
46154
|
-
index_miss: 2,
|
|
46155
|
-
retrieval_miss: 3,
|
|
46156
|
-
use_miss: 4,
|
|
46157
|
-
unattributed: 5
|
|
46158
|
-
};
|
|
46159
|
-
var RETRIEVAL_STAGE_RANK = {
|
|
46160
|
-
cap: 1,
|
|
46161
|
-
rank: 2,
|
|
46162
|
-
filter: 3,
|
|
46163
|
-
unknown: 4
|
|
46164
|
-
};
|
|
46165
|
-
function computeOverallLabel(golds) {
|
|
46166
|
-
if (golds.length === 0) {
|
|
46167
|
-
return { class: "unattributed", reason: "no gold memories" };
|
|
46168
|
-
}
|
|
46169
|
-
let bestGold = golds[0];
|
|
46170
|
-
for (let i = 1; i < golds.length; i++) {
|
|
46171
|
-
const current = golds[i];
|
|
46172
|
-
const bestRank = CLASS_RANK[bestGold.label.class];
|
|
46173
|
-
const currRank = CLASS_RANK[current.label.class];
|
|
46174
|
-
if (currRank < bestRank) {
|
|
46175
|
-
bestGold = current;
|
|
46176
|
-
} else if (currRank === bestRank && current.label.class === "retrieval_miss") {
|
|
46177
|
-
const bestRetRank = RETRIEVAL_STAGE_RANK[bestGold.label.retrievalStage ?? "unknown"];
|
|
46178
|
-
const currRetRank = RETRIEVAL_STAGE_RANK[current.label.retrievalStage ?? "unknown"];
|
|
46179
|
-
if (currRetRank < bestRetRank) {
|
|
46180
|
-
bestGold = current;
|
|
46181
|
-
}
|
|
46182
|
-
}
|
|
46183
|
-
}
|
|
46184
|
-
return { ...bestGold.label };
|
|
46185
|
-
}
|
|
46186
|
-
function isDiagnosticScore(name) {
|
|
46187
|
-
return name.endsWith("_agreement") || name.includes("_id_leak") || name === "search_hits";
|
|
46188
|
-
}
|
|
46189
|
-
function isTaskFailed(task) {
|
|
46190
|
-
if (!task.scores || Object.keys(task.scores).length === 0) {
|
|
46191
|
-
return true;
|
|
46192
|
-
}
|
|
46193
|
-
if ("overall" in task.scores && typeof task.scores.overall === "number") {
|
|
46194
|
-
return task.scores.overall < 1;
|
|
46195
|
-
}
|
|
46196
|
-
const primaryScores = Object.entries(task.scores).filter(([name, score]) => typeof score === "number" && !isDiagnosticScore(name)).map(([, score]) => score);
|
|
46197
|
-
return primaryScores.length === 0 || Math.min(...primaryScores) < 1;
|
|
46198
|
-
}
|
|
46199
|
-
function withMemoizedListMemories(env) {
|
|
46200
|
-
let cache = null;
|
|
46201
|
-
return {
|
|
46202
|
-
...env,
|
|
46203
|
-
listMemories() {
|
|
46204
|
-
if (!cache) {
|
|
46205
|
-
cache = env.listMemories();
|
|
46206
|
-
}
|
|
46207
|
-
return cache;
|
|
46208
|
-
}
|
|
46209
|
-
};
|
|
46210
|
-
}
|
|
46211
|
-
async function attributeGoldMemory(goldStatement, question, env, options = {}, recalledText) {
|
|
46212
|
-
if (options.threshold !== void 0) {
|
|
46213
|
-
if (typeof options.threshold !== "number" || !Number.isFinite(options.threshold) || options.threshold < 0 || options.threshold > 1) {
|
|
46214
|
-
throw new RangeError("attribution threshold must be a finite number between 0 and 1");
|
|
46215
|
-
}
|
|
46216
|
-
}
|
|
46217
|
-
const threshold = options.threshold ?? 0.6;
|
|
46218
|
-
const simFn = options.similarity ?? lexicalSimilarity;
|
|
46219
|
-
const goldInRecalledText = typeof recalledText === "string" && simFn(goldStatement, recalledText) >= threshold;
|
|
46220
|
-
const stages = {
|
|
46221
|
-
extraction: { status: "unavailable" },
|
|
46222
|
-
index: { status: "unavailable" },
|
|
46223
|
-
retrieval: { status: "unavailable" },
|
|
46224
|
-
use: { status: "unavailable" }
|
|
46225
|
-
};
|
|
46226
|
-
let memories = [];
|
|
46227
|
-
let extractionRan = false;
|
|
46228
|
-
let extractionErrorDetail;
|
|
46229
|
-
if (typeof env.listMemories === "function") {
|
|
46230
|
-
try {
|
|
46231
|
-
memories = await env.listMemories();
|
|
46232
|
-
extractionRan = true;
|
|
46233
|
-
} catch {
|
|
46234
|
-
extractionRan = false;
|
|
46235
|
-
extractionErrorDetail = "listMemories failed";
|
|
46236
|
-
}
|
|
46237
|
-
}
|
|
46238
|
-
let bestSim = -1;
|
|
46239
|
-
let matchedMem = null;
|
|
46240
|
-
if (extractionRan) {
|
|
46241
|
-
if (memories.length === 0) {
|
|
46242
|
-
if (goldInRecalledText) {
|
|
46243
|
-
const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
|
|
46244
|
-
stages.extraction = { status: "pass", detail: impliedDetail };
|
|
46245
|
-
stages.index = { status: "pass", detail: impliedDetail };
|
|
46246
|
-
stages.retrieval = { status: "pass", detail: impliedDetail };
|
|
46247
|
-
stages.use = {
|
|
46248
|
-
status: "fail",
|
|
46249
|
-
detail: "Gold memory present in context but answer was incorrect"
|
|
46250
|
-
};
|
|
46251
|
-
return {
|
|
46252
|
-
goldMemory: goldStatement,
|
|
46253
|
-
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
46254
|
-
stages
|
|
46255
|
-
};
|
|
46256
|
-
}
|
|
46257
|
-
const detail = "store contains no memories";
|
|
46258
|
-
stages.extraction = { status: "fail", detail };
|
|
46259
|
-
stages.index = { status: "unavailable", detail: "not reached" };
|
|
46260
|
-
stages.retrieval = { status: "unavailable", detail: "not reached" };
|
|
46261
|
-
stages.use = { status: "unavailable", detail: "not reached" };
|
|
46262
|
-
return {
|
|
46263
|
-
goldMemory: goldStatement,
|
|
46264
|
-
label: { class: "extraction_miss", reason: detail },
|
|
46265
|
-
stages
|
|
46266
|
-
};
|
|
46267
|
-
}
|
|
46268
|
-
for (const mem of memories) {
|
|
46269
|
-
const sim = simFn(goldStatement, mem.content);
|
|
46270
|
-
if (sim > bestSim) {
|
|
46271
|
-
bestSim = sim;
|
|
46272
|
-
matchedMem = mem;
|
|
46273
|
-
}
|
|
46274
|
-
}
|
|
46275
|
-
if (bestSim < threshold || !matchedMem) {
|
|
46276
|
-
if (goldInRecalledText) {
|
|
46277
|
-
const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
|
|
46278
|
-
stages.extraction = { status: "pass", detail: impliedDetail };
|
|
46279
|
-
stages.index = { status: "pass", detail: impliedDetail };
|
|
46280
|
-
stages.retrieval = { status: "pass", detail: impliedDetail };
|
|
46281
|
-
stages.use = {
|
|
46282
|
-
status: "fail",
|
|
46283
|
-
detail: "Gold memory present in context but answer was incorrect"
|
|
46284
|
-
};
|
|
46285
|
-
return {
|
|
46286
|
-
goldMemory: goldStatement,
|
|
46287
|
-
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
46288
|
-
stages
|
|
46289
|
-
};
|
|
46715
|
+
}
|
|
46716
|
+
searchSamples.push(searchRes.ms);
|
|
46290
46717
|
}
|
|
46291
|
-
const
|
|
46292
|
-
|
|
46293
|
-
|
|
46294
|
-
stages.retrieval = { status: "unavailable", detail: "not reached" };
|
|
46295
|
-
stages.use = { status: "unavailable", detail: "not reached" };
|
|
46296
|
-
return {
|
|
46297
|
-
goldMemory: goldStatement,
|
|
46298
|
-
label: { class: "extraction_miss", reason: detail },
|
|
46299
|
-
stages
|
|
46300
|
-
};
|
|
46301
|
-
}
|
|
46302
|
-
stages.extraction = {
|
|
46303
|
-
status: "pass",
|
|
46304
|
-
detail: `Matched memory ${matchedMem.id} (sim ${bestSim.toFixed(3)})`
|
|
46305
|
-
};
|
|
46306
|
-
} else {
|
|
46307
|
-
stages.extraction = {
|
|
46308
|
-
status: "unavailable",
|
|
46309
|
-
detail: extractionErrorDetail ?? "listMemories unavailable"
|
|
46310
|
-
};
|
|
46311
|
-
}
|
|
46312
|
-
const matchedMemoryId = matchedMem ? matchedMem.id : void 0;
|
|
46313
|
-
const recallLimit = env.recallLimit;
|
|
46314
|
-
const replayLimit = env.replayLimit ?? Math.max(25, recallLimit * 5);
|
|
46315
|
-
let indexCheckPassed = false;
|
|
46316
|
-
let indexCheckFailed = false;
|
|
46317
|
-
if (typeof env.oracleSearch === "function" && extractionRan) {
|
|
46318
|
-
try {
|
|
46319
|
-
const oracleResults = await env.oracleSearch(goldStatement, replayLimit);
|
|
46320
|
-
const idMatched = matchedMemoryId ? oracleResults.some((r) => r.id === matchedMemoryId) : false;
|
|
46321
|
-
if (idMatched) {
|
|
46322
|
-
indexCheckPassed = true;
|
|
46323
|
-
} else {
|
|
46324
|
-
const memMap = new Map(memories.map((m) => [m.id, m]));
|
|
46325
|
-
indexCheckPassed = oracleResults.some((r) => {
|
|
46326
|
-
const mem = memMap.get(r.id);
|
|
46327
|
-
return mem ? simFn(goldStatement, mem.content) >= threshold : false;
|
|
46328
|
-
});
|
|
46718
|
+
const deadCode = timeSync(() => store.deadCode());
|
|
46719
|
+
if (!deadCode.result.ok) {
|
|
46720
|
+
throw new Error(`dead_code failed: ${deadCode.result.code}`);
|
|
46329
46721
|
}
|
|
46330
|
-
|
|
46331
|
-
|
|
46332
|
-
|
|
46333
|
-
|
|
46334
|
-
|
|
46722
|
+
await store.drain();
|
|
46723
|
+
let dbBytes = statSync(dbPath).size;
|
|
46724
|
+
try {
|
|
46725
|
+
dbBytes += statSync(dbPath + "-wal").size;
|
|
46726
|
+
} catch {
|
|
46335
46727
|
}
|
|
46336
|
-
|
|
46337
|
-
|
|
46338
|
-
|
|
46339
|
-
|
|
46340
|
-
|
|
46341
|
-
|
|
46342
|
-
|
|
46343
|
-
|
|
46344
|
-
|
|
46345
|
-
|
|
46346
|
-
|
|
46347
|
-
|
|
46348
|
-
|
|
46349
|
-
|
|
46350
|
-
|
|
46351
|
-
|
|
46352
|
-
|
|
46353
|
-
|
|
46354
|
-
|
|
46355
|
-
|
|
46356
|
-
|
|
46357
|
-
|
|
46358
|
-
|
|
46359
|
-
const
|
|
46360
|
-
|
|
46361
|
-
(m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
|
|
46728
|
+
const kloc = Math.max(1, repo.approximateLoc / 1e3);
|
|
46729
|
+
const dbBytesPerKloc = dbBytes / kloc;
|
|
46730
|
+
const modifiedSamples = [];
|
|
46731
|
+
for (let i = 0; i < iterations; i++) {
|
|
46732
|
+
const fileIdx = i % storeFiles.length;
|
|
46733
|
+
const original = storeFiles[fileIdx];
|
|
46734
|
+
if (!original) continue;
|
|
46735
|
+
const ownSyms = new Set(original.symbols.map((sym) => sym.qualifiedName));
|
|
46736
|
+
const churnName = `mod.benchChurnSymbol${i}`;
|
|
46737
|
+
const representativeEdges = (original.edges ?? []).filter((e) => !ownSyms.has(e.dstQualifiedName)).slice(0, 2).map((e) => ({ ...e, srcQualifiedName: churnName }));
|
|
46738
|
+
const modified = {
|
|
46739
|
+
...original,
|
|
46740
|
+
contentHash: `${original.contentHash}-mod-${i}`,
|
|
46741
|
+
symbols: [
|
|
46742
|
+
{
|
|
46743
|
+
qualifiedName: churnName,
|
|
46744
|
+
name: `benchChurnSymbol${i}`,
|
|
46745
|
+
kind: "function",
|
|
46746
|
+
span: { startByte: 0, endByte: 0 }
|
|
46747
|
+
}
|
|
46748
|
+
],
|
|
46749
|
+
edges: representativeEdges
|
|
46750
|
+
};
|
|
46751
|
+
const modResult = await timeAsync(
|
|
46752
|
+
() => store.upsertFileBatch([modified])
|
|
46362
46753
|
);
|
|
46363
|
-
if (
|
|
46364
|
-
|
|
46365
|
-
|
|
46366
|
-
|
|
46367
|
-
|
|
46368
|
-
|
|
46369
|
-
};
|
|
46370
|
-
} else {
|
|
46371
|
-
retrievalStageMiss = "unknown";
|
|
46372
|
-
stages.retrieval = {
|
|
46373
|
-
status: "fail",
|
|
46374
|
-
detail: `absent from recall at replayLimit ${replayLimit}; filter vs rank indistinguishable without candidate-stage evidence`
|
|
46375
|
-
};
|
|
46754
|
+
if (!modResult.result.ok) {
|
|
46755
|
+
throw new Error(`modified update failed: ${modResult.result.code}`);
|
|
46756
|
+
}
|
|
46757
|
+
modifiedSamples.push(modResult.ms);
|
|
46758
|
+
const restoreResult = await store.upsertFileBatch(storeFiles);
|
|
46759
|
+
if (!restoreResult.ok) {
|
|
46760
|
+
throw new Error(`restore failed: ${restoreResult.code}`);
|
|
46376
46761
|
}
|
|
46762
|
+
sampleRss();
|
|
46377
46763
|
}
|
|
46378
|
-
|
|
46379
|
-
|
|
46764
|
+
sampleRss();
|
|
46765
|
+
const peakRssBytes = peakRss;
|
|
46766
|
+
return {
|
|
46767
|
+
schemaVersion: CODING_GRAPH_BENCH_SCHEMA_VERSION,
|
|
46768
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
46769
|
+
machine: captureMachineFingerprint(),
|
|
46770
|
+
fixture: {
|
|
46771
|
+
config: fixtureConfig,
|
|
46772
|
+
approximateLoc: repo.approximateLoc,
|
|
46773
|
+
fileCount: repo.files.length,
|
|
46774
|
+
symbolCount: repo.files.reduce((sum, f) => sum + f.symbols.length, 0),
|
|
46775
|
+
edgeCount: repo.files.reduce((sum, f) => sum + f.edges.length, 0)
|
|
46776
|
+
},
|
|
46777
|
+
fullIndexMs: { ms: fullIndexMsValue },
|
|
46778
|
+
fullIndexLocsPerSecond: Math.round(locsPerSecond),
|
|
46779
|
+
incrementalUpdate: computeMicroMetric(incrementalSamples),
|
|
46780
|
+
incrementalModifiedUpdate: computeMicroMetric(
|
|
46781
|
+
modifiedSamples.length > 0 ? modifiedSamples : [0]
|
|
46782
|
+
),
|
|
46783
|
+
tracePath: computeMicroMetric(traceSamples.length > 0 ? traceSamples : [0]),
|
|
46784
|
+
searchGraph: computeMicroMetric(searchSamples),
|
|
46785
|
+
deadCodeMs: { ms: deadCode.ms },
|
|
46786
|
+
dbBytesPerKloc: Math.round(dbBytesPerKloc),
|
|
46787
|
+
peakRssBytes,
|
|
46788
|
+
dbBytes,
|
|
46789
|
+
graphNodeCount,
|
|
46790
|
+
graphEdgeCount
|
|
46791
|
+
};
|
|
46792
|
+
} finally {
|
|
46793
|
+
await store.close();
|
|
46380
46794
|
}
|
|
46381
|
-
}
|
|
46382
|
-
|
|
46383
|
-
}
|
|
46384
|
-
if (!retrievalCheckPassed && goldInRecalledText) {
|
|
46385
|
-
retrievalCheckPassed = true;
|
|
46386
|
-
stages.retrieval = { status: "pass", detail: "Found in recalledText context" };
|
|
46795
|
+
} finally {
|
|
46796
|
+
await rm15(dir, { recursive: true, force: true });
|
|
46387
46797
|
}
|
|
46388
|
-
|
|
46389
|
-
|
|
46390
|
-
|
|
46391
|
-
|
|
46392
|
-
|
|
46393
|
-
|
|
46394
|
-
|
|
46395
|
-
|
|
46798
|
+
}
|
|
46799
|
+
|
|
46800
|
+
// src/coding-graph/regression.ts
|
|
46801
|
+
var METRIC_DIRECTION = {
|
|
46802
|
+
fullIndexMs: "lower-is-better",
|
|
46803
|
+
fullIndexLocsPerSecond: "higher-is-better",
|
|
46804
|
+
incrementalUpdateP95Ms: "lower-is-better",
|
|
46805
|
+
incrementalUpdateP50Ms: "lower-is-better",
|
|
46806
|
+
incrementalModifiedUpdateP95Ms: "lower-is-better",
|
|
46807
|
+
incrementalModifiedUpdateP50Ms: "lower-is-better",
|
|
46808
|
+
tracePathP95Ms: "lower-is-better",
|
|
46809
|
+
searchGraphP95Ms: "lower-is-better",
|
|
46810
|
+
deadCodeMs: "lower-is-better",
|
|
46811
|
+
dbBytesPerKloc: "lower-is-better"
|
|
46812
|
+
};
|
|
46813
|
+
function extractMetrics(report) {
|
|
46814
|
+
return {
|
|
46815
|
+
fullIndexMs: report.fullIndexMs.ms,
|
|
46816
|
+
fullIndexLocsPerSecond: report.fullIndexLocsPerSecond,
|
|
46817
|
+
incrementalUpdateP50Ms: report.incrementalUpdate.p50,
|
|
46818
|
+
incrementalUpdateP95Ms: report.incrementalUpdate.p95,
|
|
46819
|
+
incrementalModifiedUpdateP50Ms: report.incrementalModifiedUpdate.p50,
|
|
46820
|
+
incrementalModifiedUpdateP95Ms: report.incrementalModifiedUpdate.p95,
|
|
46821
|
+
tracePathP95Ms: report.tracePath.p95,
|
|
46822
|
+
searchGraphP95Ms: report.searchGraph.p95,
|
|
46823
|
+
deadCodeMs: report.deadCodeMs.ms,
|
|
46824
|
+
dbBytesPerKloc: report.dbBytesPerKloc
|
|
46825
|
+
};
|
|
46826
|
+
}
|
|
46827
|
+
var NODE_MAJOR_CACHE = /* @__PURE__ */ new WeakMap();
|
|
46828
|
+
function nodeMajor(fp) {
|
|
46829
|
+
const cached = NODE_MAJOR_CACHE.get(fp);
|
|
46830
|
+
if (cached !== void 0) return cached;
|
|
46831
|
+
const major = fp.nodeVersion.replace(/^v/, "").split(".")[0] ?? fp.nodeVersion;
|
|
46832
|
+
NODE_MAJOR_CACHE.set(fp, major);
|
|
46833
|
+
return major;
|
|
46834
|
+
}
|
|
46835
|
+
function compareMachineFingerprints(report, baseline) {
|
|
46836
|
+
const differing = [];
|
|
46837
|
+
if (report.arch !== baseline.arch) differing.push("arch");
|
|
46838
|
+
if (report.platform !== baseline.platform) differing.push("platform");
|
|
46839
|
+
if (nodeMajor(report) !== nodeMajor(baseline)) differing.push("nodeVersion(major)");
|
|
46840
|
+
if (report.cpuModel !== null && baseline.cpuModel !== null && report.cpuModel !== baseline.cpuModel) {
|
|
46841
|
+
differing.push("cpuModel");
|
|
46396
46842
|
}
|
|
46397
|
-
if (
|
|
46398
|
-
|
|
46399
|
-
|
|
46400
|
-
|
|
46843
|
+
if (report.cpuCores !== baseline.cpuCores) differing.push("cpuCores");
|
|
46844
|
+
return { differingFields: differing };
|
|
46845
|
+
}
|
|
46846
|
+
function invalidFingerprintFields(fp) {
|
|
46847
|
+
const bad = [];
|
|
46848
|
+
if (typeof fp.arch !== "string") bad.push("arch");
|
|
46849
|
+
if (typeof fp.platform !== "string") bad.push("platform");
|
|
46850
|
+
if (typeof fp.nodeVersion !== "string") bad.push("nodeVersion");
|
|
46851
|
+
if (typeof fp.cpuCores !== "number" || !Number.isFinite(fp.cpuCores)) bad.push("cpuCores");
|
|
46852
|
+
if (fp.cpuModel !== null && typeof fp.cpuModel !== "string") bad.push("cpuModel");
|
|
46853
|
+
return bad;
|
|
46854
|
+
}
|
|
46855
|
+
function checkCodingGraphRegression(report, baseline, tolerancePercent = DEFAULT_TOLERANCE_PERCENT) {
|
|
46856
|
+
const reportFixture = report.fixture.config;
|
|
46857
|
+
const baselineFixture = baseline.fixtureConfig;
|
|
46858
|
+
const mismatchedKeys = Object.keys(baselineFixture).filter((key) => reportFixture[key] !== baselineFixture[key]);
|
|
46859
|
+
if (mismatchedKeys.length > 0) {
|
|
46860
|
+
const diffs = mismatchedKeys.map(
|
|
46861
|
+
(key) => `${key}: report=${reportFixture[key]} baseline=${baselineFixture[key]}`
|
|
46862
|
+
).join(", ");
|
|
46863
|
+
return {
|
|
46864
|
+
passed: false,
|
|
46865
|
+
regressions: [],
|
|
46866
|
+
summary: `Fixture mismatch (${diffs}). Metrics are not comparable across different fixtures.`
|
|
46401
46867
|
};
|
|
46868
|
+
}
|
|
46869
|
+
if (report.schemaVersion !== baseline.schemaVersion) {
|
|
46402
46870
|
return {
|
|
46403
|
-
|
|
46404
|
-
|
|
46405
|
-
|
|
46871
|
+
passed: false,
|
|
46872
|
+
regressions: [],
|
|
46873
|
+
summary: `Schema-version mismatch: report=${report.schemaVersion} baseline=${baseline.schemaVersion}. Metrics are not comparable across schema versions \u2014 regenerate the baseline (#1688).`
|
|
46406
46874
|
};
|
|
46407
46875
|
}
|
|
46408
|
-
|
|
46409
|
-
|
|
46876
|
+
const metricChecks = [
|
|
46877
|
+
["incrementalUpdate.p50", report.incrementalUpdate?.p50],
|
|
46878
|
+
["incrementalUpdate.p95", report.incrementalUpdate?.p95],
|
|
46879
|
+
["incrementalModifiedUpdate.p50", report.incrementalModifiedUpdate?.p50],
|
|
46880
|
+
["incrementalModifiedUpdate.p95", report.incrementalModifiedUpdate?.p95],
|
|
46881
|
+
["tracePath.p95", report.tracePath?.p95],
|
|
46882
|
+
["searchGraph.p95", report.searchGraph?.p95],
|
|
46883
|
+
["fullIndexMs.ms", report.fullIndexMs?.ms],
|
|
46884
|
+
["fullIndexLocsPerSecond", report.fullIndexLocsPerSecond],
|
|
46885
|
+
["deadCodeMs.ms", report.deadCodeMs?.ms],
|
|
46886
|
+
["dbBytesPerKloc", report.dbBytesPerKloc]
|
|
46887
|
+
];
|
|
46888
|
+
const missingFields = metricChecks.filter(([, v]) => typeof v !== "number" || !Number.isFinite(v)).map(([k]) => k);
|
|
46889
|
+
if (missingFields.length > 0) {
|
|
46410
46890
|
return {
|
|
46411
|
-
|
|
46412
|
-
|
|
46413
|
-
|
|
46891
|
+
passed: false,
|
|
46892
|
+
regressions: [],
|
|
46893
|
+
summary: "Report claims schemaVersion " + report.schemaVersion + " but is missing required metric field(s): " + missingFields.join(", ") + " \u2014 the report is incomplete or corrupt. Regenerate it (#1688)."
|
|
46414
46894
|
};
|
|
46415
46895
|
}
|
|
46416
|
-
|
|
46417
|
-
|
|
46896
|
+
const badBaselineMetrics = Object.keys(METRIC_DIRECTION).filter((key) => {
|
|
46897
|
+
const v = baseline.metrics[key];
|
|
46898
|
+
return v != null && (typeof v !== "number" || !Number.isFinite(v));
|
|
46899
|
+
});
|
|
46900
|
+
if (badBaselineMetrics.length > 0) {
|
|
46418
46901
|
return {
|
|
46419
|
-
|
|
46420
|
-
|
|
46421
|
-
|
|
46422
|
-
retrievalStage: retrievalStageMiss ?? "unknown",
|
|
46423
|
-
reason: stages.retrieval.detail
|
|
46424
|
-
},
|
|
46425
|
-
stages
|
|
46902
|
+
passed: false,
|
|
46903
|
+
regressions: [],
|
|
46904
|
+
summary: "Baseline has a non-numeric required metric field(s): " + badBaselineMetrics.join(", ") + " \u2014 the baseline is corrupt. Regenerate it (#1688)."
|
|
46426
46905
|
};
|
|
46427
46906
|
}
|
|
46428
|
-
const
|
|
46429
|
-
|
|
46430
|
-
|
|
46431
|
-
|
|
46432
|
-
|
|
46433
|
-
|
|
46434
|
-
|
|
46435
|
-
|
|
46436
|
-
|
|
46437
|
-
|
|
46438
|
-
|
|
46439
|
-
return null;
|
|
46907
|
+
const reportFpBad = report.machine == null ? ["<missing>"] : invalidFingerprintFields(report.machine);
|
|
46908
|
+
const baselineFpBad = baseline.machine == null ? ["<missing>"] : invalidFingerprintFields(baseline.machine);
|
|
46909
|
+
if (reportFpBad.length > 0 || baselineFpBad.length > 0) {
|
|
46910
|
+
const which = [];
|
|
46911
|
+
if (reportFpBad.length > 0) which.push("report (" + reportFpBad.join(", ") + ")");
|
|
46912
|
+
if (baselineFpBad.length > 0) which.push("baseline (" + baselineFpBad.join(", ") + ")");
|
|
46913
|
+
return {
|
|
46914
|
+
passed: false,
|
|
46915
|
+
regressions: [],
|
|
46916
|
+
summary: "Report or baseline machine fingerprint is missing or has an invalid field type(s): " + which.join("; ") + " \u2014 the artifact is incomplete or corrupt. Regenerate it (#1688)."
|
|
46917
|
+
};
|
|
46440
46918
|
}
|
|
46441
|
-
const
|
|
46442
|
-
|
|
46443
|
-
|
|
46444
|
-
|
|
46445
|
-
|
|
46446
|
-
|
|
46919
|
+
const mismatch = compareMachineFingerprints(report.machine, baseline.machine);
|
|
46920
|
+
if (mismatch.differingFields.length > 0) {
|
|
46921
|
+
return {
|
|
46922
|
+
passed: true,
|
|
46923
|
+
skipped: true,
|
|
46924
|
+
regressions: [],
|
|
46925
|
+
summary: "Machine-fingerprint mismatch \u2014 comparison skipped to avoid a false-positive hardware-variance failure. Differing fields: " + mismatch.differingFields.join(", ") + ". Regenerate the baseline on this machine for a real comparison (#1688).",
|
|
46926
|
+
machineMismatch: {
|
|
46927
|
+
report: report.machine,
|
|
46928
|
+
baseline: baseline.machine,
|
|
46929
|
+
differingFields: mismatch.differingFields
|
|
46930
|
+
}
|
|
46931
|
+
};
|
|
46447
46932
|
}
|
|
46448
|
-
const
|
|
46449
|
-
|
|
46450
|
-
|
|
46451
|
-
|
|
46452
|
-
|
|
46453
|
-
|
|
46454
|
-
|
|
46455
|
-
|
|
46456
|
-
|
|
46457
|
-
|
|
46458
|
-
|
|
46459
|
-
|
|
46460
|
-
|
|
46461
|
-
|
|
46462
|
-
|
|
46463
|
-
|
|
46464
|
-
|
|
46465
|
-
|
|
46466
|
-
|
|
46467
|
-
|
|
46468
|
-
|
|
46469
|
-
rank: 0,
|
|
46470
|
-
unknown: 0
|
|
46471
|
-
};
|
|
46472
|
-
const items = [];
|
|
46473
|
-
const skippedTasks = [];
|
|
46474
|
-
for (const task of result.results.tasks) {
|
|
46475
|
-
if (task.details?.benchmarkFailure && typeof task.details.benchmarkFailure === "object") {
|
|
46476
|
-
skippedTasks.push({
|
|
46477
|
-
taskId: task.taskId,
|
|
46478
|
-
reason: "trial execution failure (not an answer failure)"
|
|
46479
|
-
});
|
|
46480
|
-
continue;
|
|
46481
|
-
}
|
|
46482
|
-
if (!task.goldMemories || task.goldMemories.length === 0) {
|
|
46483
|
-
skippedTasks.push({
|
|
46484
|
-
taskId: task.taskId,
|
|
46485
|
-
reason: "No goldMemories specified"
|
|
46486
|
-
});
|
|
46487
|
-
continue;
|
|
46488
|
-
}
|
|
46489
|
-
if (!isTaskFailed(task)) {
|
|
46490
|
-
skippedTasks.push({
|
|
46491
|
-
taskId: task.taskId,
|
|
46492
|
-
reason: "Task passed (score >= 1)"
|
|
46933
|
+
const measured = extractMetrics(report);
|
|
46934
|
+
const baselineMetrics = baseline.metrics;
|
|
46935
|
+
const regressions = [];
|
|
46936
|
+
for (const key of Object.keys(METRIC_DIRECTION)) {
|
|
46937
|
+
const baseVal = baselineMetrics[key];
|
|
46938
|
+
const measVal = measured[key];
|
|
46939
|
+
if (baseVal == null || measVal == null) continue;
|
|
46940
|
+
if (baseVal === 0) continue;
|
|
46941
|
+
const direction = METRIC_DIRECTION[key];
|
|
46942
|
+
const ratio2 = measVal / baseVal;
|
|
46943
|
+
const percentChange2 = direction === "lower-is-better" ? (ratio2 - 1) * 100 : (1 - ratio2) * 100;
|
|
46944
|
+
const regressed = percentChange2 > tolerancePercent;
|
|
46945
|
+
if (regressed) {
|
|
46946
|
+
regressions.push({
|
|
46947
|
+
key,
|
|
46948
|
+
baseline: baseVal,
|
|
46949
|
+
measured: measVal,
|
|
46950
|
+
percentChange: Math.round(percentChange2 * 10) / 10,
|
|
46951
|
+
direction,
|
|
46952
|
+
tolerancePercent,
|
|
46953
|
+
regressed: true
|
|
46493
46954
|
});
|
|
46494
|
-
continue;
|
|
46495
|
-
}
|
|
46496
|
-
const taskAttr = await attributeTask(task, memoizedEnv, options);
|
|
46497
|
-
if (taskAttr) {
|
|
46498
|
-
items.push(taskAttr);
|
|
46499
|
-
}
|
|
46500
|
-
}
|
|
46501
|
-
items.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
|
|
46502
|
-
skippedTasks.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
|
|
46503
|
-
for (const item of items) {
|
|
46504
|
-
totals[item.overall.class]++;
|
|
46505
|
-
if (item.overall.class === "retrieval_miss") {
|
|
46506
|
-
const stage = item.overall.retrievalStage ?? "unknown";
|
|
46507
|
-
retrievalStages[stage]++;
|
|
46508
46955
|
}
|
|
46509
46956
|
}
|
|
46957
|
+
const passed = regressions.length === 0;
|
|
46958
|
+
const summary = passed ? "All metrics within tolerance." : `${regressions.length} metric(s) regressed beyond ${tolerancePercent}% tolerance:
|
|
46959
|
+
` + regressions.map(
|
|
46960
|
+
(r) => ` ${r.key}: ${r.baseline} \u2192 ${r.measured} (${r.percentChange > 0 ? "+" : ""}${r.percentChange}% vs baseline)`
|
|
46961
|
+
).join("\n");
|
|
46962
|
+
return { passed, regressions, summary };
|
|
46963
|
+
}
|
|
46964
|
+
function buildBaselineFromReport(report, note) {
|
|
46510
46965
|
return {
|
|
46511
|
-
|
|
46512
|
-
|
|
46513
|
-
|
|
46514
|
-
|
|
46515
|
-
|
|
46516
|
-
|
|
46966
|
+
schemaVersion: report.schemaVersion,
|
|
46967
|
+
machine: report.machine,
|
|
46968
|
+
fixtureConfig: report.fixture.config,
|
|
46969
|
+
metrics: extractMetrics(report),
|
|
46970
|
+
createdAt: report.timestamp,
|
|
46971
|
+
note
|
|
46517
46972
|
};
|
|
46518
46973
|
}
|
|
46519
|
-
function renderAttributionReportTable(report) {
|
|
46520
|
-
const lines = [];
|
|
46521
|
-
lines.push(`Attribution Report (Run: ${report.runId})`);
|
|
46522
|
-
lines.push(`Failed-task predicate: minimum primary answer score < 1 (scores.overall or non-diagnostic scores)`);
|
|
46523
|
-
lines.push(`Attributed tasks: ${report.attributedTasks}, Skipped tasks: ${report.skippedTasks.length}`);
|
|
46524
|
-
lines.push("");
|
|
46525
|
-
lines.push("Totals by Class:");
|
|
46526
|
-
lines.push(` extraction_miss: ${report.totals.extraction_miss}`);
|
|
46527
|
-
lines.push(` index_miss: ${report.totals.index_miss}`);
|
|
46528
|
-
lines.push(` retrieval_miss: ${report.totals.retrieval_miss}`);
|
|
46529
|
-
lines.push(` use_miss: ${report.totals.use_miss}`);
|
|
46530
|
-
lines.push(` unattributed: ${report.totals.unattributed}`);
|
|
46531
|
-
lines.push("");
|
|
46532
|
-
lines.push("Retrieval Miss Stages:");
|
|
46533
|
-
lines.push(` filter: ${report.retrievalStages.filter}`);
|
|
46534
|
-
lines.push(` cap: ${report.retrievalStages.cap}`);
|
|
46535
|
-
lines.push(` rank: ${report.retrievalStages.rank}`);
|
|
46536
|
-
lines.push(` unknown: ${report.retrievalStages.unknown}`);
|
|
46537
|
-
lines.push("");
|
|
46538
|
-
lines.push("Task Attributions:");
|
|
46539
|
-
if (report.items.length === 0) {
|
|
46540
|
-
lines.push(" (none)");
|
|
46541
|
-
} else {
|
|
46542
|
-
for (const item of report.items) {
|
|
46543
|
-
const stageStr = item.overall.retrievalStage ? ` (${item.overall.retrievalStage})` : "";
|
|
46544
|
-
const labelStr = `${item.overall.class}${stageStr}`;
|
|
46545
|
-
const reasonStr = item.overall.reason ? ` - ${item.overall.reason}` : "";
|
|
46546
|
-
lines.push(` ${item.taskId.padEnd(20)} ${labelStr.padEnd(24)}${reasonStr}`);
|
|
46547
|
-
}
|
|
46548
|
-
}
|
|
46549
|
-
if (report.skippedTasks.length > 0) {
|
|
46550
|
-
lines.push("");
|
|
46551
|
-
lines.push("Skipped Tasks:");
|
|
46552
|
-
for (const skipped of report.skippedTasks) {
|
|
46553
|
-
lines.push(` ${skipped.taskId.padEnd(20)} ${skipped.reason}`);
|
|
46554
|
-
}
|
|
46555
|
-
}
|
|
46556
|
-
return `${lines.join("\n")}
|
|
46557
|
-
`;
|
|
46558
|
-
}
|
|
46559
|
-
function serializeAttributionReport(report) {
|
|
46560
|
-
return `${JSON.stringify(report, null, 2)}
|
|
46561
|
-
`;
|
|
46562
|
-
}
|
|
46563
46974
|
|
|
46564
46975
|
// src/attribute-cli.ts
|
|
46976
|
+
import { QmdClient } from "@remnic/core";
|
|
46565
46977
|
import { lstat as lstat6, readdir as readdir8, readFile as readFile25 } from "fs/promises";
|
|
46566
46978
|
import path40 from "path";
|
|
46567
46979
|
function parseFrontmatter2(fileContent) {
|
|
@@ -46658,6 +47070,53 @@ async function scanMemoryDir(dirPath) {
|
|
|
46658
47070
|
}
|
|
46659
47071
|
return memories;
|
|
46660
47072
|
}
|
|
47073
|
+
async function resolveQmdMemory(memoryDir, collection, resultPath) {
|
|
47074
|
+
const root = path40.resolve(memoryDir);
|
|
47075
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
47076
|
+
const addCandidate = (candidate) => {
|
|
47077
|
+
const resolved = path40.resolve(candidate);
|
|
47078
|
+
const relative = path40.relative(root, resolved);
|
|
47079
|
+
if (relative !== ".." && !relative.startsWith(`..${path40.sep}`) && !path40.isAbsolute(relative)) {
|
|
47080
|
+
candidates.add(resolved);
|
|
47081
|
+
}
|
|
47082
|
+
};
|
|
47083
|
+
const addRelative = (relativePath) => {
|
|
47084
|
+
const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
47085
|
+
if (!normalized) return;
|
|
47086
|
+
addCandidate(path40.join(root, normalized));
|
|
47087
|
+
if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
|
|
47088
|
+
addCandidate(path40.join(root, "facts", normalized));
|
|
47089
|
+
}
|
|
47090
|
+
};
|
|
47091
|
+
if (path40.isAbsolute(resultPath)) {
|
|
47092
|
+
addCandidate(resultPath);
|
|
47093
|
+
} else {
|
|
47094
|
+
addRelative(resultPath);
|
|
47095
|
+
const normalized = resultPath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
47096
|
+
if (normalized.startsWith(`${collection}/`)) {
|
|
47097
|
+
addRelative(normalized.slice(collection.length + 1));
|
|
47098
|
+
}
|
|
47099
|
+
}
|
|
47100
|
+
let resolvedMemory = null;
|
|
47101
|
+
for (const candidate of candidates) {
|
|
47102
|
+
try {
|
|
47103
|
+
const stats = await lstat6(candidate);
|
|
47104
|
+
if (!stats.isFile() || stats.isSymbolicLink()) continue;
|
|
47105
|
+
const parsed = parseFrontmatter2(await readFile25(candidate, "utf8"));
|
|
47106
|
+
if (!parsed.id || parsed.id.trim().length === 0) {
|
|
47107
|
+
throw new Error("QMD result has no canonical frontmatter id");
|
|
47108
|
+
}
|
|
47109
|
+
const memory = { id: parsed.id, content: parsed.body.trim() };
|
|
47110
|
+
if (resolvedMemory && resolvedMemory.id !== memory.id) {
|
|
47111
|
+
throw new Error("QMD result path resolves to more than one canonical memory id");
|
|
47112
|
+
}
|
|
47113
|
+
resolvedMemory = memory;
|
|
47114
|
+
} catch (error) {
|
|
47115
|
+
if (error instanceof Error && error.message.startsWith("QMD result")) throw error;
|
|
47116
|
+
}
|
|
47117
|
+
}
|
|
47118
|
+
return resolvedMemory;
|
|
47119
|
+
}
|
|
46661
47120
|
async function runAttributeCliCommand(options) {
|
|
46662
47121
|
const summary = await resolveBenchmarkResultReference(options.resultsDir, options.runRef);
|
|
46663
47122
|
if (!summary) {
|
|
@@ -46677,6 +47136,12 @@ async function runAttributeCliCommand(options) {
|
|
|
46677
47136
|
`
|
|
46678
47137
|
};
|
|
46679
47138
|
}
|
|
47139
|
+
if (Boolean(options.qmdPath) !== Boolean(options.collection)) {
|
|
47140
|
+
return {
|
|
47141
|
+
exitCode: 1,
|
|
47142
|
+
output: "Error: --qmd <path> and --collection <name> must be provided together.\n"
|
|
47143
|
+
};
|
|
47144
|
+
}
|
|
46680
47145
|
let memorySnapshot;
|
|
46681
47146
|
const listMemoriesFn = async () => {
|
|
46682
47147
|
if (memorySnapshot) return memorySnapshot;
|
|
@@ -46697,14 +47162,64 @@ async function runAttributeCliCommand(options) {
|
|
|
46697
47162
|
}
|
|
46698
47163
|
const recallLimitRaw = result.config?.remnicConfig?.recallLimit;
|
|
46699
47164
|
const recallLimit = typeof recallLimitRaw === "number" && recallLimitRaw > 0 ? recallLimitRaw : 10;
|
|
46700
|
-
const
|
|
47165
|
+
const needsLegacyFallback = result.results.tasks.some(
|
|
47166
|
+
(task) => task.attributionWitness === void 0 && isTaskFailed(task) && Array.isArray(task.goldMemories) && task.goldMemories.length > 0 && !(task.details?.benchmarkFailure && typeof task.details.benchmarkFailure === "object")
|
|
47167
|
+
);
|
|
47168
|
+
if (needsLegacyFallback && options.qmdPath && !options.memoryDir) {
|
|
47169
|
+
return {
|
|
47170
|
+
exitCode: 1,
|
|
47171
|
+
output: "Error: explicit legacy QMD fallback requires --memory-dir <path>.\n"
|
|
47172
|
+
};
|
|
47173
|
+
}
|
|
46701
47174
|
const env = {
|
|
46702
47175
|
listMemories: listMemoriesFn,
|
|
46703
|
-
oracleSearch: async (query, limit) => (await rankMemories2(query, limit)).map(({ id }) => ({ id })),
|
|
46704
|
-
recall: rankMemories2,
|
|
46705
47176
|
recallLimit
|
|
46706
47177
|
};
|
|
46707
|
-
|
|
47178
|
+
let qmdClient;
|
|
47179
|
+
if (needsLegacyFallback && options.qmdPath && options.collection && options.memoryDir) {
|
|
47180
|
+
qmdClient = new QmdClient(options.collection, recallLimit, {
|
|
47181
|
+
qmdPath: options.qmdPath,
|
|
47182
|
+
qmdStrictPath: true
|
|
47183
|
+
});
|
|
47184
|
+
const qmdAvailable = await qmdClient.probe().catch(() => false);
|
|
47185
|
+
const search = async (query, limit) => {
|
|
47186
|
+
if (!qmdAvailable) {
|
|
47187
|
+
throw new Error("QMD unavailable");
|
|
47188
|
+
}
|
|
47189
|
+
const degradations = [];
|
|
47190
|
+
const results = await qmdClient.search(
|
|
47191
|
+
query,
|
|
47192
|
+
options.collection,
|
|
47193
|
+
limit,
|
|
47194
|
+
void 0,
|
|
47195
|
+
{ onDegradation: (degradation) => degradations.push(degradation) }
|
|
47196
|
+
);
|
|
47197
|
+
if (degradations.length > 0) {
|
|
47198
|
+
throw new Error("QMD search degraded");
|
|
47199
|
+
}
|
|
47200
|
+
const memories = [];
|
|
47201
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
47202
|
+
for (const resultItem of results) {
|
|
47203
|
+
const memory = await resolveQmdMemory(options.memoryDir, options.collection, resultItem.path);
|
|
47204
|
+
if (!memory) {
|
|
47205
|
+
throw new Error("QMD result canonical identity unavailable");
|
|
47206
|
+
}
|
|
47207
|
+
if (!seenIds.has(memory.id)) {
|
|
47208
|
+
seenIds.add(memory.id);
|
|
47209
|
+
memories.push(memory);
|
|
47210
|
+
}
|
|
47211
|
+
}
|
|
47212
|
+
return memories;
|
|
47213
|
+
};
|
|
47214
|
+
env.oracleSearch = async (query, limit) => (await search(query, limit)).map(({ id }) => ({ id }));
|
|
47215
|
+
env.recall = search;
|
|
47216
|
+
}
|
|
47217
|
+
let report;
|
|
47218
|
+
try {
|
|
47219
|
+
report = await attributeRun(result, env, { threshold: options.threshold });
|
|
47220
|
+
} finally {
|
|
47221
|
+
await qmdClient?.dispose();
|
|
47222
|
+
}
|
|
46708
47223
|
const output = options.json ? serializeAttributionReport(report) : renderAttributionReportTable(report);
|
|
46709
47224
|
return {
|
|
46710
47225
|
exitCode: 0,
|