@remnic/bench 9.35.4 → 9.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +333 -2
- package/dist/index.js +2462 -114
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -2009,6 +2009,10 @@ function isTaskResultLike(value) {
|
|
|
2009
2009
|
return false;
|
|
2010
2010
|
}
|
|
2011
2011
|
const tokens = value.tokens;
|
|
2012
|
+
const goldMemories = value.goldMemories;
|
|
2013
|
+
if (goldMemories !== void 0 && (!Array.isArray(goldMemories) || !goldMemories.every((item) => typeof item === "string"))) {
|
|
2014
|
+
return false;
|
|
2015
|
+
}
|
|
2012
2016
|
return typeof value.taskId === "string" && typeof value.question === "string" && typeof value.expected === "string" && typeof value.actual === "string" && isObjectRecord(value.scores) && Object.values(value.scores).every(isFiniteNumber) && isFiniteNumber(value.latencyMs) && isObjectRecord(tokens) && isFiniteNumber(tokens.input) && isFiniteNumber(tokens.output);
|
|
2013
2017
|
}
|
|
2014
2018
|
function isStoredBenchmarkBaseline(value) {
|
|
@@ -21717,6 +21721,7 @@ async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate,
|
|
|
21717
21721
|
scores: buildFailureScores(ctx.metricsSpec.metrics),
|
|
21718
21722
|
latencyMs: 0,
|
|
21719
21723
|
tokens: { input: 0, output: 0 },
|
|
21724
|
+
...trial.goldMemories ? { goldMemories: trial.goldMemories } : {},
|
|
21720
21725
|
details: {
|
|
21721
21726
|
// Preserve the trial's category so a failed trial is still attributed
|
|
21722
21727
|
// to its per-category bucket (computeCategoryAggregates), keeping the
|
|
@@ -22001,6 +22006,7 @@ async function executeTrial(ctx, trial, answerSupportGate, pendingPairedAnswerRe
|
|
|
22001
22006
|
input: answered.tokens.input + judgeResult.tokens.input,
|
|
22002
22007
|
output: answered.tokens.output + judgeResult.tokens.output
|
|
22003
22008
|
},
|
|
22009
|
+
...trial.goldMemories ? { goldMemories: trial.goldMemories } : {},
|
|
22004
22010
|
details
|
|
22005
22011
|
};
|
|
22006
22012
|
if (answerReplayKey && currentProfile === "baseline" && answered.fallbackReason === void 0) {
|
|
@@ -22518,6 +22524,171 @@ async function loadDataset5(mode, datasetDir, limit) {
|
|
|
22518
22524
|
return loaded;
|
|
22519
22525
|
}
|
|
22520
22526
|
|
|
22527
|
+
// src/benchmarks/published/locomo/gold-memories.ts
|
|
22528
|
+
function deriveLoCoMoGoldMemories(observation, evidence) {
|
|
22529
|
+
if (!observation || typeof observation !== "object" || Array.isArray(observation) || !evidence || !Array.isArray(evidence) || evidence.length === 0) {
|
|
22530
|
+
return void 0;
|
|
22531
|
+
}
|
|
22532
|
+
const diaMap = /* @__PURE__ */ new Map();
|
|
22533
|
+
const sessRecord = observation;
|
|
22534
|
+
for (const sessionVal of Object.values(sessRecord)) {
|
|
22535
|
+
if (!sessionVal || typeof sessionVal !== "object" || Array.isArray(sessionVal)) {
|
|
22536
|
+
continue;
|
|
22537
|
+
}
|
|
22538
|
+
const speakerRecord = sessionVal;
|
|
22539
|
+
for (const speakerVal of Object.values(speakerRecord)) {
|
|
22540
|
+
if (!Array.isArray(speakerVal)) {
|
|
22541
|
+
continue;
|
|
22542
|
+
}
|
|
22543
|
+
for (const item of speakerVal) {
|
|
22544
|
+
if (!Array.isArray(item) || item.length < 2) {
|
|
22545
|
+
continue;
|
|
22546
|
+
}
|
|
22547
|
+
const stmt = item[0];
|
|
22548
|
+
const diaIds = item[1];
|
|
22549
|
+
if (typeof stmt !== "string" || stmt.trim().length === 0) {
|
|
22550
|
+
continue;
|
|
22551
|
+
}
|
|
22552
|
+
const targetDias = [];
|
|
22553
|
+
if (typeof diaIds === "string") {
|
|
22554
|
+
targetDias.push(diaIds);
|
|
22555
|
+
} else if (Array.isArray(diaIds)) {
|
|
22556
|
+
for (const d of diaIds) {
|
|
22557
|
+
if (typeof d === "string") {
|
|
22558
|
+
targetDias.push(d);
|
|
22559
|
+
}
|
|
22560
|
+
}
|
|
22561
|
+
}
|
|
22562
|
+
for (const d of targetDias) {
|
|
22563
|
+
let list = diaMap.get(d);
|
|
22564
|
+
if (!list) {
|
|
22565
|
+
list = [];
|
|
22566
|
+
diaMap.set(d, list);
|
|
22567
|
+
}
|
|
22568
|
+
list.push(stmt);
|
|
22569
|
+
}
|
|
22570
|
+
}
|
|
22571
|
+
}
|
|
22572
|
+
}
|
|
22573
|
+
if (diaMap.size === 0) {
|
|
22574
|
+
return void 0;
|
|
22575
|
+
}
|
|
22576
|
+
const result = [];
|
|
22577
|
+
const seen = /* @__PURE__ */ new Set();
|
|
22578
|
+
for (const ev of evidence) {
|
|
22579
|
+
if (typeof ev !== "string") {
|
|
22580
|
+
continue;
|
|
22581
|
+
}
|
|
22582
|
+
const stmts = diaMap.get(ev);
|
|
22583
|
+
if (!stmts) {
|
|
22584
|
+
continue;
|
|
22585
|
+
}
|
|
22586
|
+
for (const stmt of stmts) {
|
|
22587
|
+
if (!seen.has(stmt)) {
|
|
22588
|
+
seen.add(stmt);
|
|
22589
|
+
result.push(stmt);
|
|
22590
|
+
}
|
|
22591
|
+
}
|
|
22592
|
+
}
|
|
22593
|
+
return result.length > 0 ? result : void 0;
|
|
22594
|
+
}
|
|
22595
|
+
|
|
22596
|
+
// src/benchmarks/published/locomo/strict-parse.ts
|
|
22597
|
+
function parseDataset2(raw, filename) {
|
|
22598
|
+
const parsed = JSON.parse(raw);
|
|
22599
|
+
if (!Array.isArray(parsed)) {
|
|
22600
|
+
throw new Error(
|
|
22601
|
+
`LoCoMo dataset file ${filename} must contain an array of conversations.`
|
|
22602
|
+
);
|
|
22603
|
+
}
|
|
22604
|
+
return parsed.map((entry, index) => parseConversation(entry, filename, index));
|
|
22605
|
+
}
|
|
22606
|
+
function parseConversation(entry, filename, index) {
|
|
22607
|
+
const location = `LoCoMo dataset file ${filename} conversation ${index + 1}`;
|
|
22608
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
|
22609
|
+
throw new Error(`${location} must be an object.`);
|
|
22610
|
+
}
|
|
22611
|
+
const record = entry;
|
|
22612
|
+
if (typeof record.sample_id !== "string") {
|
|
22613
|
+
throw new Error(`${location} must include a string sample_id.`);
|
|
22614
|
+
}
|
|
22615
|
+
if (!record.conversation || typeof record.conversation !== "object" || Array.isArray(record.conversation)) {
|
|
22616
|
+
throw new Error(`${location} must include a conversation object.`);
|
|
22617
|
+
}
|
|
22618
|
+
const qa = normalizeQaArray(record.qa, location);
|
|
22619
|
+
const conversation = normalizeLoCoMoConversationSessions(
|
|
22620
|
+
record.conversation,
|
|
22621
|
+
location
|
|
22622
|
+
);
|
|
22623
|
+
return {
|
|
22624
|
+
sample_id: record.sample_id,
|
|
22625
|
+
conversation,
|
|
22626
|
+
qa,
|
|
22627
|
+
event_summary: record.event_summary,
|
|
22628
|
+
observation: record.observation,
|
|
22629
|
+
session_summary: record.session_summary
|
|
22630
|
+
};
|
|
22631
|
+
}
|
|
22632
|
+
function normalizeLoCoMoConversationSessions(conversation, location) {
|
|
22633
|
+
const normalized = { ...conversation };
|
|
22634
|
+
const sessionKeys = Object.keys(conversation).filter((key) => /^session_\d+$/.test(key)).sort(
|
|
22635
|
+
(a, b) => Number.parseInt(a.replace("session_", ""), 10) - Number.parseInt(b.replace("session_", ""), 10)
|
|
22636
|
+
);
|
|
22637
|
+
if (sessionKeys.length === 0) {
|
|
22638
|
+
throw new Error(`${location} conversation must include at least one session_N array.`);
|
|
22639
|
+
}
|
|
22640
|
+
for (const sessionKey of sessionKeys) {
|
|
22641
|
+
const session = conversation[sessionKey];
|
|
22642
|
+
if (!Array.isArray(session)) {
|
|
22643
|
+
throw new Error(`${location} conversation.${sessionKey} must be an array of turns.`);
|
|
22644
|
+
}
|
|
22645
|
+
normalized[sessionKey] = session.map(
|
|
22646
|
+
(turn, index) => normalizeLoCoMoTurn(turn, `${location} conversation.${sessionKey}[${index}]`)
|
|
22647
|
+
);
|
|
22648
|
+
}
|
|
22649
|
+
return normalized;
|
|
22650
|
+
}
|
|
22651
|
+
function normalizeLoCoMoTurn(turn, location) {
|
|
22652
|
+
if (!turn || typeof turn !== "object" || Array.isArray(turn)) {
|
|
22653
|
+
throw new Error(`${location} must be a turn object.`);
|
|
22654
|
+
}
|
|
22655
|
+
const record = turn;
|
|
22656
|
+
const speaker = requireNonEmptyString(record.speaker, `${location}.speaker`);
|
|
22657
|
+
const dia_id = requireNonEmptyString(record.dia_id, `${location}.dia_id`);
|
|
22658
|
+
const text = requireNonEmptyString(record.text, `${location}.text`);
|
|
22659
|
+
const normalized = { speaker, dia_id, text };
|
|
22660
|
+
if (record.query !== void 0) {
|
|
22661
|
+
normalized.query = requireString2(record.query, `${location}.query`);
|
|
22662
|
+
}
|
|
22663
|
+
if (record.blip_caption !== void 0) {
|
|
22664
|
+
normalized.blip_caption = requireString2(record.blip_caption, `${location}.blip_caption`);
|
|
22665
|
+
}
|
|
22666
|
+
return normalized;
|
|
22667
|
+
}
|
|
22668
|
+
function requireString2(value, location) {
|
|
22669
|
+
if (typeof value !== "string") {
|
|
22670
|
+
throw new Error(`${location} must be a string.`);
|
|
22671
|
+
}
|
|
22672
|
+
return value;
|
|
22673
|
+
}
|
|
22674
|
+
function requireNonEmptyString(value, location) {
|
|
22675
|
+
const text = requireString2(value, location);
|
|
22676
|
+
if (text.trim().length === 0) {
|
|
22677
|
+
throw new Error(`${location} must be a non-empty string.`);
|
|
22678
|
+
}
|
|
22679
|
+
return text;
|
|
22680
|
+
}
|
|
22681
|
+
function normalizeQaArray(value, location) {
|
|
22682
|
+
if (!Array.isArray(value)) {
|
|
22683
|
+
throw new Error(
|
|
22684
|
+
`${location} must include a qa array with question/answer/evidence/category fields.`
|
|
22685
|
+
);
|
|
22686
|
+
}
|
|
22687
|
+
return value.map(
|
|
22688
|
+
(entry, index) => normalizeLoCoMoQa(entry, `${location} qa[${index}]`)
|
|
22689
|
+
);
|
|
22690
|
+
}
|
|
22691
|
+
|
|
22521
22692
|
// src/benchmarks/published/locomo/task-selection.ts
|
|
22522
22693
|
var LOCOMO_TASK_SELECTION_VERSION = 1;
|
|
22523
22694
|
function parseLoCoMoTaskSelectionManifest(value, label = "LoCoMo task selection") {
|
|
@@ -23121,18 +23292,21 @@ function buildLoCoMoPlan(conversation, multiHopRecallComposition) {
|
|
|
23121
23292
|
qa,
|
|
23122
23293
|
questionIndex,
|
|
23123
23294
|
sessionIds,
|
|
23124
|
-
multiHopRecallComposition
|
|
23295
|
+
multiHopRecallComposition,
|
|
23296
|
+
conversation.observation
|
|
23125
23297
|
)
|
|
23126
23298
|
);
|
|
23127
23299
|
return { ingestSessions, trials };
|
|
23128
23300
|
}
|
|
23129
|
-
function buildTrial(conversationId, qa, questionIndex, sessionIds, multiHopRecallComposition) {
|
|
23301
|
+
function buildTrial(conversationId, qa, questionIndex, sessionIds, multiHopRecallComposition, observation) {
|
|
23130
23302
|
const categoryName = CATEGORY_NAMES[qa.category] ?? `category_${qa.category}`;
|
|
23303
|
+
const goldMemories = deriveLoCoMoGoldMemories(observation, qa.evidence);
|
|
23131
23304
|
return {
|
|
23132
23305
|
taskId: `${conversationId}-q${questionIndex}-${categoryName}`,
|
|
23133
23306
|
question: qa.question,
|
|
23134
23307
|
expected: qa.answer,
|
|
23135
23308
|
recallSessionIds: sessionIds,
|
|
23309
|
+
...goldMemories ? { goldMemories } : {},
|
|
23136
23310
|
answerFormat: "short-with-specifics",
|
|
23137
23311
|
recallTextTransform: ({ question, recalledText }) => transformLoCoMoRecallText({
|
|
23138
23312
|
question,
|
|
@@ -23796,100 +23970,6 @@ async function loadLoCoMoDataset(mode, datasetDir, limit) {
|
|
|
23796
23970
|
items: loaded.items
|
|
23797
23971
|
};
|
|
23798
23972
|
}
|
|
23799
|
-
function parseDataset2(raw, filename) {
|
|
23800
|
-
const parsed = JSON.parse(raw);
|
|
23801
|
-
if (!Array.isArray(parsed)) {
|
|
23802
|
-
throw new Error(
|
|
23803
|
-
`LoCoMo dataset file ${filename} must contain an array of conversations.`
|
|
23804
|
-
);
|
|
23805
|
-
}
|
|
23806
|
-
return parsed.map((entry, index) => parseConversation(entry, filename, index));
|
|
23807
|
-
}
|
|
23808
|
-
function parseConversation(entry, filename, index) {
|
|
23809
|
-
const location = `LoCoMo dataset file ${filename} conversation ${index + 1}`;
|
|
23810
|
-
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
|
23811
|
-
throw new Error(`${location} must be an object.`);
|
|
23812
|
-
}
|
|
23813
|
-
const record = entry;
|
|
23814
|
-
if (typeof record.sample_id !== "string") {
|
|
23815
|
-
throw new Error(`${location} must include a string sample_id.`);
|
|
23816
|
-
}
|
|
23817
|
-
if (!record.conversation || typeof record.conversation !== "object" || Array.isArray(record.conversation)) {
|
|
23818
|
-
throw new Error(`${location} must include a conversation object.`);
|
|
23819
|
-
}
|
|
23820
|
-
const qa = normalizeQaArray(record.qa, location);
|
|
23821
|
-
const conversation = normalizeLoCoMoConversationSessions(
|
|
23822
|
-
record.conversation,
|
|
23823
|
-
location
|
|
23824
|
-
);
|
|
23825
|
-
return {
|
|
23826
|
-
sample_id: record.sample_id,
|
|
23827
|
-
conversation,
|
|
23828
|
-
qa,
|
|
23829
|
-
event_summary: record.event_summary,
|
|
23830
|
-
observation: record.observation,
|
|
23831
|
-
session_summary: record.session_summary
|
|
23832
|
-
};
|
|
23833
|
-
}
|
|
23834
|
-
function normalizeLoCoMoConversationSessions(conversation, location) {
|
|
23835
|
-
const normalized = { ...conversation };
|
|
23836
|
-
const sessionKeys = Object.keys(conversation).filter((key) => /^session_\d+$/.test(key)).sort(
|
|
23837
|
-
(a, b) => Number.parseInt(a.replace("session_", ""), 10) - Number.parseInt(b.replace("session_", ""), 10)
|
|
23838
|
-
);
|
|
23839
|
-
if (sessionKeys.length === 0) {
|
|
23840
|
-
throw new Error(`${location} conversation must include at least one session_N array.`);
|
|
23841
|
-
}
|
|
23842
|
-
for (const sessionKey of sessionKeys) {
|
|
23843
|
-
const session = conversation[sessionKey];
|
|
23844
|
-
if (!Array.isArray(session)) {
|
|
23845
|
-
throw new Error(`${location} conversation.${sessionKey} must be an array of turns.`);
|
|
23846
|
-
}
|
|
23847
|
-
normalized[sessionKey] = session.map(
|
|
23848
|
-
(turn, index) => normalizeLoCoMoTurn(turn, `${location} conversation.${sessionKey}[${index}]`)
|
|
23849
|
-
);
|
|
23850
|
-
}
|
|
23851
|
-
return normalized;
|
|
23852
|
-
}
|
|
23853
|
-
function normalizeLoCoMoTurn(turn, location) {
|
|
23854
|
-
if (!turn || typeof turn !== "object" || Array.isArray(turn)) {
|
|
23855
|
-
throw new Error(`${location} must be a turn object.`);
|
|
23856
|
-
}
|
|
23857
|
-
const record = turn;
|
|
23858
|
-
const speaker = requireNonEmptyString(record.speaker, `${location}.speaker`);
|
|
23859
|
-
const dia_id = requireNonEmptyString(record.dia_id, `${location}.dia_id`);
|
|
23860
|
-
const text = requireNonEmptyString(record.text, `${location}.text`);
|
|
23861
|
-
const normalized = { speaker, dia_id, text };
|
|
23862
|
-
if (record.query !== void 0) {
|
|
23863
|
-
normalized.query = requireString2(record.query, `${location}.query`);
|
|
23864
|
-
}
|
|
23865
|
-
if (record.blip_caption !== void 0) {
|
|
23866
|
-
normalized.blip_caption = requireString2(record.blip_caption, `${location}.blip_caption`);
|
|
23867
|
-
}
|
|
23868
|
-
return normalized;
|
|
23869
|
-
}
|
|
23870
|
-
function requireString2(value, location) {
|
|
23871
|
-
if (typeof value !== "string") {
|
|
23872
|
-
throw new Error(`${location} must be a string.`);
|
|
23873
|
-
}
|
|
23874
|
-
return value;
|
|
23875
|
-
}
|
|
23876
|
-
function requireNonEmptyString(value, location) {
|
|
23877
|
-
const text = requireString2(value, location);
|
|
23878
|
-
if (text.trim().length === 0) {
|
|
23879
|
-
throw new Error(`${location} must be a non-empty string.`);
|
|
23880
|
-
}
|
|
23881
|
-
return text;
|
|
23882
|
-
}
|
|
23883
|
-
function normalizeQaArray(value, location) {
|
|
23884
|
-
if (!Array.isArray(value)) {
|
|
23885
|
-
throw new Error(
|
|
23886
|
-
`${location} must include a qa array with question/answer/evidence/category fields.`
|
|
23887
|
-
);
|
|
23888
|
-
}
|
|
23889
|
-
return value.map(
|
|
23890
|
-
(entry, index) => normalizeLoCoMoQa(entry, `${location} qa[${index}]`)
|
|
23891
|
-
);
|
|
23892
|
-
}
|
|
23893
23973
|
|
|
23894
23974
|
// src/benchmarks/published/beam/runner.ts
|
|
23895
23975
|
import { randomUUID as randomUUID6 } from "crypto";
|
|
@@ -40533,8 +40613,8 @@ var LOCOMO_CATEGORY_ORDER2 = ["single_hop", "multi_hop", "temporal", "open_domai
|
|
|
40533
40613
|
var LOCOMO_TASK_CATEGORY_PATTERN2 = /-(single_hop|multi_hop|temporal|open_domain|adversarial)$/;
|
|
40534
40614
|
var SOURCE_TURN_PATTERN = /^\[([^,\]\s]+),\s*turn\s+(\d+),\s*([^,\]]+?)(?:,\s*score\s+[^\]]+)?\]/i;
|
|
40535
40615
|
var SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
|
40536
|
-
function sanitizeLoComoResultReference(
|
|
40537
|
-
const reference = basename2(
|
|
40616
|
+
function sanitizeLoComoResultReference(path43) {
|
|
40617
|
+
const reference = basename2(path43).replace(/[\u0000-\u001f\u007f`]/g, "_");
|
|
40538
40618
|
if (!reference) throw new Error("Result path must identify a file.");
|
|
40539
40619
|
return reference;
|
|
40540
40620
|
}
|
|
@@ -42092,50 +42172,50 @@ function assertMemoryIdRef(value) {
|
|
|
42092
42172
|
throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
|
|
42093
42173
|
}
|
|
42094
42174
|
}
|
|
42095
|
-
function assertJsonConfig(value,
|
|
42175
|
+
function assertJsonConfig(value, path43 = "retrievalConfig") {
|
|
42096
42176
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
42097
42177
|
if (typeof value === "number") {
|
|
42098
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
42178
|
+
if (!Number.isFinite(value)) throw new Error(`${path43} must contain only finite JSON numbers.`);
|
|
42099
42179
|
return value;
|
|
42100
42180
|
}
|
|
42101
42181
|
if (Array.isArray(value)) {
|
|
42102
|
-
return value.map((entry, index) => assertJsonConfig(entry, `${
|
|
42182
|
+
return value.map((entry, index) => assertJsonConfig(entry, `${path43}[${index}]`));
|
|
42103
42183
|
}
|
|
42104
42184
|
if (!value || typeof value !== "object") {
|
|
42105
|
-
throw new Error(`${
|
|
42185
|
+
throw new Error(`${path43} must be JSON-serializable and provider-free.`);
|
|
42106
42186
|
}
|
|
42107
42187
|
const output = {};
|
|
42108
42188
|
for (const key of Object.keys(value).sort()) {
|
|
42109
42189
|
const child = value[key];
|
|
42110
42190
|
if (key === "openaiApiKey") {
|
|
42111
42191
|
if (child !== false) {
|
|
42112
|
-
throw new Error(`${
|
|
42192
|
+
throw new Error(`${path43}.${key} must be exactly false for provider-free capture.`);
|
|
42113
42193
|
}
|
|
42114
42194
|
output[key] = false;
|
|
42115
42195
|
continue;
|
|
42116
42196
|
}
|
|
42117
42197
|
if (isSecretKey(key)) {
|
|
42118
|
-
throw new Error(`${
|
|
42198
|
+
throw new Error(`${path43}.${key} contains secret-bearing configuration.`);
|
|
42119
42199
|
}
|
|
42120
42200
|
if (child === void 0) continue;
|
|
42121
42201
|
if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
|
|
42122
|
-
throw new Error(`${
|
|
42202
|
+
throw new Error(`${path43}.${key} is provider-capable configuration.`);
|
|
42123
42203
|
}
|
|
42124
|
-
output[key] = assertJsonConfig(child, `${
|
|
42204
|
+
output[key] = assertJsonConfig(child, `${path43}.${key}`);
|
|
42125
42205
|
}
|
|
42126
42206
|
return output;
|
|
42127
42207
|
}
|
|
42128
|
-
function sanitizeProviderFreeRetrievalConfig(value,
|
|
42208
|
+
function sanitizeProviderFreeRetrievalConfig(value, path43 = "retrievalConfig") {
|
|
42129
42209
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
42130
42210
|
if (typeof value === "number") {
|
|
42131
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
42211
|
+
if (!Number.isFinite(value)) throw new Error(`${path43} must contain only finite JSON numbers.`);
|
|
42132
42212
|
return value;
|
|
42133
42213
|
}
|
|
42134
42214
|
if (Array.isArray(value)) {
|
|
42135
|
-
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${
|
|
42215
|
+
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path43}[${index}]`));
|
|
42136
42216
|
}
|
|
42137
42217
|
if (!value || typeof value !== "object") {
|
|
42138
|
-
throw new Error(`${
|
|
42218
|
+
throw new Error(`${path43} must be JSON-serializable.`);
|
|
42139
42219
|
}
|
|
42140
42220
|
const output = {};
|
|
42141
42221
|
for (const key of Object.keys(value).sort()) {
|
|
@@ -42144,7 +42224,7 @@ function sanitizeProviderFreeRetrievalConfig(value, path40 = "retrievalConfig")
|
|
|
42144
42224
|
if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
|
|
42145
42225
|
continue;
|
|
42146
42226
|
}
|
|
42147
|
-
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${
|
|
42227
|
+
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path43}.${key}`);
|
|
42148
42228
|
}
|
|
42149
42229
|
return output;
|
|
42150
42230
|
}
|
|
@@ -42319,12 +42399,12 @@ function rotateDistractors(question, seed) {
|
|
|
42319
42399
|
pool.push(distractor);
|
|
42320
42400
|
}
|
|
42321
42401
|
}
|
|
42322
|
-
const
|
|
42323
|
-
const correctIndex =
|
|
42402
|
+
const shuffled2 = shuffleTasks(pool, seed);
|
|
42403
|
+
const correctIndex = shuffled2.indexOf(question.correct);
|
|
42324
42404
|
if (correctIndex === -1) {
|
|
42325
42405
|
throw new Error("Correct answer dropped from the distractor pool during rotation.");
|
|
42326
42406
|
}
|
|
42327
|
-
return { choices:
|
|
42407
|
+
return { choices: shuffled2, correctIndex };
|
|
42328
42408
|
}
|
|
42329
42409
|
function selectFixtureVariant(variants, seed) {
|
|
42330
42410
|
if (variants.length === 0) {
|
|
@@ -45976,6 +46056,2255 @@ function buildBaselineFromReport(report, note) {
|
|
|
45976
46056
|
note
|
|
45977
46057
|
};
|
|
45978
46058
|
}
|
|
46059
|
+
|
|
46060
|
+
// src/attribution.ts
|
|
46061
|
+
var DEFAULT_STOPWORDS = /* @__PURE__ */ new Set([
|
|
46062
|
+
"a",
|
|
46063
|
+
"an",
|
|
46064
|
+
"the",
|
|
46065
|
+
"in",
|
|
46066
|
+
"on",
|
|
46067
|
+
"at",
|
|
46068
|
+
"to",
|
|
46069
|
+
"for",
|
|
46070
|
+
"of",
|
|
46071
|
+
"with",
|
|
46072
|
+
"by",
|
|
46073
|
+
"from",
|
|
46074
|
+
"up",
|
|
46075
|
+
"about",
|
|
46076
|
+
"into",
|
|
46077
|
+
"through",
|
|
46078
|
+
"during",
|
|
46079
|
+
"before",
|
|
46080
|
+
"after",
|
|
46081
|
+
"above",
|
|
46082
|
+
"below",
|
|
46083
|
+
"and",
|
|
46084
|
+
"or",
|
|
46085
|
+
"but",
|
|
46086
|
+
"if",
|
|
46087
|
+
"then",
|
|
46088
|
+
"else",
|
|
46089
|
+
"when",
|
|
46090
|
+
"where",
|
|
46091
|
+
"why",
|
|
46092
|
+
"how",
|
|
46093
|
+
"all",
|
|
46094
|
+
"any",
|
|
46095
|
+
"both",
|
|
46096
|
+
"each",
|
|
46097
|
+
"few",
|
|
46098
|
+
"more",
|
|
46099
|
+
"most",
|
|
46100
|
+
"other",
|
|
46101
|
+
"some",
|
|
46102
|
+
"such",
|
|
46103
|
+
"no",
|
|
46104
|
+
"nor",
|
|
46105
|
+
"not",
|
|
46106
|
+
"only",
|
|
46107
|
+
"own",
|
|
46108
|
+
"same",
|
|
46109
|
+
"so",
|
|
46110
|
+
"than",
|
|
46111
|
+
"too",
|
|
46112
|
+
"very",
|
|
46113
|
+
"this",
|
|
46114
|
+
"that",
|
|
46115
|
+
"these",
|
|
46116
|
+
"those",
|
|
46117
|
+
"it",
|
|
46118
|
+
"its",
|
|
46119
|
+
"is",
|
|
46120
|
+
"are",
|
|
46121
|
+
"was",
|
|
46122
|
+
"were",
|
|
46123
|
+
"be",
|
|
46124
|
+
"been",
|
|
46125
|
+
"being",
|
|
46126
|
+
"have",
|
|
46127
|
+
"has",
|
|
46128
|
+
"had",
|
|
46129
|
+
"do",
|
|
46130
|
+
"does",
|
|
46131
|
+
"did"
|
|
46132
|
+
]);
|
|
46133
|
+
function extractContentWords(text) {
|
|
46134
|
+
const cleaned = text.toLowerCase().replace(/[^a-z0-9\s]/g, " ");
|
|
46135
|
+
const tokens = cleaned.split(/\s+/).filter(Boolean);
|
|
46136
|
+
return tokens.filter((t) => !DEFAULT_STOPWORDS.has(t));
|
|
46137
|
+
}
|
|
46138
|
+
function lexicalSimilarity(a, b) {
|
|
46139
|
+
const goldWords = extractContentWords(a);
|
|
46140
|
+
if (goldWords.length === 0) {
|
|
46141
|
+
return 0;
|
|
46142
|
+
}
|
|
46143
|
+
const candWords = new Set(extractContentWords(b));
|
|
46144
|
+
let matchCount = 0;
|
|
46145
|
+
for (const word of goldWords) {
|
|
46146
|
+
if (candWords.has(word)) {
|
|
46147
|
+
matchCount++;
|
|
46148
|
+
}
|
|
46149
|
+
}
|
|
46150
|
+
return matchCount / goldWords.length;
|
|
46151
|
+
}
|
|
46152
|
+
var CLASS_RANK = {
|
|
46153
|
+
extraction_miss: 1,
|
|
46154
|
+
index_miss: 2,
|
|
46155
|
+
retrieval_miss: 3,
|
|
46156
|
+
use_miss: 4,
|
|
46157
|
+
unattributed: 5
|
|
46158
|
+
};
|
|
46159
|
+
var RETRIEVAL_STAGE_RANK = {
|
|
46160
|
+
cap: 1,
|
|
46161
|
+
rank: 2,
|
|
46162
|
+
filter: 3,
|
|
46163
|
+
unknown: 4
|
|
46164
|
+
};
|
|
46165
|
+
function computeOverallLabel(golds) {
|
|
46166
|
+
if (golds.length === 0) {
|
|
46167
|
+
return { class: "unattributed", reason: "no gold memories" };
|
|
46168
|
+
}
|
|
46169
|
+
let bestGold = golds[0];
|
|
46170
|
+
for (let i = 1; i < golds.length; i++) {
|
|
46171
|
+
const current = golds[i];
|
|
46172
|
+
const bestRank = CLASS_RANK[bestGold.label.class];
|
|
46173
|
+
const currRank = CLASS_RANK[current.label.class];
|
|
46174
|
+
if (currRank < bestRank) {
|
|
46175
|
+
bestGold = current;
|
|
46176
|
+
} else if (currRank === bestRank && current.label.class === "retrieval_miss") {
|
|
46177
|
+
const bestRetRank = RETRIEVAL_STAGE_RANK[bestGold.label.retrievalStage ?? "unknown"];
|
|
46178
|
+
const currRetRank = RETRIEVAL_STAGE_RANK[current.label.retrievalStage ?? "unknown"];
|
|
46179
|
+
if (currRetRank < bestRetRank) {
|
|
46180
|
+
bestGold = current;
|
|
46181
|
+
}
|
|
46182
|
+
}
|
|
46183
|
+
}
|
|
46184
|
+
return { ...bestGold.label };
|
|
46185
|
+
}
|
|
46186
|
+
function isDiagnosticScore(name) {
|
|
46187
|
+
return name.endsWith("_agreement") || name.includes("_id_leak") || name === "search_hits";
|
|
46188
|
+
}
|
|
46189
|
+
function isTaskFailed(task) {
|
|
46190
|
+
if (!task.scores || Object.keys(task.scores).length === 0) {
|
|
46191
|
+
return true;
|
|
46192
|
+
}
|
|
46193
|
+
if ("overall" in task.scores && typeof task.scores.overall === "number") {
|
|
46194
|
+
return task.scores.overall < 1;
|
|
46195
|
+
}
|
|
46196
|
+
const primaryScores = Object.entries(task.scores).filter(([name, score]) => typeof score === "number" && !isDiagnosticScore(name)).map(([, score]) => score);
|
|
46197
|
+
return primaryScores.length === 0 || Math.min(...primaryScores) < 1;
|
|
46198
|
+
}
|
|
46199
|
+
function withMemoizedListMemories(env) {
|
|
46200
|
+
let cache = null;
|
|
46201
|
+
return {
|
|
46202
|
+
...env,
|
|
46203
|
+
listMemories() {
|
|
46204
|
+
if (!cache) {
|
|
46205
|
+
cache = env.listMemories();
|
|
46206
|
+
}
|
|
46207
|
+
return cache;
|
|
46208
|
+
}
|
|
46209
|
+
};
|
|
46210
|
+
}
|
|
46211
|
+
async function attributeGoldMemory(goldStatement, question, env, options = {}, recalledText) {
|
|
46212
|
+
if (options.threshold !== void 0) {
|
|
46213
|
+
if (typeof options.threshold !== "number" || !Number.isFinite(options.threshold) || options.threshold < 0 || options.threshold > 1) {
|
|
46214
|
+
throw new RangeError("attribution threshold must be a finite number between 0 and 1");
|
|
46215
|
+
}
|
|
46216
|
+
}
|
|
46217
|
+
const threshold = options.threshold ?? 0.6;
|
|
46218
|
+
const simFn = options.similarity ?? lexicalSimilarity;
|
|
46219
|
+
const goldInRecalledText = typeof recalledText === "string" && simFn(goldStatement, recalledText) >= threshold;
|
|
46220
|
+
const stages = {
|
|
46221
|
+
extraction: { status: "unavailable" },
|
|
46222
|
+
index: { status: "unavailable" },
|
|
46223
|
+
retrieval: { status: "unavailable" },
|
|
46224
|
+
use: { status: "unavailable" }
|
|
46225
|
+
};
|
|
46226
|
+
let memories = [];
|
|
46227
|
+
let extractionRan = false;
|
|
46228
|
+
let extractionErrorDetail;
|
|
46229
|
+
if (typeof env.listMemories === "function") {
|
|
46230
|
+
try {
|
|
46231
|
+
memories = await env.listMemories();
|
|
46232
|
+
extractionRan = true;
|
|
46233
|
+
} catch {
|
|
46234
|
+
extractionRan = false;
|
|
46235
|
+
extractionErrorDetail = "listMemories failed";
|
|
46236
|
+
}
|
|
46237
|
+
}
|
|
46238
|
+
let bestSim = -1;
|
|
46239
|
+
let matchedMem = null;
|
|
46240
|
+
if (extractionRan) {
|
|
46241
|
+
if (memories.length === 0) {
|
|
46242
|
+
if (goldInRecalledText) {
|
|
46243
|
+
const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
|
|
46244
|
+
stages.extraction = { status: "pass", detail: impliedDetail };
|
|
46245
|
+
stages.index = { status: "pass", detail: impliedDetail };
|
|
46246
|
+
stages.retrieval = { status: "pass", detail: impliedDetail };
|
|
46247
|
+
stages.use = {
|
|
46248
|
+
status: "fail",
|
|
46249
|
+
detail: "Gold memory present in context but answer was incorrect"
|
|
46250
|
+
};
|
|
46251
|
+
return {
|
|
46252
|
+
goldMemory: goldStatement,
|
|
46253
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
46254
|
+
stages
|
|
46255
|
+
};
|
|
46256
|
+
}
|
|
46257
|
+
const detail = "store contains no memories";
|
|
46258
|
+
stages.extraction = { status: "fail", detail };
|
|
46259
|
+
stages.index = { status: "unavailable", detail: "not reached" };
|
|
46260
|
+
stages.retrieval = { status: "unavailable", detail: "not reached" };
|
|
46261
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
46262
|
+
return {
|
|
46263
|
+
goldMemory: goldStatement,
|
|
46264
|
+
label: { class: "extraction_miss", reason: detail },
|
|
46265
|
+
stages
|
|
46266
|
+
};
|
|
46267
|
+
}
|
|
46268
|
+
for (const mem of memories) {
|
|
46269
|
+
const sim = simFn(goldStatement, mem.content);
|
|
46270
|
+
if (sim > bestSim) {
|
|
46271
|
+
bestSim = sim;
|
|
46272
|
+
matchedMem = mem;
|
|
46273
|
+
}
|
|
46274
|
+
}
|
|
46275
|
+
if (bestSim < threshold || !matchedMem) {
|
|
46276
|
+
if (goldInRecalledText) {
|
|
46277
|
+
const impliedDetail = "implied pass from recalled context (post-hoc store scan missed)";
|
|
46278
|
+
stages.extraction = { status: "pass", detail: impliedDetail };
|
|
46279
|
+
stages.index = { status: "pass", detail: impliedDetail };
|
|
46280
|
+
stages.retrieval = { status: "pass", detail: impliedDetail };
|
|
46281
|
+
stages.use = {
|
|
46282
|
+
status: "fail",
|
|
46283
|
+
detail: "Gold memory present in context but answer was incorrect"
|
|
46284
|
+
};
|
|
46285
|
+
return {
|
|
46286
|
+
goldMemory: goldStatement,
|
|
46287
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
46288
|
+
stages
|
|
46289
|
+
};
|
|
46290
|
+
}
|
|
46291
|
+
const detail = `Best similarity ${bestSim >= 0 ? bestSim.toFixed(3) : 0} below threshold ${threshold}`;
|
|
46292
|
+
stages.extraction = { status: "fail", detail };
|
|
46293
|
+
stages.index = { status: "unavailable", detail: "not reached" };
|
|
46294
|
+
stages.retrieval = { status: "unavailable", detail: "not reached" };
|
|
46295
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
46296
|
+
return {
|
|
46297
|
+
goldMemory: goldStatement,
|
|
46298
|
+
label: { class: "extraction_miss", reason: detail },
|
|
46299
|
+
stages
|
|
46300
|
+
};
|
|
46301
|
+
}
|
|
46302
|
+
stages.extraction = {
|
|
46303
|
+
status: "pass",
|
|
46304
|
+
detail: `Matched memory ${matchedMem.id} (sim ${bestSim.toFixed(3)})`
|
|
46305
|
+
};
|
|
46306
|
+
} else {
|
|
46307
|
+
stages.extraction = {
|
|
46308
|
+
status: "unavailable",
|
|
46309
|
+
detail: extractionErrorDetail ?? "listMemories unavailable"
|
|
46310
|
+
};
|
|
46311
|
+
}
|
|
46312
|
+
const matchedMemoryId = matchedMem ? matchedMem.id : void 0;
|
|
46313
|
+
const recallLimit = env.recallLimit;
|
|
46314
|
+
const replayLimit = env.replayLimit ?? Math.max(25, recallLimit * 5);
|
|
46315
|
+
let indexCheckPassed = false;
|
|
46316
|
+
let indexCheckFailed = false;
|
|
46317
|
+
if (typeof env.oracleSearch === "function" && extractionRan) {
|
|
46318
|
+
try {
|
|
46319
|
+
const oracleResults = await env.oracleSearch(goldStatement, replayLimit);
|
|
46320
|
+
const idMatched = matchedMemoryId ? oracleResults.some((r) => r.id === matchedMemoryId) : false;
|
|
46321
|
+
if (idMatched) {
|
|
46322
|
+
indexCheckPassed = true;
|
|
46323
|
+
} else {
|
|
46324
|
+
const memMap = new Map(memories.map((m) => [m.id, m]));
|
|
46325
|
+
indexCheckPassed = oracleResults.some((r) => {
|
|
46326
|
+
const mem = memMap.get(r.id);
|
|
46327
|
+
return mem ? simFn(goldStatement, mem.content) >= threshold : false;
|
|
46328
|
+
});
|
|
46329
|
+
}
|
|
46330
|
+
if (indexCheckPassed) {
|
|
46331
|
+
stages.index = { status: "pass", detail: "Found in oracle search" };
|
|
46332
|
+
} else {
|
|
46333
|
+
indexCheckFailed = true;
|
|
46334
|
+
stages.index = { status: "fail", detail: "Not found in oracle search" };
|
|
46335
|
+
}
|
|
46336
|
+
} catch {
|
|
46337
|
+
stages.index = { status: "unavailable", detail: "oracleSearch threw error" };
|
|
46338
|
+
}
|
|
46339
|
+
} else if (typeof env.oracleSearch === "function") {
|
|
46340
|
+
stages.index = {
|
|
46341
|
+
status: "unavailable",
|
|
46342
|
+
detail: "extraction check unavailable; oracle result would be ambiguous"
|
|
46343
|
+
};
|
|
46344
|
+
} else {
|
|
46345
|
+
stages.index = { status: "unavailable", detail: "index check unavailable" };
|
|
46346
|
+
}
|
|
46347
|
+
let retrievalCheckPassed = false;
|
|
46348
|
+
let retrievalStageMiss = void 0;
|
|
46349
|
+
if (typeof env.recall === "function") {
|
|
46350
|
+
try {
|
|
46351
|
+
const recallResults = await env.recall(question, recallLimit);
|
|
46352
|
+
const isGoldInRecall = recallResults.some(
|
|
46353
|
+
(m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
|
|
46354
|
+
);
|
|
46355
|
+
if (isGoldInRecall) {
|
|
46356
|
+
retrievalCheckPassed = true;
|
|
46357
|
+
stages.retrieval = { status: "pass", detail: `Recalled within recallLimit ${recallLimit}` };
|
|
46358
|
+
} else {
|
|
46359
|
+
const replayResults = await env.recall(question, replayLimit);
|
|
46360
|
+
const replayIndex = replayResults.findIndex(
|
|
46361
|
+
(m) => matchedMemoryId && m.id === matchedMemoryId || simFn(goldStatement, m.content) >= threshold
|
|
46362
|
+
);
|
|
46363
|
+
if (replayIndex >= 0) {
|
|
46364
|
+
const rank = replayIndex + 1;
|
|
46365
|
+
retrievalStageMiss = "cap";
|
|
46366
|
+
stages.retrieval = {
|
|
46367
|
+
status: "fail",
|
|
46368
|
+
detail: `Rank ${rank} exceeds recallLimit ${recallLimit}`
|
|
46369
|
+
};
|
|
46370
|
+
} else {
|
|
46371
|
+
retrievalStageMiss = "unknown";
|
|
46372
|
+
stages.retrieval = {
|
|
46373
|
+
status: "fail",
|
|
46374
|
+
detail: `absent from recall at replayLimit ${replayLimit}; filter vs rank indistinguishable without candidate-stage evidence`
|
|
46375
|
+
};
|
|
46376
|
+
}
|
|
46377
|
+
}
|
|
46378
|
+
} catch {
|
|
46379
|
+
stages.retrieval = { status: "unavailable", detail: "recall threw error" };
|
|
46380
|
+
}
|
|
46381
|
+
} else {
|
|
46382
|
+
stages.retrieval = { status: "unavailable", detail: "retrieval check unavailable" };
|
|
46383
|
+
}
|
|
46384
|
+
if (!retrievalCheckPassed && goldInRecalledText) {
|
|
46385
|
+
retrievalCheckPassed = true;
|
|
46386
|
+
stages.retrieval = { status: "pass", detail: "Found in recalledText context" };
|
|
46387
|
+
}
|
|
46388
|
+
if (retrievalCheckPassed) {
|
|
46389
|
+
if (indexCheckFailed) {
|
|
46390
|
+
stages.index = { status: "pass", detail: "implied pass from retrieval (oracle query missed)" };
|
|
46391
|
+
indexCheckPassed = true;
|
|
46392
|
+
} else if (stages.index.status === "unavailable") {
|
|
46393
|
+
stages.index = { status: "pass", detail: "implied pass from retrieval" };
|
|
46394
|
+
indexCheckPassed = true;
|
|
46395
|
+
}
|
|
46396
|
+
}
|
|
46397
|
+
if (retrievalCheckPassed) {
|
|
46398
|
+
stages.use = {
|
|
46399
|
+
status: "fail",
|
|
46400
|
+
detail: "Gold memory present in context but answer was incorrect"
|
|
46401
|
+
};
|
|
46402
|
+
return {
|
|
46403
|
+
goldMemory: goldStatement,
|
|
46404
|
+
label: { class: "use_miss", reason: "Gold memory present in context but task failed" },
|
|
46405
|
+
stages
|
|
46406
|
+
};
|
|
46407
|
+
}
|
|
46408
|
+
if (stages.extraction.status === "pass" && indexCheckFailed) {
|
|
46409
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
46410
|
+
return {
|
|
46411
|
+
goldMemory: goldStatement,
|
|
46412
|
+
label: { class: "index_miss", reason: "Gold statement missing from search index" },
|
|
46413
|
+
stages
|
|
46414
|
+
};
|
|
46415
|
+
}
|
|
46416
|
+
if (stages.extraction.status === "pass" && stages.index.status === "pass" && stages.retrieval.status === "fail") {
|
|
46417
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
46418
|
+
return {
|
|
46419
|
+
goldMemory: goldStatement,
|
|
46420
|
+
label: {
|
|
46421
|
+
class: "retrieval_miss",
|
|
46422
|
+
retrievalStage: retrievalStageMiss ?? "unknown",
|
|
46423
|
+
reason: stages.retrieval.detail
|
|
46424
|
+
},
|
|
46425
|
+
stages
|
|
46426
|
+
};
|
|
46427
|
+
}
|
|
46428
|
+
const missingReason = stages.extraction.status === "unavailable" ? `extraction check unavailable (${stages.extraction.detail})` : stages.index.status === "unavailable" && stages.retrieval.status === "unavailable" ? "index/retrieval checks unavailable in this attribution environment" : stages.index.status === "unavailable" ? "index check unavailable; a retrieval miss cannot be isolated from an index miss" : "retrieval check unavailable";
|
|
46429
|
+
stages.use = { status: "unavailable", detail: "not reached" };
|
|
46430
|
+
return {
|
|
46431
|
+
goldMemory: goldStatement,
|
|
46432
|
+
label: { class: "unattributed", reason: missingReason },
|
|
46433
|
+
stages
|
|
46434
|
+
};
|
|
46435
|
+
}
|
|
46436
|
+
async function attributeTask(task, env, options = {}) {
|
|
46437
|
+
const golds = task.goldMemories;
|
|
46438
|
+
if (!golds || golds.length === 0) {
|
|
46439
|
+
return null;
|
|
46440
|
+
}
|
|
46441
|
+
const memoizedEnv = withMemoizedListMemories(env);
|
|
46442
|
+
const recalledText = typeof task.details?.recalledText === "string" ? task.details.recalledText : void 0;
|
|
46443
|
+
const goldAttributions = [];
|
|
46444
|
+
for (const gold of golds) {
|
|
46445
|
+
const attr = await attributeGoldMemory(gold, task.question, memoizedEnv, options, recalledText);
|
|
46446
|
+
goldAttributions.push(attr);
|
|
46447
|
+
}
|
|
46448
|
+
const overall = computeOverallLabel(goldAttributions);
|
|
46449
|
+
return {
|
|
46450
|
+
taskId: task.taskId,
|
|
46451
|
+
question: task.question,
|
|
46452
|
+
golds: goldAttributions,
|
|
46453
|
+
overall
|
|
46454
|
+
};
|
|
46455
|
+
}
|
|
46456
|
+
async function attributeRun(result, env, options = {}) {
|
|
46457
|
+
const runId = result.meta?.runId ?? result.meta?.id ?? "unknown-run";
|
|
46458
|
+
const memoizedEnv = withMemoizedListMemories(env);
|
|
46459
|
+
const totals = {
|
|
46460
|
+
extraction_miss: 0,
|
|
46461
|
+
index_miss: 0,
|
|
46462
|
+
retrieval_miss: 0,
|
|
46463
|
+
use_miss: 0,
|
|
46464
|
+
unattributed: 0
|
|
46465
|
+
};
|
|
46466
|
+
const retrievalStages = {
|
|
46467
|
+
filter: 0,
|
|
46468
|
+
cap: 0,
|
|
46469
|
+
rank: 0,
|
|
46470
|
+
unknown: 0
|
|
46471
|
+
};
|
|
46472
|
+
const items = [];
|
|
46473
|
+
const skippedTasks = [];
|
|
46474
|
+
for (const task of result.results.tasks) {
|
|
46475
|
+
if (task.details?.benchmarkFailure && typeof task.details.benchmarkFailure === "object") {
|
|
46476
|
+
skippedTasks.push({
|
|
46477
|
+
taskId: task.taskId,
|
|
46478
|
+
reason: "trial execution failure (not an answer failure)"
|
|
46479
|
+
});
|
|
46480
|
+
continue;
|
|
46481
|
+
}
|
|
46482
|
+
if (!task.goldMemories || task.goldMemories.length === 0) {
|
|
46483
|
+
skippedTasks.push({
|
|
46484
|
+
taskId: task.taskId,
|
|
46485
|
+
reason: "No goldMemories specified"
|
|
46486
|
+
});
|
|
46487
|
+
continue;
|
|
46488
|
+
}
|
|
46489
|
+
if (!isTaskFailed(task)) {
|
|
46490
|
+
skippedTasks.push({
|
|
46491
|
+
taskId: task.taskId,
|
|
46492
|
+
reason: "Task passed (score >= 1)"
|
|
46493
|
+
});
|
|
46494
|
+
continue;
|
|
46495
|
+
}
|
|
46496
|
+
const taskAttr = await attributeTask(task, memoizedEnv, options);
|
|
46497
|
+
if (taskAttr) {
|
|
46498
|
+
items.push(taskAttr);
|
|
46499
|
+
}
|
|
46500
|
+
}
|
|
46501
|
+
items.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
|
|
46502
|
+
skippedTasks.sort((a, b) => a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0);
|
|
46503
|
+
for (const item of items) {
|
|
46504
|
+
totals[item.overall.class]++;
|
|
46505
|
+
if (item.overall.class === "retrieval_miss") {
|
|
46506
|
+
const stage = item.overall.retrievalStage ?? "unknown";
|
|
46507
|
+
retrievalStages[stage]++;
|
|
46508
|
+
}
|
|
46509
|
+
}
|
|
46510
|
+
return {
|
|
46511
|
+
runId,
|
|
46512
|
+
totals,
|
|
46513
|
+
retrievalStages,
|
|
46514
|
+
attributedTasks: items.length,
|
|
46515
|
+
skippedTasks,
|
|
46516
|
+
items
|
|
46517
|
+
};
|
|
46518
|
+
}
|
|
46519
|
+
function renderAttributionReportTable(report) {
|
|
46520
|
+
const lines = [];
|
|
46521
|
+
lines.push(`Attribution Report (Run: ${report.runId})`);
|
|
46522
|
+
lines.push(`Failed-task predicate: minimum primary answer score < 1 (scores.overall or non-diagnostic scores)`);
|
|
46523
|
+
lines.push(`Attributed tasks: ${report.attributedTasks}, Skipped tasks: ${report.skippedTasks.length}`);
|
|
46524
|
+
lines.push("");
|
|
46525
|
+
lines.push("Totals by Class:");
|
|
46526
|
+
lines.push(` extraction_miss: ${report.totals.extraction_miss}`);
|
|
46527
|
+
lines.push(` index_miss: ${report.totals.index_miss}`);
|
|
46528
|
+
lines.push(` retrieval_miss: ${report.totals.retrieval_miss}`);
|
|
46529
|
+
lines.push(` use_miss: ${report.totals.use_miss}`);
|
|
46530
|
+
lines.push(` unattributed: ${report.totals.unattributed}`);
|
|
46531
|
+
lines.push("");
|
|
46532
|
+
lines.push("Retrieval Miss Stages:");
|
|
46533
|
+
lines.push(` filter: ${report.retrievalStages.filter}`);
|
|
46534
|
+
lines.push(` cap: ${report.retrievalStages.cap}`);
|
|
46535
|
+
lines.push(` rank: ${report.retrievalStages.rank}`);
|
|
46536
|
+
lines.push(` unknown: ${report.retrievalStages.unknown}`);
|
|
46537
|
+
lines.push("");
|
|
46538
|
+
lines.push("Task Attributions:");
|
|
46539
|
+
if (report.items.length === 0) {
|
|
46540
|
+
lines.push(" (none)");
|
|
46541
|
+
} else {
|
|
46542
|
+
for (const item of report.items) {
|
|
46543
|
+
const stageStr = item.overall.retrievalStage ? ` (${item.overall.retrievalStage})` : "";
|
|
46544
|
+
const labelStr = `${item.overall.class}${stageStr}`;
|
|
46545
|
+
const reasonStr = item.overall.reason ? ` - ${item.overall.reason}` : "";
|
|
46546
|
+
lines.push(` ${item.taskId.padEnd(20)} ${labelStr.padEnd(24)}${reasonStr}`);
|
|
46547
|
+
}
|
|
46548
|
+
}
|
|
46549
|
+
if (report.skippedTasks.length > 0) {
|
|
46550
|
+
lines.push("");
|
|
46551
|
+
lines.push("Skipped Tasks:");
|
|
46552
|
+
for (const skipped of report.skippedTasks) {
|
|
46553
|
+
lines.push(` ${skipped.taskId.padEnd(20)} ${skipped.reason}`);
|
|
46554
|
+
}
|
|
46555
|
+
}
|
|
46556
|
+
return `${lines.join("\n")}
|
|
46557
|
+
`;
|
|
46558
|
+
}
|
|
46559
|
+
function serializeAttributionReport(report) {
|
|
46560
|
+
return `${JSON.stringify(report, null, 2)}
|
|
46561
|
+
`;
|
|
46562
|
+
}
|
|
46563
|
+
|
|
46564
|
+
// src/attribute-cli.ts
|
|
46565
|
+
import { lstat as lstat6, readdir as readdir8, readFile as readFile25 } from "fs/promises";
|
|
46566
|
+
import path40 from "path";
|
|
46567
|
+
function parseFrontmatter2(fileContent) {
|
|
46568
|
+
const lines = fileContent.split(/\r?\n/);
|
|
46569
|
+
if (lines.length > 0 && lines[0].trim() === "---") {
|
|
46570
|
+
let closingIndex = -1;
|
|
46571
|
+
for (let i = 1; i < lines.length; i++) {
|
|
46572
|
+
if (lines[i].trim() === "---") {
|
|
46573
|
+
closingIndex = i;
|
|
46574
|
+
break;
|
|
46575
|
+
}
|
|
46576
|
+
}
|
|
46577
|
+
if (closingIndex >= 1) {
|
|
46578
|
+
const fmLines = lines.slice(1, closingIndex);
|
|
46579
|
+
let id = void 0;
|
|
46580
|
+
for (const line of fmLines) {
|
|
46581
|
+
const match = line.match(/^id:\s*(.+)$/);
|
|
46582
|
+
if (match) {
|
|
46583
|
+
id = match[1].trim().replace(/^["']|["']$/g, "");
|
|
46584
|
+
break;
|
|
46585
|
+
}
|
|
46586
|
+
}
|
|
46587
|
+
const body = lines.slice(closingIndex + 1).join("\n");
|
|
46588
|
+
return { id, body };
|
|
46589
|
+
}
|
|
46590
|
+
}
|
|
46591
|
+
return { body: fileContent };
|
|
46592
|
+
}
|
|
46593
|
+
var SKIPPED_SYSTEM_DIRS = {
|
|
46594
|
+
activity: true,
|
|
46595
|
+
meetings: true,
|
|
46596
|
+
questions: true,
|
|
46597
|
+
state: true,
|
|
46598
|
+
wearables: true
|
|
46599
|
+
};
|
|
46600
|
+
async function assertReadableMemoryDir(dirPath) {
|
|
46601
|
+
let rootStats;
|
|
46602
|
+
try {
|
|
46603
|
+
rootStats = await lstat6(dirPath);
|
|
46604
|
+
} catch {
|
|
46605
|
+
throw new Error(`memory-dir "${dirPath}" is not a readable directory`);
|
|
46606
|
+
}
|
|
46607
|
+
if (rootStats.isSymbolicLink()) {
|
|
46608
|
+
throw new Error(`memory-dir "${dirPath}" must not be a symlink`);
|
|
46609
|
+
}
|
|
46610
|
+
if (!rootStats.isDirectory()) {
|
|
46611
|
+
throw new Error(`memory-dir "${dirPath}" is not a readable directory`);
|
|
46612
|
+
}
|
|
46613
|
+
}
|
|
46614
|
+
async function scanMemoryDir(dirPath) {
|
|
46615
|
+
await assertReadableMemoryDir(dirPath);
|
|
46616
|
+
const memories = [];
|
|
46617
|
+
let unreadableEntries = 0;
|
|
46618
|
+
async function walk(currentDir, depth) {
|
|
46619
|
+
let entries;
|
|
46620
|
+
try {
|
|
46621
|
+
entries = await readdir8(currentDir, { withFileTypes: true });
|
|
46622
|
+
} catch {
|
|
46623
|
+
unreadableEntries++;
|
|
46624
|
+
return;
|
|
46625
|
+
}
|
|
46626
|
+
for (const entry of entries) {
|
|
46627
|
+
if (entry.isSymbolicLink()) {
|
|
46628
|
+
continue;
|
|
46629
|
+
}
|
|
46630
|
+
const fullPath = path40.join(currentDir, entry.name);
|
|
46631
|
+
try {
|
|
46632
|
+
const stats = await lstat6(fullPath);
|
|
46633
|
+
if (stats.isSymbolicLink()) {
|
|
46634
|
+
continue;
|
|
46635
|
+
}
|
|
46636
|
+
if (stats.isDirectory()) {
|
|
46637
|
+
if (depth === 0 && SKIPPED_SYSTEM_DIRS[entry.name]) {
|
|
46638
|
+
continue;
|
|
46639
|
+
}
|
|
46640
|
+
await walk(fullPath, depth + 1);
|
|
46641
|
+
} else if (stats.isFile() && entry.name.endsWith(".md")) {
|
|
46642
|
+
const content = await readFile25(fullPath, "utf8");
|
|
46643
|
+
const { id, body } = parseFrontmatter2(content);
|
|
46644
|
+
const relPath = path40.relative(dirPath, fullPath);
|
|
46645
|
+
memories.push({
|
|
46646
|
+
id: id ?? relPath,
|
|
46647
|
+
content: body.trim()
|
|
46648
|
+
});
|
|
46649
|
+
}
|
|
46650
|
+
} catch {
|
|
46651
|
+
unreadableEntries++;
|
|
46652
|
+
}
|
|
46653
|
+
}
|
|
46654
|
+
}
|
|
46655
|
+
await walk(dirPath, 0);
|
|
46656
|
+
if (unreadableEntries > 0) {
|
|
46657
|
+
throw new Error(`memory scan incomplete: ${unreadableEntries} unreadable entries under ${dirPath}`);
|
|
46658
|
+
}
|
|
46659
|
+
return memories;
|
|
46660
|
+
}
|
|
46661
|
+
async function runAttributeCliCommand(options) {
|
|
46662
|
+
const summary = await resolveBenchmarkResultReference(options.resultsDir, options.runRef);
|
|
46663
|
+
if (!summary) {
|
|
46664
|
+
return {
|
|
46665
|
+
exitCode: 1,
|
|
46666
|
+
output: `Error: Benchmark run reference "${options.runRef}" was not found in "${options.resultsDir}".
|
|
46667
|
+
`
|
|
46668
|
+
};
|
|
46669
|
+
}
|
|
46670
|
+
let result;
|
|
46671
|
+
try {
|
|
46672
|
+
result = await loadBenchmarkResult(summary.path);
|
|
46673
|
+
} catch {
|
|
46674
|
+
return {
|
|
46675
|
+
exitCode: 1,
|
|
46676
|
+
output: `Error: failed to load benchmark result for run "${options.runRef}": file unreadable or invalid
|
|
46677
|
+
`
|
|
46678
|
+
};
|
|
46679
|
+
}
|
|
46680
|
+
let memorySnapshot;
|
|
46681
|
+
const listMemoriesFn = async () => {
|
|
46682
|
+
if (memorySnapshot) return memorySnapshot;
|
|
46683
|
+
if (!options.memoryDir) throw new Error("memoryDir not provided");
|
|
46684
|
+
memorySnapshot = await scanMemoryDir(options.memoryDir);
|
|
46685
|
+
return memorySnapshot;
|
|
46686
|
+
};
|
|
46687
|
+
if (options.memoryDir) {
|
|
46688
|
+
try {
|
|
46689
|
+
await assertReadableMemoryDir(options.memoryDir);
|
|
46690
|
+
} catch (error) {
|
|
46691
|
+
return {
|
|
46692
|
+
exitCode: 1,
|
|
46693
|
+
output: `Error: ${error instanceof Error ? error.message : `memory-dir "${options.memoryDir}" is not a readable directory`}
|
|
46694
|
+
`
|
|
46695
|
+
};
|
|
46696
|
+
}
|
|
46697
|
+
}
|
|
46698
|
+
const recallLimitRaw = result.config?.remnicConfig?.recallLimit;
|
|
46699
|
+
const recallLimit = typeof recallLimitRaw === "number" && recallLimitRaw > 0 ? recallLimitRaw : 10;
|
|
46700
|
+
const rankMemories2 = async (query, limit) => (await listMemoriesFn()).map((memory) => ({ memory, score: lexicalSimilarity(query, memory.content) })).filter(({ score }) => score > 0).sort((a, b) => b.score - a.score || a.memory.id.localeCompare(b.memory.id)).slice(0, limit).map(({ memory }) => memory);
|
|
46701
|
+
const env = {
|
|
46702
|
+
listMemories: listMemoriesFn,
|
|
46703
|
+
oracleSearch: async (query, limit) => (await rankMemories2(query, limit)).map(({ id }) => ({ id })),
|
|
46704
|
+
recall: rankMemories2,
|
|
46705
|
+
recallLimit
|
|
46706
|
+
};
|
|
46707
|
+
const report = await attributeRun(result, env, { threshold: options.threshold });
|
|
46708
|
+
const output = options.json ? serializeAttributionReport(report) : renderAttributionReportTable(report);
|
|
46709
|
+
return {
|
|
46710
|
+
exitCode: 0,
|
|
46711
|
+
output
|
|
46712
|
+
};
|
|
46713
|
+
}
|
|
46714
|
+
|
|
46715
|
+
// src/generators/drift-gen/index.ts
|
|
46716
|
+
import { createHash as createHash21 } from "crypto";
|
|
46717
|
+
import { mkdir as mkdir20, readdir as readdir10, rename as rename5, rm as rm16, writeFile as writeFile19 } from "fs/promises";
|
|
46718
|
+
import path42 from "path";
|
|
46719
|
+
|
|
46720
|
+
// src/seeded-random.ts
|
|
46721
|
+
function createSeededRandom2(seed) {
|
|
46722
|
+
if (!Number.isSafeInteger(seed) || seed < 0 || seed > 4294967295) {
|
|
46723
|
+
throw new Error("seed must be an integer in [0, 2^32 - 1]");
|
|
46724
|
+
}
|
|
46725
|
+
let state = seed >>> 0;
|
|
46726
|
+
return () => {
|
|
46727
|
+
state = state + 1831565813 >>> 0;
|
|
46728
|
+
let t = state;
|
|
46729
|
+
t = Math.imul(t ^ t >>> 15, t | 1);
|
|
46730
|
+
t ^= t + Math.imul(t ^ t >>> 7, t | 61);
|
|
46731
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
46732
|
+
};
|
|
46733
|
+
}
|
|
46734
|
+
function randomInt(rng, min, max) {
|
|
46735
|
+
if (!Number.isSafeInteger(min) || !Number.isSafeInteger(max) || max < min) {
|
|
46736
|
+
throw new Error("randomInt bounds must be safe integers with max >= min");
|
|
46737
|
+
}
|
|
46738
|
+
return min + Math.floor(rng() * (max - min + 1));
|
|
46739
|
+
}
|
|
46740
|
+
function pickOne(rng, items) {
|
|
46741
|
+
if (items.length === 0) {
|
|
46742
|
+
throw new Error("pickOne requires a non-empty array");
|
|
46743
|
+
}
|
|
46744
|
+
return items[randomInt(rng, 0, items.length - 1)];
|
|
46745
|
+
}
|
|
46746
|
+
function shuffled(rng, items) {
|
|
46747
|
+
const copy = [...items];
|
|
46748
|
+
for (let i = copy.length - 1; i > 0; i--) {
|
|
46749
|
+
const j = randomInt(rng, 0, i);
|
|
46750
|
+
[copy[i], copy[j]] = [copy[j], copy[i]];
|
|
46751
|
+
}
|
|
46752
|
+
return copy;
|
|
46753
|
+
}
|
|
46754
|
+
|
|
46755
|
+
// src/generators/drift-gen/names.ts
|
|
46756
|
+
var PERSON_NAMES = Object.freeze([
|
|
46757
|
+
"Avery Quill",
|
|
46758
|
+
"Blair Fenwick",
|
|
46759
|
+
"Casey Thornbury",
|
|
46760
|
+
"Devon Marsh",
|
|
46761
|
+
"Emery Latchford",
|
|
46762
|
+
"Finley Crowder",
|
|
46763
|
+
"Greer Halloway",
|
|
46764
|
+
"Harper Vexley",
|
|
46765
|
+
"Indra Colefax",
|
|
46766
|
+
"Jules Ashgrove",
|
|
46767
|
+
"Kendall Bryce",
|
|
46768
|
+
"Lennox Farrow",
|
|
46769
|
+
"Marlow Hutchins",
|
|
46770
|
+
"Noor Delacroix",
|
|
46771
|
+
"Oakes Pemberton",
|
|
46772
|
+
"Perrin Vale",
|
|
46773
|
+
"Quinn Ellsworth",
|
|
46774
|
+
"Rowan Setterfield",
|
|
46775
|
+
"Sage Windmere",
|
|
46776
|
+
"Tatum Okafor-Reyes",
|
|
46777
|
+
"Umber Callahan",
|
|
46778
|
+
"Vesper Lindqvist",
|
|
46779
|
+
"Wren Abernathy",
|
|
46780
|
+
"Xiomara Puddle",
|
|
46781
|
+
"Yael Cormorant",
|
|
46782
|
+
"Zephyr Nakamura-Ross",
|
|
46783
|
+
"Ainsley Duckworth",
|
|
46784
|
+
"Briar Montrose",
|
|
46785
|
+
"Cameron Silverside",
|
|
46786
|
+
"Dakota Fernsby",
|
|
46787
|
+
"Ellis Wintergreen",
|
|
46788
|
+
"Frankie Aldercrest",
|
|
46789
|
+
"Gale Norwood",
|
|
46790
|
+
"Hollis Bramblewood",
|
|
46791
|
+
"Ida Quenneville",
|
|
46792
|
+
"Jorah Templesmith",
|
|
46793
|
+
"Kai Osterberg",
|
|
46794
|
+
"Lark Dunmore",
|
|
46795
|
+
"Micah Featherstone",
|
|
46796
|
+
"Nova Cresswell",
|
|
46797
|
+
"Onyx Ravensworth",
|
|
46798
|
+
"Piper Goldenrod",
|
|
46799
|
+
"Reese Thistleton",
|
|
46800
|
+
"Skyler Marchbanks",
|
|
46801
|
+
"Tobin Everhart",
|
|
46802
|
+
"Uma Kettleburn",
|
|
46803
|
+
"Vaughn Ironwood",
|
|
46804
|
+
"Willa Starling",
|
|
46805
|
+
"Xander Mosswood",
|
|
46806
|
+
"Yuki Alderman",
|
|
46807
|
+
"Zola Brightwater",
|
|
46808
|
+
"Arden Foxglove",
|
|
46809
|
+
"Bellamy Crane",
|
|
46810
|
+
"Cove Whitlock",
|
|
46811
|
+
"Darby Mullins",
|
|
46812
|
+
"Eden Harrowgate",
|
|
46813
|
+
"Fern Oakhurst",
|
|
46814
|
+
"Gray Pennyworth",
|
|
46815
|
+
"Haven Solstice",
|
|
46816
|
+
"Ira Blackwood",
|
|
46817
|
+
"Juniper Wells",
|
|
46818
|
+
"Kit Ramsbottom",
|
|
46819
|
+
"Linden Frost",
|
|
46820
|
+
"Mabel Twinning",
|
|
46821
|
+
"Nico Ashdown",
|
|
46822
|
+
"Opal Ferngale",
|
|
46823
|
+
"Paz Underhill",
|
|
46824
|
+
"Ramona Quickstep",
|
|
46825
|
+
"Story Lockwood",
|
|
46826
|
+
"Teagan Rushmore",
|
|
46827
|
+
"Ursa Middleton",
|
|
46828
|
+
"Vera Nightingale",
|
|
46829
|
+
"West Corwin",
|
|
46830
|
+
"Xen Appleby",
|
|
46831
|
+
"Yara Stormont",
|
|
46832
|
+
"Zane Cobblestone",
|
|
46833
|
+
"Alba Merriweather",
|
|
46834
|
+
"Bram Hollycroft",
|
|
46835
|
+
"Cleo Vandermeer",
|
|
46836
|
+
"Dune Castellan",
|
|
46837
|
+
"Esme Larkspur",
|
|
46838
|
+
"Flint Rosewater",
|
|
46839
|
+
"Gwen Tidewell",
|
|
46840
|
+
"Hart Evergreen",
|
|
46841
|
+
"Isla Puddleby",
|
|
46842
|
+
"Jem Coldbrook",
|
|
46843
|
+
"Koa Silvermane",
|
|
46844
|
+
"Lux Bannister",
|
|
46845
|
+
"Moss Killdare",
|
|
46846
|
+
"Nell Featherby",
|
|
46847
|
+
"Orin Saltmarsh",
|
|
46848
|
+
"Prue Winterbourne",
|
|
46849
|
+
"Rio Glasswing",
|
|
46850
|
+
"Sol Amberfield",
|
|
46851
|
+
"Tess Hollowell",
|
|
46852
|
+
"Ulf Greenmantle",
|
|
46853
|
+
"Vale Northcott",
|
|
46854
|
+
"Wynn Ravenscar",
|
|
46855
|
+
"Yves Millbrook",
|
|
46856
|
+
"Zia Summerfield"
|
|
46857
|
+
]);
|
|
46858
|
+
var COMPANIES = Object.freeze([
|
|
46859
|
+
"Norvig Dynamics",
|
|
46860
|
+
"Quillboard Labs",
|
|
46861
|
+
"Tessellate Works",
|
|
46862
|
+
"Bramblewick Systems",
|
|
46863
|
+
"Copperkite Analytics",
|
|
46864
|
+
"Duskmill Robotics",
|
|
46865
|
+
"Everspool Software",
|
|
46866
|
+
"Fernwhistle Media",
|
|
46867
|
+
"Gladehollow Logistics",
|
|
46868
|
+
"Halcyon Foundry",
|
|
46869
|
+
"Inkmoor Studios",
|
|
46870
|
+
"Juniperline Freight",
|
|
46871
|
+
"Kelpwater Energy",
|
|
46872
|
+
"Lanternfell Games",
|
|
46873
|
+
"Mossbridge Capital",
|
|
46874
|
+
"Nightloom Textiles",
|
|
46875
|
+
"Orchardgate Health",
|
|
46876
|
+
"Pinwheel Cartography",
|
|
46877
|
+
"Quenchpoint Beverages",
|
|
46878
|
+
"Rooksedge Security",
|
|
46879
|
+
"Saltfern Biotech",
|
|
46880
|
+
"Thistledown Press",
|
|
46881
|
+
"Umbershade Optics",
|
|
46882
|
+
"Violetmarsh Farms"
|
|
46883
|
+
]);
|
|
46884
|
+
var CITIES = Object.freeze([
|
|
46885
|
+
"Cinder Falls",
|
|
46886
|
+
"Bracken Hollow",
|
|
46887
|
+
"Marrow Bay",
|
|
46888
|
+
"Gilded Prairie",
|
|
46889
|
+
"Foxfire Junction",
|
|
46890
|
+
"Willowmere Heights",
|
|
46891
|
+
"Copper Basin",
|
|
46892
|
+
"Larkspur Valley",
|
|
46893
|
+
"Ember Grove",
|
|
46894
|
+
"Tidewater Crossing",
|
|
46895
|
+
"Hollow Pines",
|
|
46896
|
+
"Sable Ridge",
|
|
46897
|
+
"Quartz Harbor",
|
|
46898
|
+
"Meadowbrook Flats",
|
|
46899
|
+
"Ashen Cove",
|
|
46900
|
+
"Thornfield Downs"
|
|
46901
|
+
]);
|
|
46902
|
+
var ROLES = Object.freeze([
|
|
46903
|
+
"data engineer",
|
|
46904
|
+
"product designer",
|
|
46905
|
+
"field technician",
|
|
46906
|
+
"staff writer",
|
|
46907
|
+
"release manager",
|
|
46908
|
+
"support lead",
|
|
46909
|
+
"research analyst",
|
|
46910
|
+
"site planner",
|
|
46911
|
+
"platform architect",
|
|
46912
|
+
"operations coordinator",
|
|
46913
|
+
"quality auditor",
|
|
46914
|
+
"studio producer"
|
|
46915
|
+
]);
|
|
46916
|
+
var HOBBIES = Object.freeze([
|
|
46917
|
+
"kite surfing",
|
|
46918
|
+
"letterpress printing",
|
|
46919
|
+
"fossil hunting",
|
|
46920
|
+
"marble sculpting",
|
|
46921
|
+
"night birding",
|
|
46922
|
+
"canyon sketching",
|
|
46923
|
+
"sourdough baking",
|
|
46924
|
+
"river kayaking",
|
|
46925
|
+
"lantern making",
|
|
46926
|
+
"orchard grafting",
|
|
46927
|
+
"tide pooling",
|
|
46928
|
+
"glass blowing"
|
|
46929
|
+
]);
|
|
46930
|
+
var PRODUCTS = Object.freeze([
|
|
46931
|
+
"quillboard sync",
|
|
46932
|
+
"lanternfell tracker",
|
|
46933
|
+
"mosspad notebook",
|
|
46934
|
+
"copperkite dashboard",
|
|
46935
|
+
"duskmill planner",
|
|
46936
|
+
"everspool console",
|
|
46937
|
+
"pinwheel atlas",
|
|
46938
|
+
"saltfern ledger",
|
|
46939
|
+
"thistledown reader",
|
|
46940
|
+
"umbershade viewer",
|
|
46941
|
+
"inkmoor canvas",
|
|
46942
|
+
"rooksedge vault"
|
|
46943
|
+
]);
|
|
46944
|
+
var PETS = Object.freeze([
|
|
46945
|
+
"a tabby cat named Biscuit",
|
|
46946
|
+
"a beagle named Waffles",
|
|
46947
|
+
"a parakeet named Comet",
|
|
46948
|
+
"a corgi named Turnip",
|
|
46949
|
+
"a gecko named Pebble",
|
|
46950
|
+
"a rabbit named Clover",
|
|
46951
|
+
"an aquarium of guppies",
|
|
46952
|
+
"a sheepdog named Bramble",
|
|
46953
|
+
"a canary named Solstice",
|
|
46954
|
+
"a tortoise named Meander",
|
|
46955
|
+
"a ferret named Nimbus",
|
|
46956
|
+
"a calico cat named Marzipan"
|
|
46957
|
+
]);
|
|
46958
|
+
var PROJECTS = Object.freeze([
|
|
46959
|
+
"the harborlight migration",
|
|
46960
|
+
"the emberwick redesign",
|
|
46961
|
+
"the saltmarsh rollout",
|
|
46962
|
+
"the glasswing prototype",
|
|
46963
|
+
"the thornbury audit",
|
|
46964
|
+
"the meadowlark archive",
|
|
46965
|
+
"the cindertrail survey",
|
|
46966
|
+
"the quenchpoint launch",
|
|
46967
|
+
"the fernshade cleanup",
|
|
46968
|
+
"the marrowgate integration",
|
|
46969
|
+
"the willowmere pilot",
|
|
46970
|
+
"the sableridge handover"
|
|
46971
|
+
]);
|
|
46972
|
+
|
|
46973
|
+
// src/generators/drift-gen/schedule.ts
|
|
46974
|
+
var ATTRIBUTE_SPECS = Object.freeze([
|
|
46975
|
+
{
|
|
46976
|
+
attribute: "employer",
|
|
46977
|
+
values: COMPANIES,
|
|
46978
|
+
clause: (v) => `works at ${v}`,
|
|
46979
|
+
firstPersonClause: (v) => `work at ${v}`,
|
|
46980
|
+
questionCurrent: (s) => `Where does ${s} work these days?`,
|
|
46981
|
+
questionHistorical: (s) => `Which employer did ${s} have before the most recent change?`,
|
|
46982
|
+
questionTransition: (s) => `How did ${s}'s employer change?`,
|
|
46983
|
+
noun: "employer"
|
|
46984
|
+
},
|
|
46985
|
+
{
|
|
46986
|
+
attribute: "role",
|
|
46987
|
+
values: ROLES,
|
|
46988
|
+
clause: (v) => `is ${/^[aeiou]/.test(v) ? "an" : "a"} ${v}`,
|
|
46989
|
+
firstPersonClause: (v) => `am ${/^[aeiou]/.test(v) ? "an" : "a"} ${v}`,
|
|
46990
|
+
questionCurrent: (s) => `What does ${s} do for a living now?`,
|
|
46991
|
+
questionHistorical: (s) => `What was ${s}'s job title before the most recent change?`,
|
|
46992
|
+
questionTransition: (s) => `How did ${s}'s job change?`,
|
|
46993
|
+
noun: "job title"
|
|
46994
|
+
},
|
|
46995
|
+
{
|
|
46996
|
+
attribute: "city",
|
|
46997
|
+
values: CITIES,
|
|
46998
|
+
clause: (v) => `lives in ${v}`,
|
|
46999
|
+
firstPersonClause: (v) => `live in ${v}`,
|
|
47000
|
+
questionCurrent: (s) => `Which city is ${s} living in currently?`,
|
|
47001
|
+
questionHistorical: (s) => `Where did ${s} live before the most recent move?`,
|
|
47002
|
+
questionTransition: (s) => `How did ${s}'s home city change?`,
|
|
47003
|
+
noun: "home city"
|
|
47004
|
+
},
|
|
47005
|
+
{
|
|
47006
|
+
attribute: "hobby",
|
|
47007
|
+
values: HOBBIES,
|
|
47008
|
+
clause: (v) => `has gotten into ${v}`,
|
|
47009
|
+
firstPersonClause: (v) => `have gotten into ${v}`,
|
|
47010
|
+
questionCurrent: (s) => `What pastime is ${s} into at the moment?`,
|
|
47011
|
+
questionHistorical: (s) => `What pastime was ${s} into before the most recent switch?`,
|
|
47012
|
+
questionTransition: (s) => `How did ${s}'s main pastime change?`,
|
|
47013
|
+
noun: "main pastime"
|
|
47014
|
+
},
|
|
47015
|
+
{
|
|
47016
|
+
attribute: "pet",
|
|
47017
|
+
values: PETS,
|
|
47018
|
+
clause: (v) => `has ${v}`,
|
|
47019
|
+
firstPersonClause: (v) => `have ${v}`,
|
|
47020
|
+
questionCurrent: (s) => `What animal companion does ${s} keep right now?`,
|
|
47021
|
+
questionHistorical: (s) => `What animal companion did ${s} keep before the most recent change?`,
|
|
47022
|
+
questionTransition: (s) => `How did ${s}'s animal companion situation change?`,
|
|
47023
|
+
noun: "animal companion"
|
|
47024
|
+
},
|
|
47025
|
+
{
|
|
47026
|
+
attribute: "favorite-tool",
|
|
47027
|
+
values: PRODUCTS,
|
|
47028
|
+
clause: (v) => `relies on the ${v} for daily planning`,
|
|
47029
|
+
firstPersonClause: (v) => `rely on the ${v} for daily planning`,
|
|
47030
|
+
questionCurrent: (s) => `Which planning app does ${s} rely on at the moment?`,
|
|
47031
|
+
questionHistorical: (s) => `Which planning app did ${s} rely on before the most recent switch?`,
|
|
47032
|
+
questionTransition: (s) => `How did ${s}'s planning app choice change?`,
|
|
47033
|
+
noun: "planning app"
|
|
47034
|
+
},
|
|
47035
|
+
{
|
|
47036
|
+
attribute: "project",
|
|
47037
|
+
values: PROJECTS,
|
|
47038
|
+
clause: (v) => `is leading ${v}`,
|
|
47039
|
+
firstPersonClause: (v) => `am leading ${v}`,
|
|
47040
|
+
questionCurrent: (s) => `Which initiative is ${s} leading right now?`,
|
|
47041
|
+
questionHistorical: (s) => `Which initiative did ${s} lead before the most recent handover?`,
|
|
47042
|
+
questionTransition: (s) => `How did the initiative ${s} leads change?`,
|
|
47043
|
+
noun: "current initiative"
|
|
47044
|
+
}
|
|
47045
|
+
]);
|
|
47046
|
+
var MIN_DRIFT_GAP = 2;
|
|
47047
|
+
var MAX_DRIFT_GAP = 5;
|
|
47048
|
+
var AGGREGATION_EPOCH_INTERVAL = 2;
|
|
47049
|
+
var AGGREGATION_PROBES_PER_EPOCH = 4;
|
|
47050
|
+
var MIN_AGGREGATION_FACTS = 3;
|
|
47051
|
+
var MAX_AGGREGATION_FACTS = 6;
|
|
47052
|
+
var CONTACTS_PER_USER = 15;
|
|
47053
|
+
function buildCorpusSchedule(options) {
|
|
47054
|
+
validateScheduleOptions(options);
|
|
47055
|
+
const rng = createSeededRandom2(options.seed);
|
|
47056
|
+
const personaPool = shuffled(rng, PERSON_NAMES);
|
|
47057
|
+
const users = [];
|
|
47058
|
+
const allFacts = [];
|
|
47059
|
+
const allProbes = [];
|
|
47060
|
+
for (let u = 0; u < options.users; u++) {
|
|
47061
|
+
const userId = `u${u + 1}`;
|
|
47062
|
+
const persona = personaPool[u % personaPool.length];
|
|
47063
|
+
const contacts = buildContacts(rng, persona);
|
|
47064
|
+
const subjects = [persona, ...contacts];
|
|
47065
|
+
const facts = [];
|
|
47066
|
+
const factById = /* @__PURE__ */ new Map();
|
|
47067
|
+
const activeByPair = /* @__PURE__ */ new Map();
|
|
47068
|
+
const pending = [];
|
|
47069
|
+
for (let epoch = 1; epoch <= options.epochs; epoch++) {
|
|
47070
|
+
const due = takeDueSupersessions(pending, epoch);
|
|
47071
|
+
due.sort((a, b) => {
|
|
47072
|
+
const aPriority = factById.get(a.factId)?.kind === "contradicted" ? 0 : 1;
|
|
47073
|
+
const bPriority = factById.get(b.factId)?.kind === "contradicted" ? 0 : 1;
|
|
47074
|
+
return aPriority === bPriority ? 0 : aPriority < bPriority ? -1 : 1;
|
|
47075
|
+
});
|
|
47076
|
+
let created = 0;
|
|
47077
|
+
for (const item of due) {
|
|
47078
|
+
if (created >= options.factsPerEpoch) {
|
|
47079
|
+
pending.push({ epoch: epoch + 1, factId: item.factId });
|
|
47080
|
+
continue;
|
|
47081
|
+
}
|
|
47082
|
+
const oldFact = factById.get(item.factId);
|
|
47083
|
+
if (!oldFact || oldFact.supersededBy !== null) continue;
|
|
47084
|
+
const successor = createSuccessorFact(rng, options, oldFact, epoch, facts.length);
|
|
47085
|
+
oldFact.supersededEpoch = epoch;
|
|
47086
|
+
oldFact.supersededBy = successor.id;
|
|
47087
|
+
registerFact(successor, facts, factById, activeByPair);
|
|
47088
|
+
scheduleLifecycle(rng, options, successor, epoch, pending);
|
|
47089
|
+
created++;
|
|
47090
|
+
}
|
|
47091
|
+
while (created < options.factsPerEpoch) {
|
|
47092
|
+
const fresh = createFreshFact(rng, options, userId, subjects, activeByPair, epoch, facts.length);
|
|
47093
|
+
registerFact(fresh, facts, factById, activeByPair);
|
|
47094
|
+
scheduleLifecycle(rng, options, fresh, epoch, pending);
|
|
47095
|
+
created++;
|
|
47096
|
+
}
|
|
47097
|
+
}
|
|
47098
|
+
for (const fact3 of facts) {
|
|
47099
|
+
if (fact3.supersededEpoch === null) {
|
|
47100
|
+
fact3.kind = "stable";
|
|
47101
|
+
} else if (fact3.supersededEpoch === fact3.introducedEpoch + 1) {
|
|
47102
|
+
fact3.kind = "contradicted";
|
|
47103
|
+
} else {
|
|
47104
|
+
fact3.kind = "drifting";
|
|
47105
|
+
}
|
|
47106
|
+
}
|
|
47107
|
+
attachSingleFactProbes(facts, factById, options.epochs);
|
|
47108
|
+
const aggregation = buildAggregationProbes(rng, userId, facts, options.epochs);
|
|
47109
|
+
users.push({ userId, persona, facts });
|
|
47110
|
+
allFacts.push(...facts);
|
|
47111
|
+
for (const fact3 of facts) allProbes.push(...fact3.probes);
|
|
47112
|
+
allProbes.push(...aggregation);
|
|
47113
|
+
}
|
|
47114
|
+
allProbes.sort(compareProbes);
|
|
47115
|
+
return { users, facts: allFacts, probes: allProbes };
|
|
47116
|
+
}
|
|
47117
|
+
function validateScheduleOptions(options) {
|
|
47118
|
+
if (!Number.isSafeInteger(options.users) || options.users < 1) {
|
|
47119
|
+
throw new Error("drift-gen users must be a positive integer");
|
|
47120
|
+
}
|
|
47121
|
+
if (!Number.isSafeInteger(options.epochs) || options.epochs < 2) {
|
|
47122
|
+
throw new Error("drift-gen epochs must be an integer >= 2 (supersession needs at least two epochs)");
|
|
47123
|
+
}
|
|
47124
|
+
if (!Number.isSafeInteger(options.factsPerEpoch) || options.factsPerEpoch < 1) {
|
|
47125
|
+
throw new Error("drift-gen factsPerEpoch must be a positive integer");
|
|
47126
|
+
}
|
|
47127
|
+
for (const [name, value] of [
|
|
47128
|
+
["driftingRatio", options.driftingRatio],
|
|
47129
|
+
["contradictedRatio", options.contradictedRatio]
|
|
47130
|
+
]) {
|
|
47131
|
+
if (!Number.isFinite(value) || value < 0 || value > 1) {
|
|
47132
|
+
throw new Error(`drift-gen ${name} must be a finite number in [0, 1]`);
|
|
47133
|
+
}
|
|
47134
|
+
}
|
|
47135
|
+
if (options.driftingRatio + options.contradictedRatio > 1) {
|
|
47136
|
+
throw new Error("drift-gen driftingRatio + contradictedRatio must not exceed 1");
|
|
47137
|
+
}
|
|
47138
|
+
const pairCapacity = (CONTACTS_PER_USER + 1) * ATTRIBUTE_SPECS.length;
|
|
47139
|
+
const worstCaseFresh = options.epochs * options.factsPerEpoch;
|
|
47140
|
+
if (worstCaseFresh > pairCapacity) {
|
|
47141
|
+
throw new Error(
|
|
47142
|
+
`drift-gen cannot allocate ${worstCaseFresh} facts per user: only ${pairCapacity} unique subject/attribute pairs exist. Lower epochs or factsPerEpoch.`
|
|
47143
|
+
);
|
|
47144
|
+
}
|
|
47145
|
+
}
|
|
47146
|
+
function buildContacts(rng, persona) {
|
|
47147
|
+
const pool = shuffled(rng, PERSON_NAMES.filter((name) => name !== persona));
|
|
47148
|
+
return pool.slice(0, CONTACTS_PER_USER);
|
|
47149
|
+
}
|
|
47150
|
+
function registerFact(fact3, facts, factById, activeByPair) {
|
|
47151
|
+
facts.push(fact3);
|
|
47152
|
+
factById.set(fact3.id, fact3);
|
|
47153
|
+
activeByPair.set(`${fact3.subject}|${fact3.attribute}`, fact3);
|
|
47154
|
+
}
|
|
47155
|
+
function takeDueSupersessions(pending, epoch) {
|
|
47156
|
+
const due = pending.filter((p) => p.epoch === epoch);
|
|
47157
|
+
let write = 0;
|
|
47158
|
+
for (const item of pending) {
|
|
47159
|
+
if (item.epoch !== epoch) pending[write++] = item;
|
|
47160
|
+
}
|
|
47161
|
+
pending.length = write;
|
|
47162
|
+
return due;
|
|
47163
|
+
}
|
|
47164
|
+
function rollKind(rng, options, epoch) {
|
|
47165
|
+
if (epoch >= options.epochs) return "stable";
|
|
47166
|
+
const roll = rng();
|
|
47167
|
+
if (roll < options.contradictedRatio) return "contradicted";
|
|
47168
|
+
if (roll < options.contradictedRatio + options.driftingRatio) {
|
|
47169
|
+
return epoch + MIN_DRIFT_GAP <= options.epochs ? "drifting" : "contradicted";
|
|
47170
|
+
}
|
|
47171
|
+
return "stable";
|
|
47172
|
+
}
|
|
47173
|
+
function scheduleLifecycle(rng, options, fact3, epoch, pending) {
|
|
47174
|
+
if (fact3.kind === "contradicted") {
|
|
47175
|
+
pending.push({ epoch: epoch + 1, factId: fact3.id });
|
|
47176
|
+
} else if (fact3.kind === "drifting") {
|
|
47177
|
+
const maxGap = Math.min(MAX_DRIFT_GAP, options.epochs - epoch);
|
|
47178
|
+
const gap = randomInt(rng, MIN_DRIFT_GAP, Math.max(MIN_DRIFT_GAP, maxGap));
|
|
47179
|
+
pending.push({ epoch: epoch + gap, factId: fact3.id });
|
|
47180
|
+
}
|
|
47181
|
+
}
|
|
47182
|
+
function specFor(attribute) {
|
|
47183
|
+
const spec = ATTRIBUTE_SPECS.find((s) => s.attribute === attribute);
|
|
47184
|
+
if (!spec) throw new Error(`unknown drift-gen attribute: ${attribute}`);
|
|
47185
|
+
return spec;
|
|
47186
|
+
}
|
|
47187
|
+
function formatFactStatement(subject, attribute, value) {
|
|
47188
|
+
return `${subject} ${specFor(attribute).clause(value)}.`;
|
|
47189
|
+
}
|
|
47190
|
+
function createFreshFact(rng, options, userId, subjects, activeByPair, epoch, ordinal) {
|
|
47191
|
+
for (let attempt = 0; attempt < 500; attempt++) {
|
|
47192
|
+
const subject = pickOne(rng, subjects);
|
|
47193
|
+
const spec = pickOne(rng, ATTRIBUTE_SPECS);
|
|
47194
|
+
if (activeByPair.has(`${subject}|${spec.attribute}`)) continue;
|
|
47195
|
+
const value = pickOne(rng, spec.values);
|
|
47196
|
+
return {
|
|
47197
|
+
id: `gf-${userId}-${epoch}-${ordinal + 1}`,
|
|
47198
|
+
userId,
|
|
47199
|
+
statement: formatFactStatement(subject, spec.attribute, value),
|
|
47200
|
+
subject,
|
|
47201
|
+
attribute: spec.attribute,
|
|
47202
|
+
value,
|
|
47203
|
+
introducedEpoch: epoch,
|
|
47204
|
+
supersededEpoch: null,
|
|
47205
|
+
supersededBy: null,
|
|
47206
|
+
kind: rollKind(rng, options, epoch),
|
|
47207
|
+
probes: []
|
|
47208
|
+
};
|
|
47209
|
+
}
|
|
47210
|
+
throw new Error(
|
|
47211
|
+
"drift-gen exhausted unique subject/attribute pairs; lower factsPerEpoch or epochs"
|
|
47212
|
+
);
|
|
47213
|
+
}
|
|
47214
|
+
function createSuccessorFact(rng, options, oldFact, epoch, ordinal) {
|
|
47215
|
+
const spec = specFor(oldFact.attribute);
|
|
47216
|
+
const alternatives = spec.values.filter((v) => v !== oldFact.value);
|
|
47217
|
+
const value = pickOne(rng, alternatives);
|
|
47218
|
+
return {
|
|
47219
|
+
id: `gf-${oldFact.userId}-${epoch}-${ordinal + 1}`,
|
|
47220
|
+
userId: oldFact.userId,
|
|
47221
|
+
statement: formatFactStatement(oldFact.subject, oldFact.attribute, value),
|
|
47222
|
+
subject: oldFact.subject,
|
|
47223
|
+
attribute: oldFact.attribute,
|
|
47224
|
+
value,
|
|
47225
|
+
introducedEpoch: epoch,
|
|
47226
|
+
supersededEpoch: null,
|
|
47227
|
+
supersededBy: null,
|
|
47228
|
+
kind: rollKind(rng, options, epoch),
|
|
47229
|
+
probes: []
|
|
47230
|
+
};
|
|
47231
|
+
}
|
|
47232
|
+
function attachSingleFactProbes(facts, factById, epochs) {
|
|
47233
|
+
for (const fact3 of facts) {
|
|
47234
|
+
const spec = specFor(fact3.attribute);
|
|
47235
|
+
let n = 0;
|
|
47236
|
+
const probeEpoch = fact3.introducedEpoch + 1;
|
|
47237
|
+
const stillActiveAtProbe = fact3.supersededEpoch === null || fact3.supersededEpoch > probeEpoch;
|
|
47238
|
+
if (probeEpoch <= epochs && stillActiveAtProbe) {
|
|
47239
|
+
fact3.probes.push({
|
|
47240
|
+
id: `${fact3.id}-p${++n}`,
|
|
47241
|
+
userId: fact3.userId,
|
|
47242
|
+
epoch: probeEpoch,
|
|
47243
|
+
question: spec.questionCurrent(fact3.subject),
|
|
47244
|
+
expectedAnswer: fact3.value,
|
|
47245
|
+
requiredFactIds: [fact3.id],
|
|
47246
|
+
category: "current"
|
|
47247
|
+
});
|
|
47248
|
+
}
|
|
47249
|
+
if (fact3.supersededEpoch !== null && fact3.supersededBy !== null) {
|
|
47250
|
+
const successor = factById.get(fact3.supersededBy);
|
|
47251
|
+
const afterEpoch = fact3.supersededEpoch + 1;
|
|
47252
|
+
const successorCurrentAtProbe = successor !== void 0 && (successor.supersededEpoch === null || successor.supersededEpoch > afterEpoch);
|
|
47253
|
+
if (successor && successorCurrentAtProbe && afterEpoch <= epochs) {
|
|
47254
|
+
fact3.probes.push({
|
|
47255
|
+
id: `${fact3.id}-p${++n}`,
|
|
47256
|
+
userId: fact3.userId,
|
|
47257
|
+
epoch: afterEpoch,
|
|
47258
|
+
question: spec.questionHistorical(fact3.subject),
|
|
47259
|
+
expectedAnswer: fact3.value,
|
|
47260
|
+
requiredFactIds: [fact3.id],
|
|
47261
|
+
category: "historical"
|
|
47262
|
+
});
|
|
47263
|
+
fact3.probes.push({
|
|
47264
|
+
id: `${fact3.id}-p${++n}`,
|
|
47265
|
+
userId: fact3.userId,
|
|
47266
|
+
epoch: afterEpoch,
|
|
47267
|
+
question: spec.questionTransition(fact3.subject),
|
|
47268
|
+
expectedAnswer: `from ${fact3.value} to ${successor.value}`,
|
|
47269
|
+
requiredFactIds: [fact3.id, successor.id],
|
|
47270
|
+
category: "transition"
|
|
47271
|
+
});
|
|
47272
|
+
}
|
|
47273
|
+
}
|
|
47274
|
+
}
|
|
47275
|
+
}
|
|
47276
|
+
function activeFactsAt(facts, epoch) {
|
|
47277
|
+
return facts.filter(
|
|
47278
|
+
(f) => f.introducedEpoch <= epoch && (f.supersededEpoch === null || f.supersededEpoch > epoch)
|
|
47279
|
+
);
|
|
47280
|
+
}
|
|
47281
|
+
function buildAggregationProbes(rng, userId, facts, epochs) {
|
|
47282
|
+
const probes = [];
|
|
47283
|
+
for (let epoch = AGGREGATION_EPOCH_INTERVAL; epoch <= epochs; epoch += AGGREGATION_EPOCH_INTERVAL) {
|
|
47284
|
+
const active = activeFactsAt(facts, epoch);
|
|
47285
|
+
if (active.length < MIN_AGGREGATION_FACTS) continue;
|
|
47286
|
+
for (let p = 0; p < AGGREGATION_PROBES_PER_EPOCH; p++) {
|
|
47287
|
+
const count = Math.min(
|
|
47288
|
+
randomInt(rng, MIN_AGGREGATION_FACTS, MAX_AGGREGATION_FACTS),
|
|
47289
|
+
active.length
|
|
47290
|
+
);
|
|
47291
|
+
const chosen = shuffled(rng, active).slice(0, count);
|
|
47292
|
+
const parts = chosen.map(
|
|
47293
|
+
(f) => `what is ${f.subject}'s ${specFor(f.attribute).noun}`
|
|
47294
|
+
);
|
|
47295
|
+
probes.push({
|
|
47296
|
+
id: `gp-${userId}-${epoch}-agg${p + 1}`,
|
|
47297
|
+
userId,
|
|
47298
|
+
epoch,
|
|
47299
|
+
question: `Answer in order: ${parts.join("; ")}?`,
|
|
47300
|
+
expectedAnswer: chosen.map((f) => f.value).join("; "),
|
|
47301
|
+
requiredFactIds: chosen.map((f) => f.id),
|
|
47302
|
+
category: "aggregation"
|
|
47303
|
+
});
|
|
47304
|
+
}
|
|
47305
|
+
}
|
|
47306
|
+
return probes;
|
|
47307
|
+
}
|
|
47308
|
+
function compareProbes(a, b) {
|
|
47309
|
+
if (a.epoch !== b.epoch) return a.epoch < b.epoch ? -1 : 1;
|
|
47310
|
+
if (a.userId !== b.userId) return a.userId < b.userId ? -1 : 1;
|
|
47311
|
+
if (a.id !== b.id) return a.id < b.id ? -1 : 1;
|
|
47312
|
+
return 0;
|
|
47313
|
+
}
|
|
47314
|
+
var PROBE_CATEGORIES = Object.freeze([
|
|
47315
|
+
"current",
|
|
47316
|
+
"historical",
|
|
47317
|
+
"transition",
|
|
47318
|
+
"aggregation"
|
|
47319
|
+
]);
|
|
47320
|
+
|
|
47321
|
+
// src/generators/drift-gen/render.ts
|
|
47322
|
+
var FRESH_FRAMES = Object.freeze([
|
|
47323
|
+
"By the way, {clause}.",
|
|
47324
|
+
"I wanted to mention that {clause}.",
|
|
47325
|
+
"Oh, before I forget: {clause}.",
|
|
47326
|
+
"Fun fact from this month: {clause}.",
|
|
47327
|
+
"Something new on my end: {clause}.",
|
|
47328
|
+
"Quick note for your records: {clause}.",
|
|
47329
|
+
"In case it ever comes up, {clause}.",
|
|
47330
|
+
"Here is a bit of news: {clause}.",
|
|
47331
|
+
"You might find this useful later: {clause}.",
|
|
47332
|
+
"For context, {clause}.",
|
|
47333
|
+
"Small life update: {clause}.",
|
|
47334
|
+
"I keep meaning to tell you that {clause}.",
|
|
47335
|
+
"Worth remembering: {clause}.",
|
|
47336
|
+
"Adding this to the pile: {clause}.",
|
|
47337
|
+
"It finally happened: {clause}.",
|
|
47338
|
+
"Not sure I mentioned it, but {clause}.",
|
|
47339
|
+
"One more thing from this week: {clause}.",
|
|
47340
|
+
"File this away somewhere: {clause}.",
|
|
47341
|
+
"A little background on that front: {clause}.",
|
|
47342
|
+
"Just so you have the full picture, {clause}.",
|
|
47343
|
+
"Today I learned that {clause}.",
|
|
47344
|
+
"The latest around here is that {clause}."
|
|
47345
|
+
]);
|
|
47346
|
+
var UPDATE_FRAMES = Object.freeze([
|
|
47347
|
+
"Actually, an update: {clause} now, not {oldValue} anymore.",
|
|
47348
|
+
"Change of plans since we last talked: {clause}, moving on from {oldValue}.",
|
|
47349
|
+
"Correction to something I said before: {clause} these days, no longer {oldValue}.",
|
|
47350
|
+
"Heads up, things shifted: {clause}, which replaces {oldValue}.",
|
|
47351
|
+
"Scratch the old note about {oldValue}: {clause} now.",
|
|
47352
|
+
"That changed recently: {clause}, after a stretch with {oldValue}.",
|
|
47353
|
+
"New development: {clause}. The {oldValue} chapter is over.",
|
|
47354
|
+
"Since last month, {clause} \u2014 quite a switch from {oldValue}.",
|
|
47355
|
+
"Please update your notes: {clause}, superseding {oldValue}.",
|
|
47356
|
+
"Big change on that front: {clause} instead of {oldValue}.",
|
|
47357
|
+
"Turns out {clause} now; {oldValue} did not stick.",
|
|
47358
|
+
"As of this month, {clause}. Farewell to {oldValue}.",
|
|
47359
|
+
"I made the jump: {clause}, leaving {oldValue} behind.",
|
|
47360
|
+
"Things moved fast: {clause} now, after {oldValue}.",
|
|
47361
|
+
"Quick revision to the record: {clause}, formerly {oldValue}.",
|
|
47362
|
+
"Update from this side: {clause}. The {oldValue} era ended.",
|
|
47363
|
+
"It is official now: {clause}, replacing {oldValue}.",
|
|
47364
|
+
"Another shift to log: {clause}, whereas before it was {oldValue}.",
|
|
47365
|
+
"Latest news: {clause}, which is a change from {oldValue}.",
|
|
47366
|
+
"For accuracy going forward: {clause}, not {oldValue}.",
|
|
47367
|
+
"The situation evolved: {clause} as of now, previously {oldValue}.",
|
|
47368
|
+
"Mark this down: {clause}, taking over from {oldValue}."
|
|
47369
|
+
]);
|
|
47370
|
+
var ACK_LINES = Object.freeze([
|
|
47371
|
+
"Noted, thanks for the update.",
|
|
47372
|
+
"Got it, I will remember that.",
|
|
47373
|
+
"Thanks for letting me know.",
|
|
47374
|
+
"Understood, recorded.",
|
|
47375
|
+
"That is good to know.",
|
|
47376
|
+
"Appreciate the heads up.",
|
|
47377
|
+
"Noted \u2014 anything else changing?",
|
|
47378
|
+
"I have that down now.",
|
|
47379
|
+
"Thanks, updating my notes.",
|
|
47380
|
+
"Good to know, thanks for sharing.",
|
|
47381
|
+
"Recorded. How is everything else?",
|
|
47382
|
+
"Nice, thanks for the detail."
|
|
47383
|
+
]);
|
|
47384
|
+
var ELABORATION_LINES = Object.freeze([
|
|
47385
|
+
"It has been keeping things interesting, honestly.",
|
|
47386
|
+
"So far it feels like the right call.",
|
|
47387
|
+
"Still settling into it, but it is going well.",
|
|
47388
|
+
"Ask me again in a month how that is going.",
|
|
47389
|
+
"There is a longer story there for another day.",
|
|
47390
|
+
"It came together faster than expected.",
|
|
47391
|
+
"Everyone around here seems pleased about it.",
|
|
47392
|
+
"We will see how that holds up over time.",
|
|
47393
|
+
"It took a while, but it finally worked out.",
|
|
47394
|
+
"That one has been a long time coming.",
|
|
47395
|
+
"No regrets so far on that front.",
|
|
47396
|
+
"More details on that next time we talk."
|
|
47397
|
+
]);
|
|
47398
|
+
var CORPUS_START_YEAR = 2021;
|
|
47399
|
+
var CORPUS_START_MONTH = 3;
|
|
47400
|
+
function epochDate(epoch, dayOfMonth) {
|
|
47401
|
+
if (!Number.isSafeInteger(epoch) || epoch < 1) {
|
|
47402
|
+
throw new Error("epochDate epoch must be an integer >= 1");
|
|
47403
|
+
}
|
|
47404
|
+
if (!Number.isSafeInteger(dayOfMonth) || dayOfMonth < 1 || dayOfMonth > 28) {
|
|
47405
|
+
throw new Error("epochDate dayOfMonth must be an integer in [1, 28]");
|
|
47406
|
+
}
|
|
47407
|
+
const monthIndex = CORPUS_START_MONTH - 1 + (epoch - 1);
|
|
47408
|
+
const year = CORPUS_START_YEAR + Math.floor(monthIndex / 12);
|
|
47409
|
+
const month = monthIndex % 12 + 1;
|
|
47410
|
+
const mm = String(month).padStart(2, "0");
|
|
47411
|
+
const dd = String(dayOfMonth).padStart(2, "0");
|
|
47412
|
+
return `${year}-${mm}-${dd}`;
|
|
47413
|
+
}
|
|
47414
|
+
function renderClause(fact3, persona) {
|
|
47415
|
+
const spec = ATTRIBUTE_SPECS.find((s) => s.attribute === fact3.attribute);
|
|
47416
|
+
if (!spec) throw new Error(`unknown drift-gen attribute: ${fact3.attribute}`);
|
|
47417
|
+
return fact3.subject === persona ? `I ${spec.firstPersonClause(fact3.value)}` : `${fact3.subject} ${spec.clause(fact3.value)}`;
|
|
47418
|
+
}
|
|
47419
|
+
function renderFactTurns(rng, fact3, persona, supersedes) {
|
|
47420
|
+
const clause = renderClause(fact3, persona);
|
|
47421
|
+
const opening = supersedes ? pickOne(rng, UPDATE_FRAMES).replaceAll("{clause}", clause).replaceAll("{oldValue}", supersedes.value) : pickOne(rng, FRESH_FRAMES).replaceAll("{clause}", clause);
|
|
47422
|
+
const turns = [{ role: "user", content: opening }];
|
|
47423
|
+
const extra = randomInt(rng, 0, 2);
|
|
47424
|
+
if (extra >= 1) {
|
|
47425
|
+
turns.push({ role: "assistant", content: pickOne(rng, ACK_LINES) });
|
|
47426
|
+
}
|
|
47427
|
+
if (extra === 2) {
|
|
47428
|
+
turns.push({ role: "user", content: pickOne(rng, ELABORATION_LINES) });
|
|
47429
|
+
}
|
|
47430
|
+
return turns;
|
|
47431
|
+
}
|
|
47432
|
+
function renderUserSessions(rng, user, epochs) {
|
|
47433
|
+
const supersededBy = /* @__PURE__ */ new Map();
|
|
47434
|
+
for (const fact3 of user.facts) {
|
|
47435
|
+
if (fact3.supersededBy !== null) {
|
|
47436
|
+
const successor = user.facts.find((f) => f.id === fact3.supersededBy);
|
|
47437
|
+
if (successor) supersededBy.set(successor.id, fact3);
|
|
47438
|
+
}
|
|
47439
|
+
}
|
|
47440
|
+
const sessions = [];
|
|
47441
|
+
for (let epoch = 1; epoch <= epochs; epoch++) {
|
|
47442
|
+
const introduced = user.facts.filter((f) => f.introducedEpoch === epoch);
|
|
47443
|
+
const turns = [];
|
|
47444
|
+
for (const fact3 of introduced) {
|
|
47445
|
+
turns.push(...renderFactTurns(rng, fact3, user.persona, supersededBy.get(fact3.id)));
|
|
47446
|
+
}
|
|
47447
|
+
sessions.push({
|
|
47448
|
+
sessionId: `s-${user.userId}-e${epoch}`,
|
|
47449
|
+
userId: user.userId,
|
|
47450
|
+
epoch,
|
|
47451
|
+
date: epochDate(epoch, randomInt(rng, 2, 27)),
|
|
47452
|
+
turns
|
|
47453
|
+
});
|
|
47454
|
+
}
|
|
47455
|
+
return sessions;
|
|
47456
|
+
}
|
|
47457
|
+
|
|
47458
|
+
// src/generators/drift-gen/validate.ts
|
|
47459
|
+
import { createHash as createHash20 } from "crypto";
|
|
47460
|
+
import { lstat as lstat7, readFile as readFile26, readdir as readdir9 } from "fs/promises";
|
|
47461
|
+
import path41 from "path";
|
|
47462
|
+
var FACT_COUNT_TOLERANCE = 0.1;
|
|
47463
|
+
var RATIO_TOLERANCE = 0.05;
|
|
47464
|
+
var MAX_QUESTION_ANSWER_LEAKAGE = 0.6;
|
|
47465
|
+
var MIN_STATISTICAL_BASE = 40;
|
|
47466
|
+
var STOPWORDS2 = /* @__PURE__ */ new Set([
|
|
47467
|
+
"a",
|
|
47468
|
+
"an",
|
|
47469
|
+
"and",
|
|
47470
|
+
"as",
|
|
47471
|
+
"at",
|
|
47472
|
+
"before",
|
|
47473
|
+
"by",
|
|
47474
|
+
"did",
|
|
47475
|
+
"do",
|
|
47476
|
+
"does",
|
|
47477
|
+
"for",
|
|
47478
|
+
"from",
|
|
47479
|
+
"has",
|
|
47480
|
+
"have",
|
|
47481
|
+
"how",
|
|
47482
|
+
"in",
|
|
47483
|
+
"is",
|
|
47484
|
+
"it",
|
|
47485
|
+
"its",
|
|
47486
|
+
"most",
|
|
47487
|
+
"now",
|
|
47488
|
+
"of",
|
|
47489
|
+
"on",
|
|
47490
|
+
"one",
|
|
47491
|
+
"order",
|
|
47492
|
+
"recent",
|
|
47493
|
+
"s",
|
|
47494
|
+
"the",
|
|
47495
|
+
"these",
|
|
47496
|
+
"to",
|
|
47497
|
+
"was",
|
|
47498
|
+
"what",
|
|
47499
|
+
"when",
|
|
47500
|
+
"where",
|
|
47501
|
+
"which",
|
|
47502
|
+
"who",
|
|
47503
|
+
"with"
|
|
47504
|
+
]);
|
|
47505
|
+
function contentWords(text) {
|
|
47506
|
+
const words = /* @__PURE__ */ new Set();
|
|
47507
|
+
for (const raw of text.toLowerCase().split(/[^a-z0-9]+/)) {
|
|
47508
|
+
if (raw.length > 0 && !STOPWORDS2.has(raw)) words.add(raw);
|
|
47509
|
+
}
|
|
47510
|
+
return words;
|
|
47511
|
+
}
|
|
47512
|
+
function questionAnswerLeakage(question, answer) {
|
|
47513
|
+
const answerWords = contentWords(answer);
|
|
47514
|
+
if (answerWords.size === 0) return 0;
|
|
47515
|
+
const questionWords = contentWords(question);
|
|
47516
|
+
let overlap = 0;
|
|
47517
|
+
for (const word of answerWords) {
|
|
47518
|
+
if (questionWords.has(word)) overlap++;
|
|
47519
|
+
}
|
|
47520
|
+
return overlap / answerWords.size;
|
|
47521
|
+
}
|
|
47522
|
+
var FACT_KINDS = /* @__PURE__ */ new Set(["stable", "drifting", "contradicted"]);
|
|
47523
|
+
var PROBE_CATEGORIES2 = /* @__PURE__ */ new Set(["current", "historical", "transition", "aggregation"]);
|
|
47524
|
+
var SESSION_TURN_ROLES = /* @__PURE__ */ new Set(["user", "assistant"]);
|
|
47525
|
+
function isGoldFactShape(row) {
|
|
47526
|
+
if (typeof row !== "object" || row === null) return false;
|
|
47527
|
+
const f = row;
|
|
47528
|
+
return typeof f.id === "string" && typeof f.userId === "string" && typeof f.statement === "string" && typeof f.subject === "string" && typeof f.attribute === "string" && typeof f.value === "string" && Number.isSafeInteger(f.introducedEpoch) && (f.supersededEpoch === null || Number.isSafeInteger(f.supersededEpoch)) && (f.supersededBy === null || typeof f.supersededBy === "string") && typeof f.kind === "string" && FACT_KINDS.has(f.kind) && Array.isArray(f.probes);
|
|
47529
|
+
}
|
|
47530
|
+
function isGoldProbeShape(row) {
|
|
47531
|
+
if (typeof row !== "object" || row === null) return false;
|
|
47532
|
+
const p = row;
|
|
47533
|
+
return typeof p.id === "string" && typeof p.userId === "string" && Number.isSafeInteger(p.epoch) && typeof p.question === "string" && typeof p.expectedAnswer === "string" && Array.isArray(p.requiredFactIds) && p.requiredFactIds.every((id) => typeof id === "string") && typeof p.category === "string" && PROBE_CATEGORIES2.has(p.category);
|
|
47534
|
+
}
|
|
47535
|
+
function isDriftSessionShape(row) {
|
|
47536
|
+
if (typeof row !== "object" || row === null) return false;
|
|
47537
|
+
const s = row;
|
|
47538
|
+
return typeof s.sessionId === "string" && typeof s.userId === "string" && Number.isSafeInteger(s.epoch) && typeof s.date === "string" && Array.isArray(s.turns) && s.turns.every(
|
|
47539
|
+
(t) => typeof t === "object" && t !== null && typeof t.role === "string" && SESSION_TURN_ROLES.has(t.role) && typeof t.content === "string"
|
|
47540
|
+
);
|
|
47541
|
+
}
|
|
47542
|
+
async function readJsonl(filePath, errors, isShape) {
|
|
47543
|
+
let raw;
|
|
47544
|
+
try {
|
|
47545
|
+
if ((await lstat7(filePath)).isSymbolicLink()) {
|
|
47546
|
+
errors.push(`symlinked corpus file rejected: ${filePath}`);
|
|
47547
|
+
return [];
|
|
47548
|
+
}
|
|
47549
|
+
raw = await readFile26(filePath, "utf8");
|
|
47550
|
+
} catch {
|
|
47551
|
+
errors.push(`missing file: ${filePath}`);
|
|
47552
|
+
return [];
|
|
47553
|
+
}
|
|
47554
|
+
const rows = [];
|
|
47555
|
+
const lines = raw.split("\n");
|
|
47556
|
+
for (let i = 0; i < lines.length; i++) {
|
|
47557
|
+
const line = lines[i].trim();
|
|
47558
|
+
if (line.length === 0) continue;
|
|
47559
|
+
let parsed;
|
|
47560
|
+
try {
|
|
47561
|
+
parsed = JSON.parse(line);
|
|
47562
|
+
} catch {
|
|
47563
|
+
errors.push(`${filePath}:${i + 1}: invalid JSON line`);
|
|
47564
|
+
continue;
|
|
47565
|
+
}
|
|
47566
|
+
if (!isShape(parsed)) {
|
|
47567
|
+
errors.push(`${filePath}:${i + 1}: row does not match the expected record shape`);
|
|
47568
|
+
continue;
|
|
47569
|
+
}
|
|
47570
|
+
rows.push(parsed);
|
|
47571
|
+
}
|
|
47572
|
+
return rows;
|
|
47573
|
+
}
|
|
47574
|
+
async function isNonSymlinkDirectory(dirPath, errors) {
|
|
47575
|
+
try {
|
|
47576
|
+
const stats = await lstat7(dirPath);
|
|
47577
|
+
if (stats.isSymbolicLink()) {
|
|
47578
|
+
errors.push(`symlinked corpus directory rejected: ${dirPath}`);
|
|
47579
|
+
return false;
|
|
47580
|
+
}
|
|
47581
|
+
if (!stats.isDirectory()) {
|
|
47582
|
+
errors.push(`corpus path is not a directory: ${dirPath}`);
|
|
47583
|
+
return false;
|
|
47584
|
+
}
|
|
47585
|
+
return true;
|
|
47586
|
+
} catch {
|
|
47587
|
+
errors.push(`missing directory: ${dirPath}`);
|
|
47588
|
+
return false;
|
|
47589
|
+
}
|
|
47590
|
+
}
|
|
47591
|
+
async function hasNoSymlinkComponents(rootDir, targetPath, errors, description) {
|
|
47592
|
+
let current = rootDir;
|
|
47593
|
+
for (const part of path41.relative(rootDir, targetPath).split(path41.sep)) {
|
|
47594
|
+
if (part.length === 0 || part === ".") continue;
|
|
47595
|
+
current = path41.join(current, part);
|
|
47596
|
+
try {
|
|
47597
|
+
if ((await lstat7(current)).isSymbolicLink()) {
|
|
47598
|
+
errors.push(`${description} contains a symlinked path component: ${path41.relative(rootDir, current)}`);
|
|
47599
|
+
return false;
|
|
47600
|
+
}
|
|
47601
|
+
} catch {
|
|
47602
|
+
errors.push(`${description} is missing: ${path41.relative(rootDir, current)}`);
|
|
47603
|
+
return false;
|
|
47604
|
+
}
|
|
47605
|
+
}
|
|
47606
|
+
return true;
|
|
47607
|
+
}
|
|
47608
|
+
function corpusRelativePath(corpusDir, targetPath) {
|
|
47609
|
+
return path41.relative(corpusDir, targetPath).split(path41.sep).join("/");
|
|
47610
|
+
}
|
|
47611
|
+
async function loadSeedDir(corpusDir, seed, errors) {
|
|
47612
|
+
const seedDir = path41.join(corpusDir, String(seed));
|
|
47613
|
+
const empty = { seed, facts: [], probes: [], sessions: [], consumedFiles: [] };
|
|
47614
|
+
if (!await isNonSymlinkDirectory(seedDir, errors)) return empty;
|
|
47615
|
+
const goldDir = path41.join(seedDir, "gold");
|
|
47616
|
+
const factsPath = path41.join(goldDir, "facts.jsonl");
|
|
47617
|
+
const probesPath = path41.join(goldDir, "probes.jsonl");
|
|
47618
|
+
let facts = [];
|
|
47619
|
+
let probes = [];
|
|
47620
|
+
const consumedFiles = [];
|
|
47621
|
+
if (await isNonSymlinkDirectory(goldDir, errors)) {
|
|
47622
|
+
consumedFiles.push(
|
|
47623
|
+
corpusRelativePath(corpusDir, factsPath),
|
|
47624
|
+
corpusRelativePath(corpusDir, probesPath)
|
|
47625
|
+
);
|
|
47626
|
+
facts = await readJsonl(factsPath, errors, isGoldFactShape);
|
|
47627
|
+
probes = await readJsonl(probesPath, errors, isGoldProbeShape);
|
|
47628
|
+
}
|
|
47629
|
+
const sessions = [];
|
|
47630
|
+
const usersDir = path41.join(seedDir, "users");
|
|
47631
|
+
if (!await isNonSymlinkDirectory(usersDir, errors)) {
|
|
47632
|
+
return { seed, facts, probes, sessions, consumedFiles };
|
|
47633
|
+
}
|
|
47634
|
+
const userIds = [];
|
|
47635
|
+
try {
|
|
47636
|
+
const entries = await readdir9(usersDir, { withFileTypes: true });
|
|
47637
|
+
for (const entry of entries) {
|
|
47638
|
+
const userDir = path41.join(usersDir, entry.name);
|
|
47639
|
+
if (entry.isSymbolicLink()) {
|
|
47640
|
+
errors.push(`symlinked corpus entry rejected: ${userDir}`);
|
|
47641
|
+
continue;
|
|
47642
|
+
}
|
|
47643
|
+
if (entry.isDirectory()) userIds.push(entry.name);
|
|
47644
|
+
}
|
|
47645
|
+
} catch {
|
|
47646
|
+
errors.push(`missing directory: ${usersDir}`);
|
|
47647
|
+
return { seed, facts, probes, sessions, consumedFiles };
|
|
47648
|
+
}
|
|
47649
|
+
for (const userId of userIds.sort()) {
|
|
47650
|
+
const userDir = path41.join(usersDir, userId);
|
|
47651
|
+
const sessionsPath = path41.join(userDir, "sessions.jsonl");
|
|
47652
|
+
consumedFiles.push(corpusRelativePath(corpusDir, sessionsPath));
|
|
47653
|
+
for (const session of await readJsonl(sessionsPath, errors, isDriftSessionShape)) {
|
|
47654
|
+
if (session.userId !== userId) {
|
|
47655
|
+
errors.push(`${session.sessionId}: userId ${session.userId} does not match directory ${userId}`);
|
|
47656
|
+
continue;
|
|
47657
|
+
}
|
|
47658
|
+
sessions.push(session);
|
|
47659
|
+
}
|
|
47660
|
+
}
|
|
47661
|
+
return { seed, facts, probes, sessions, consumedFiles };
|
|
47662
|
+
}
|
|
47663
|
+
function checkFactIntegrity(loaded, epochs, errors) {
|
|
47664
|
+
const byId = new Map(loaded.facts.map((f) => [f.id, f]));
|
|
47665
|
+
if (byId.size !== loaded.facts.length) {
|
|
47666
|
+
errors.push(`seed ${loaded.seed}: duplicate fact ids`);
|
|
47667
|
+
}
|
|
47668
|
+
for (const fact3 of loaded.facts) {
|
|
47669
|
+
try {
|
|
47670
|
+
if (fact3.statement !== formatFactStatement(fact3.subject, fact3.attribute, fact3.value)) {
|
|
47671
|
+
errors.push(`${fact3.id}: statement does not match subject, attribute, and value`);
|
|
47672
|
+
}
|
|
47673
|
+
} catch {
|
|
47674
|
+
errors.push(`${fact3.id}: attribute is not recognized`);
|
|
47675
|
+
}
|
|
47676
|
+
if (fact3.introducedEpoch < 1 || fact3.introducedEpoch > epochs) {
|
|
47677
|
+
errors.push(`${fact3.id}: introducedEpoch ${fact3.introducedEpoch} out of range 1..${epochs}`);
|
|
47678
|
+
}
|
|
47679
|
+
if (fact3.supersededBy === null !== (fact3.supersededEpoch === null)) {
|
|
47680
|
+
errors.push(`${fact3.id}: supersededBy and supersededEpoch must be set together`);
|
|
47681
|
+
}
|
|
47682
|
+
const realizedKind = fact3.supersededEpoch === null ? "stable" : fact3.supersededEpoch === fact3.introducedEpoch + 1 ? "contradicted" : "drifting";
|
|
47683
|
+
if (fact3.kind !== realizedKind) {
|
|
47684
|
+
errors.push(`${fact3.id}: kind "${fact3.kind}" does not match realized lifecycle "${realizedKind}"`);
|
|
47685
|
+
}
|
|
47686
|
+
if (fact3.supersededBy !== null && fact3.supersededEpoch !== null) {
|
|
47687
|
+
const successor = byId.get(fact3.supersededBy);
|
|
47688
|
+
if (!successor) {
|
|
47689
|
+
errors.push(`${fact3.id}: supersededBy ${fact3.supersededBy} does not exist`);
|
|
47690
|
+
continue;
|
|
47691
|
+
}
|
|
47692
|
+
if (successor.introducedEpoch <= fact3.introducedEpoch) {
|
|
47693
|
+
errors.push(`${fact3.id}: successor ${successor.id} must be introduced at a later epoch`);
|
|
47694
|
+
}
|
|
47695
|
+
if (successor.introducedEpoch !== fact3.supersededEpoch) {
|
|
47696
|
+
errors.push(`${fact3.id}: supersededEpoch ${fact3.supersededEpoch} does not match successor introduction ${successor.introducedEpoch}`);
|
|
47697
|
+
}
|
|
47698
|
+
if (successor.subject !== fact3.subject || successor.attribute !== fact3.attribute) {
|
|
47699
|
+
errors.push(`${fact3.id}: successor ${successor.id} targets a different subject/attribute`);
|
|
47700
|
+
}
|
|
47701
|
+
if (successor.userId !== fact3.userId) {
|
|
47702
|
+
errors.push(`${fact3.id}: successor ${successor.id} belongs to a different user`);
|
|
47703
|
+
}
|
|
47704
|
+
if (successor.value === fact3.value) {
|
|
47705
|
+
errors.push(`${fact3.id}: successor ${successor.id} repeats the same value`);
|
|
47706
|
+
}
|
|
47707
|
+
}
|
|
47708
|
+
}
|
|
47709
|
+
const factsBySlot = /* @__PURE__ */ new Map();
|
|
47710
|
+
for (const fact3 of loaded.facts) {
|
|
47711
|
+
const slot = `${fact3.userId}\0${fact3.subject}\0${fact3.attribute}`;
|
|
47712
|
+
const facts = factsBySlot.get(slot) ?? [];
|
|
47713
|
+
facts.push(fact3);
|
|
47714
|
+
factsBySlot.set(slot, facts);
|
|
47715
|
+
}
|
|
47716
|
+
for (const facts of factsBySlot.values()) {
|
|
47717
|
+
facts.sort((a, b) => a.introducedEpoch - b.introducedEpoch || a.id.localeCompare(b.id));
|
|
47718
|
+
for (let index = 1; index < facts.length; index++) {
|
|
47719
|
+
const previous = facts[index - 1];
|
|
47720
|
+
const current = facts[index];
|
|
47721
|
+
if (previous.supersededEpoch === null || previous.supersededEpoch > current.introducedEpoch) {
|
|
47722
|
+
errors.push(`${current.id}: overlaps active lifecycle for ${previous.id}`);
|
|
47723
|
+
}
|
|
47724
|
+
}
|
|
47725
|
+
}
|
|
47726
|
+
}
|
|
47727
|
+
function expectedProbeAnswer(probe, facts) {
|
|
47728
|
+
switch (probe.category) {
|
|
47729
|
+
case "current":
|
|
47730
|
+
case "historical":
|
|
47731
|
+
return facts.length === 1 ? facts[0].value : null;
|
|
47732
|
+
case "transition":
|
|
47733
|
+
return facts.length === 2 ? `from ${facts[0].value} to ${facts[1].value}` : null;
|
|
47734
|
+
case "aggregation":
|
|
47735
|
+
return facts.map((fact3) => fact3.value).join("; ");
|
|
47736
|
+
}
|
|
47737
|
+
}
|
|
47738
|
+
function checkProbeIntegrity(loaded, epochs, errors) {
|
|
47739
|
+
const byId = new Map(loaded.facts.map((f) => [f.id, f]));
|
|
47740
|
+
const seenProbeIds = /* @__PURE__ */ new Set();
|
|
47741
|
+
for (const probe of loaded.probes) {
|
|
47742
|
+
if (seenProbeIds.has(probe.id)) {
|
|
47743
|
+
errors.push(`${probe.id}: duplicate probe id`);
|
|
47744
|
+
}
|
|
47745
|
+
seenProbeIds.add(probe.id);
|
|
47746
|
+
if (probe.epoch < 1 || probe.epoch > epochs) {
|
|
47747
|
+
errors.push(`${probe.id}: epoch ${probe.epoch} out of range 1..${epochs}`);
|
|
47748
|
+
}
|
|
47749
|
+
if (probe.requiredFactIds.length === 0) {
|
|
47750
|
+
errors.push(`${probe.id}: requiredFactIds is empty`);
|
|
47751
|
+
}
|
|
47752
|
+
for (const factId of probe.requiredFactIds) {
|
|
47753
|
+
const fact3 = byId.get(factId);
|
|
47754
|
+
if (!fact3) {
|
|
47755
|
+
errors.push(`${probe.id}: requiredFactId ${factId} does not exist`);
|
|
47756
|
+
continue;
|
|
47757
|
+
}
|
|
47758
|
+
if (fact3.introducedEpoch > probe.epoch) {
|
|
47759
|
+
errors.push(`${probe.id}: fact ${factId} is introduced at epoch ${fact3.introducedEpoch}, after the probe epoch ${probe.epoch}`);
|
|
47760
|
+
}
|
|
47761
|
+
if (probe.category === "aggregation" && fact3.supersededEpoch !== null && fact3.supersededEpoch <= probe.epoch) {
|
|
47762
|
+
errors.push(`${probe.id}: aggregation probe targets fact ${factId} already superseded at epoch ${fact3.supersededEpoch}`);
|
|
47763
|
+
}
|
|
47764
|
+
}
|
|
47765
|
+
const requiredFactCount = probe.category === "transition" ? 2 : probe.category === "aggregation" ? null : 1;
|
|
47766
|
+
if (requiredFactCount !== null && probe.requiredFactIds.length !== requiredFactCount) {
|
|
47767
|
+
errors.push(
|
|
47768
|
+
`${probe.id}: ${probe.category} probe must require exactly ${requiredFactCount} fact${requiredFactCount === 1 ? "" : "s"}`
|
|
47769
|
+
);
|
|
47770
|
+
}
|
|
47771
|
+
const requiredFacts = probe.requiredFactIds.map((factId) => byId.get(factId));
|
|
47772
|
+
if (requiredFacts.every((fact3) => fact3 !== void 0)) {
|
|
47773
|
+
for (const fact3 of requiredFacts) {
|
|
47774
|
+
if (fact3.userId !== probe.userId) {
|
|
47775
|
+
errors.push(`${probe.id}: required fact ${fact3.id} belongs to user ${fact3.userId}, not ${probe.userId}`);
|
|
47776
|
+
}
|
|
47777
|
+
}
|
|
47778
|
+
const expectedAnswer = expectedProbeAnswer(probe, requiredFacts);
|
|
47779
|
+
if (expectedAnswer !== null && probe.expectedAnswer !== expectedAnswer) {
|
|
47780
|
+
errors.push(`${probe.id}: expectedAnswer does not match the referenced facts`);
|
|
47781
|
+
}
|
|
47782
|
+
if (probe.category === "transition" && requiredFacts.length === 2 && requiredFacts[0].supersededBy !== requiredFacts[1].id) {
|
|
47783
|
+
errors.push(`${probe.id}: transition probe facts are not linked by supersession`);
|
|
47784
|
+
}
|
|
47785
|
+
if (probe.category === "transition" && requiredFacts.length === 2 && requiredFacts[1].supersededEpoch !== null && requiredFacts[1].supersededEpoch <= probe.epoch) {
|
|
47786
|
+
errors.push(`${probe.id}: transition probe targets successor ${requiredFacts[1].id} already superseded at epoch ${requiredFacts[1].supersededEpoch}`);
|
|
47787
|
+
}
|
|
47788
|
+
}
|
|
47789
|
+
if (probe.category === "current") {
|
|
47790
|
+
const fact3 = byId.get(probe.requiredFactIds[0]);
|
|
47791
|
+
if (fact3 && fact3.supersededEpoch !== null && fact3.supersededEpoch <= probe.epoch) {
|
|
47792
|
+
errors.push(`${probe.id}: current probe targets fact ${fact3.id} already superseded at epoch ${fact3.supersededEpoch}`);
|
|
47793
|
+
}
|
|
47794
|
+
}
|
|
47795
|
+
if (probe.category === "historical") {
|
|
47796
|
+
const fact3 = byId.get(probe.requiredFactIds[0]);
|
|
47797
|
+
if (fact3 && (fact3.supersededEpoch === null || fact3.supersededEpoch > probe.epoch)) {
|
|
47798
|
+
errors.push(`${probe.id}: historical probe targets fact ${fact3.id} not superseded by epoch ${probe.epoch}`);
|
|
47799
|
+
}
|
|
47800
|
+
const successor = fact3?.supersededBy ? byId.get(fact3.supersededBy) : void 0;
|
|
47801
|
+
if (successor?.supersededEpoch !== null && successor?.supersededEpoch !== void 0 && successor.supersededEpoch <= probe.epoch) {
|
|
47802
|
+
errors.push(`${probe.id}: historical probe targets stale successor ${successor.id} superseded at epoch ${successor.supersededEpoch}`);
|
|
47803
|
+
}
|
|
47804
|
+
}
|
|
47805
|
+
if (probe.category === "aggregation") {
|
|
47806
|
+
if (probe.requiredFactIds.length < 3 || probe.requiredFactIds.length > 6) {
|
|
47807
|
+
errors.push(`${probe.id}: aggregation probe must require 3-6 facts, has ${probe.requiredFactIds.length}`);
|
|
47808
|
+
}
|
|
47809
|
+
}
|
|
47810
|
+
const leakage = questionAnswerLeakage(probe.question, probe.expectedAnswer);
|
|
47811
|
+
if (leakage > MAX_QUESTION_ANSWER_LEAKAGE) {
|
|
47812
|
+
errors.push(`${probe.id}: question leaks ${(leakage * 100).toFixed(0)}% of answer content words (max ${MAX_QUESTION_ANSWER_LEAKAGE * 100}%)`);
|
|
47813
|
+
}
|
|
47814
|
+
}
|
|
47815
|
+
}
|
|
47816
|
+
function checkEmbeddedFactProbes(loaded, errors) {
|
|
47817
|
+
const canonicalById = new Map(loaded.probes.map((probe) => [probe.id, probe]));
|
|
47818
|
+
for (const fact3 of loaded.facts) {
|
|
47819
|
+
for (const probe of fact3.probes) {
|
|
47820
|
+
if (!isGoldProbeShape(probe)) {
|
|
47821
|
+
errors.push(`${fact3.id}: embedded probe does not match the expected record shape`);
|
|
47822
|
+
continue;
|
|
47823
|
+
}
|
|
47824
|
+
const canonical = canonicalById.get(probe.id);
|
|
47825
|
+
if (!canonical || JSON.stringify(canonical) !== JSON.stringify(probe)) {
|
|
47826
|
+
errors.push(`${fact3.id}: embedded probe ${probe.id} does not match gold/probes.jsonl`);
|
|
47827
|
+
} else if (!canonical.requiredFactIds.includes(fact3.id)) {
|
|
47828
|
+
errors.push(`${fact3.id}: embedded probe ${probe.id} does not reference its owning fact`);
|
|
47829
|
+
}
|
|
47830
|
+
}
|
|
47831
|
+
}
|
|
47832
|
+
}
|
|
47833
|
+
function checkSessions(loaded, users, epochs, errors) {
|
|
47834
|
+
const sessionText = /* @__PURE__ */ new Map();
|
|
47835
|
+
const sessionIds = /* @__PURE__ */ new Set();
|
|
47836
|
+
for (const session of loaded.sessions) {
|
|
47837
|
+
if (sessionIds.has(session.sessionId)) {
|
|
47838
|
+
errors.push(`seed ${loaded.seed}: duplicate sessionId ${session.sessionId}`);
|
|
47839
|
+
}
|
|
47840
|
+
sessionIds.add(session.sessionId);
|
|
47841
|
+
if (session.epoch < 1 || session.epoch > epochs) {
|
|
47842
|
+
errors.push(`${session.sessionId}: epoch ${session.epoch} out of range`);
|
|
47843
|
+
} else {
|
|
47844
|
+
const dateMatch = /^(\d{4})-(\d{2})-(\d{2})$/.exec(session.date);
|
|
47845
|
+
const day = dateMatch ? Number(dateMatch[3]) : 0;
|
|
47846
|
+
if (!dateMatch || day < 1 || day > 28 || session.date !== epochDate(session.epoch, day)) {
|
|
47847
|
+
errors.push(`${session.sessionId}: date must be a valid canonical date in epoch ${session.epoch}`);
|
|
47848
|
+
}
|
|
47849
|
+
}
|
|
47850
|
+
const key = `${session.userId}|${session.epoch}`;
|
|
47851
|
+
if (sessionText.has(key)) {
|
|
47852
|
+
errors.push(`${session.sessionId}: duplicate session for ${key}`);
|
|
47853
|
+
continue;
|
|
47854
|
+
}
|
|
47855
|
+
sessionText.set(key, session.turns.map((t) => t.content).join("\n").toLowerCase());
|
|
47856
|
+
}
|
|
47857
|
+
if (sessionText.size !== users * epochs) {
|
|
47858
|
+
errors.push(
|
|
47859
|
+
`seed ${loaded.seed}: expected ${users * epochs} unique user/epoch sessions, found ${sessionText.size}`
|
|
47860
|
+
);
|
|
47861
|
+
}
|
|
47862
|
+
for (const fact3 of loaded.facts) {
|
|
47863
|
+
const text = sessionText.get(`${fact3.userId}|${fact3.introducedEpoch}`);
|
|
47864
|
+
if (text === void 0) {
|
|
47865
|
+
errors.push(`${fact3.id}: no session found for ${fact3.userId} epoch ${fact3.introducedEpoch}`);
|
|
47866
|
+
continue;
|
|
47867
|
+
}
|
|
47868
|
+
if (!text.includes(fact3.value.toLowerCase())) {
|
|
47869
|
+
errors.push(`${fact3.id}: introducing session never states the value "${fact3.value}"`);
|
|
47870
|
+
}
|
|
47871
|
+
}
|
|
47872
|
+
}
|
|
47873
|
+
function checkDistribution(loaded, manifest, errors, warnings) {
|
|
47874
|
+
const { epochs } = manifest.counts;
|
|
47875
|
+
const target = manifest.generator.factsPerEpoch;
|
|
47876
|
+
const perUserEpoch = /* @__PURE__ */ new Map();
|
|
47877
|
+
for (const fact3 of loaded.facts) {
|
|
47878
|
+
const key = `${fact3.userId}|${fact3.introducedEpoch}`;
|
|
47879
|
+
perUserEpoch.set(key, (perUserEpoch.get(key) ?? 0) + 1);
|
|
47880
|
+
}
|
|
47881
|
+
const userIds = new Set(loaded.sessions.map((session) => session.userId));
|
|
47882
|
+
if (userIds.size !== manifest.counts.users) {
|
|
47883
|
+
errors.push(`seed ${loaded.seed}: expected ${manifest.counts.users} users, found ${userIds.size}`);
|
|
47884
|
+
}
|
|
47885
|
+
for (const userId of [...userIds].sort()) {
|
|
47886
|
+
for (let epoch = 1; epoch <= epochs; epoch++) {
|
|
47887
|
+
const key = `${userId}|${epoch}`;
|
|
47888
|
+
const count = perUserEpoch.get(key) ?? 0;
|
|
47889
|
+
if (Math.abs(count - target) > target * FACT_COUNT_TOLERANCE) {
|
|
47890
|
+
errors.push(`seed ${loaded.seed}: ${userId} epoch ${epoch} introduces ${count} facts, outside \xB110% of target ${target}`);
|
|
47891
|
+
}
|
|
47892
|
+
}
|
|
47893
|
+
}
|
|
47894
|
+
const eligible = loaded.facts.filter(
|
|
47895
|
+
(f) => f.introducedEpoch + MIN_DRIFT_GAP <= epochs
|
|
47896
|
+
);
|
|
47897
|
+
const drifting = eligible.filter((f) => f.kind === "drifting").length;
|
|
47898
|
+
const contradicted = eligible.filter((f) => f.kind === "contradicted").length;
|
|
47899
|
+
const checks = [
|
|
47900
|
+
["drifting", drifting, manifest.generator.driftingRatio],
|
|
47901
|
+
["contradicted", contradicted, manifest.generator.contradictedRatio]
|
|
47902
|
+
];
|
|
47903
|
+
for (const [label, count, expected] of checks) {
|
|
47904
|
+
if (eligible.length === 0) continue;
|
|
47905
|
+
const measured = count / eligible.length;
|
|
47906
|
+
const delta2 = Math.abs(measured - expected);
|
|
47907
|
+
const tolerance = Math.max(
|
|
47908
|
+
RATIO_TOLERANCE,
|
|
47909
|
+
3 * Math.sqrt(expected * (1 - expected) / eligible.length)
|
|
47910
|
+
);
|
|
47911
|
+
if (delta2 <= tolerance) continue;
|
|
47912
|
+
const message = `seed ${loaded.seed}: ${label} ratio ${measured.toFixed(3)} deviates from ${expected} by more than ${tolerance.toFixed(3)} (eligible base ${eligible.length})`;
|
|
47913
|
+
if (eligible.length < MIN_STATISTICAL_BASE) {
|
|
47914
|
+
warnings.push(`${message} \u2014 base too small, reported as warning`);
|
|
47915
|
+
} else {
|
|
47916
|
+
errors.push(message);
|
|
47917
|
+
}
|
|
47918
|
+
}
|
|
47919
|
+
}
|
|
47920
|
+
function isIntegerAtLeast(value, minimum) {
|
|
47921
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= minimum;
|
|
47922
|
+
}
|
|
47923
|
+
function hasValidDriftRatios(driftingRatio, contradictedRatio) {
|
|
47924
|
+
return typeof driftingRatio === "number" && Number.isFinite(driftingRatio) && driftingRatio >= 0 && driftingRatio <= 1 && typeof contradictedRatio === "number" && Number.isFinite(contradictedRatio) && contradictedRatio >= 0 && contradictedRatio <= 1 && driftingRatio + contradictedRatio <= 1;
|
|
47925
|
+
}
|
|
47926
|
+
function isManifestShape(value) {
|
|
47927
|
+
if (typeof value !== "object" || value === null) return false;
|
|
47928
|
+
const m = value;
|
|
47929
|
+
const counts2 = m.counts;
|
|
47930
|
+
const generator = m.generator;
|
|
47931
|
+
return typeof m.name === "string" && typeof m.version === "string" && typeof m.generatorVersion === "string" && typeof m.createdAt === "string" && Array.isArray(m.licenses) && m.licenses.length > 0 && m.licenses.every(
|
|
47932
|
+
(license) => typeof license === "object" && license !== null && typeof license.source === "string" && typeof license.license === "string"
|
|
47933
|
+
) && Array.isArray(m.seeds) && m.seeds.length > 0 && m.seeds.every((s) => Number.isSafeInteger(s) && s >= 0) && new Set(m.seeds).size === m.seeds.length && typeof counts2 === "object" && counts2 !== null && !Array.isArray(counts2) && isIntegerAtLeast(counts2.users, 1) && isIntegerAtLeast(counts2.epochs, 2) && isIntegerAtLeast(counts2.facts, 1) && isIntegerAtLeast(counts2.probes, 1) && typeof generator === "object" && generator !== null && !Array.isArray(generator) && isIntegerAtLeast(generator.factsPerEpoch, 1) && hasValidDriftRatios(generator.driftingRatio, generator.contradictedRatio) && typeof m.files === "object" && m.files !== null && !Array.isArray(m.files) && Object.entries(m.files).every(
|
|
47934
|
+
([k, v]) => typeof k === "string" && typeof v === "string"
|
|
47935
|
+
);
|
|
47936
|
+
}
|
|
47937
|
+
async function checkFileHashes(corpusDir, manifest, errors) {
|
|
47938
|
+
const resolvedRoot = path41.resolve(corpusDir);
|
|
47939
|
+
for (const [relPath, expected] of Object.entries(manifest.files)) {
|
|
47940
|
+
const absPath = path41.resolve(corpusDir, relPath);
|
|
47941
|
+
if (absPath !== resolvedRoot && !absPath.startsWith(resolvedRoot + path41.sep)) {
|
|
47942
|
+
errors.push(`manifest lists a path outside the corpus root: ${relPath}`);
|
|
47943
|
+
continue;
|
|
47944
|
+
}
|
|
47945
|
+
if (!await hasNoSymlinkComponents(resolvedRoot, absPath, errors, `manifest path ${relPath}`)) {
|
|
47946
|
+
continue;
|
|
47947
|
+
}
|
|
47948
|
+
let data;
|
|
47949
|
+
try {
|
|
47950
|
+
data = await readFile26(absPath);
|
|
47951
|
+
} catch {
|
|
47952
|
+
errors.push(`manifest lists missing file: ${relPath}`);
|
|
47953
|
+
continue;
|
|
47954
|
+
}
|
|
47955
|
+
const actual = createHash20("sha256").update(data).digest("hex");
|
|
47956
|
+
if (actual !== expected) {
|
|
47957
|
+
errors.push(`sha256 mismatch for ${relPath}: manifest ${expected}, actual ${actual}`);
|
|
47958
|
+
}
|
|
47959
|
+
}
|
|
47960
|
+
}
|
|
47961
|
+
function checkConsumedFilesAreHashed(loaded, manifest, errors) {
|
|
47962
|
+
for (const relPath of loaded.consumedFiles) {
|
|
47963
|
+
if (!Object.prototype.hasOwnProperty.call(manifest.files, relPath)) {
|
|
47964
|
+
errors.push(`manifest is missing a sha256 entry for consumed corpus file: ${relPath}`);
|
|
47965
|
+
}
|
|
47966
|
+
}
|
|
47967
|
+
}
|
|
47968
|
+
async function validateDriftCorpus(corpusDir) {
|
|
47969
|
+
const errors = [];
|
|
47970
|
+
const warnings = [];
|
|
47971
|
+
const emptyStats = {
|
|
47972
|
+
users: 0,
|
|
47973
|
+
epochs: 0,
|
|
47974
|
+
facts: 0,
|
|
47975
|
+
probes: 0,
|
|
47976
|
+
sessions: 0,
|
|
47977
|
+
factsPerEpochMean: 0,
|
|
47978
|
+
driftingRatio: 0,
|
|
47979
|
+
contradictedRatio: 0,
|
|
47980
|
+
probesByCategory: { current: 0, historical: 0, transition: 0, aggregation: 0 },
|
|
47981
|
+
maxQuestionAnswerLeakage: 0
|
|
47982
|
+
};
|
|
47983
|
+
try {
|
|
47984
|
+
const rootStat = await lstat7(corpusDir);
|
|
47985
|
+
if (rootStat.isSymbolicLink()) {
|
|
47986
|
+
return {
|
|
47987
|
+
ok: false,
|
|
47988
|
+
errors: [`corpus root must not be a symlink: ${corpusDir}`],
|
|
47989
|
+
warnings,
|
|
47990
|
+
stats: emptyStats
|
|
47991
|
+
};
|
|
47992
|
+
}
|
|
47993
|
+
if (!rootStat.isDirectory()) {
|
|
47994
|
+
return { ok: false, errors: [`not a directory: ${corpusDir}`], warnings, stats: emptyStats };
|
|
47995
|
+
}
|
|
47996
|
+
} catch {
|
|
47997
|
+
return { ok: false, errors: [`corpus directory not found: ${corpusDir}`], warnings, stats: emptyStats };
|
|
47998
|
+
}
|
|
47999
|
+
const manifestPath = path41.join(corpusDir, "dataset.manifest.json");
|
|
48000
|
+
if (!await hasNoSymlinkComponents(corpusDir, manifestPath, errors, "dataset manifest")) {
|
|
48001
|
+
return { ok: false, errors, warnings, stats: emptyStats };
|
|
48002
|
+
}
|
|
48003
|
+
let manifestRaw;
|
|
48004
|
+
try {
|
|
48005
|
+
manifestRaw = JSON.parse(await readFile26(manifestPath, "utf8"));
|
|
48006
|
+
} catch {
|
|
48007
|
+
return {
|
|
48008
|
+
ok: false,
|
|
48009
|
+
errors: [`missing or invalid dataset.manifest.json in ${corpusDir}`],
|
|
48010
|
+
warnings,
|
|
48011
|
+
stats: emptyStats
|
|
48012
|
+
};
|
|
48013
|
+
}
|
|
48014
|
+
if (!isManifestShape(manifestRaw)) {
|
|
48015
|
+
return {
|
|
48016
|
+
ok: false,
|
|
48017
|
+
errors: ["dataset.manifest.json does not match the expected manifest shape (name, version, seeds, counts, generator, files)"],
|
|
48018
|
+
warnings,
|
|
48019
|
+
stats: emptyStats
|
|
48020
|
+
};
|
|
48021
|
+
}
|
|
48022
|
+
const manifest = manifestRaw;
|
|
48023
|
+
await checkFileHashes(corpusDir, manifest, errors);
|
|
48024
|
+
let totalFacts = 0;
|
|
48025
|
+
let totalProbes = 0;
|
|
48026
|
+
let totalSessions = 0;
|
|
48027
|
+
let driftingCount = 0;
|
|
48028
|
+
let contradictedCount = 0;
|
|
48029
|
+
let maxLeakage = 0;
|
|
48030
|
+
const probesByCategory = {
|
|
48031
|
+
current: 0,
|
|
48032
|
+
historical: 0,
|
|
48033
|
+
transition: 0,
|
|
48034
|
+
aggregation: 0
|
|
48035
|
+
};
|
|
48036
|
+
for (const seed of manifest.seeds) {
|
|
48037
|
+
const loaded = await loadSeedDir(corpusDir, seed, errors);
|
|
48038
|
+
checkConsumedFilesAreHashed(loaded, manifest, errors);
|
|
48039
|
+
checkEmbeddedFactProbes(loaded, errors);
|
|
48040
|
+
checkFactIntegrity(loaded, manifest.counts.epochs, errors);
|
|
48041
|
+
checkProbeIntegrity(loaded, manifest.counts.epochs, errors);
|
|
48042
|
+
checkSessions(loaded, manifest.counts.users, manifest.counts.epochs, errors);
|
|
48043
|
+
checkDistribution(loaded, manifest, errors, warnings);
|
|
48044
|
+
totalFacts += loaded.facts.length;
|
|
48045
|
+
totalProbes += loaded.probes.length;
|
|
48046
|
+
totalSessions += loaded.sessions.length;
|
|
48047
|
+
for (const fact3 of loaded.facts) {
|
|
48048
|
+
if (fact3.kind === "drifting") driftingCount++;
|
|
48049
|
+
if (fact3.kind === "contradicted") contradictedCount++;
|
|
48050
|
+
}
|
|
48051
|
+
for (const probe of loaded.probes) {
|
|
48052
|
+
if (probe.category in probesByCategory) probesByCategory[probe.category]++;
|
|
48053
|
+
maxLeakage = Math.max(
|
|
48054
|
+
maxLeakage,
|
|
48055
|
+
questionAnswerLeakage(probe.question, probe.expectedAnswer)
|
|
48056
|
+
);
|
|
48057
|
+
}
|
|
48058
|
+
}
|
|
48059
|
+
if (totalFacts !== manifest.counts.facts) {
|
|
48060
|
+
errors.push(`manifest counts.facts ${manifest.counts.facts} does not match corpus ${totalFacts}`);
|
|
48061
|
+
}
|
|
48062
|
+
if (totalProbes !== manifest.counts.probes) {
|
|
48063
|
+
errors.push(`manifest counts.probes ${manifest.counts.probes} does not match corpus ${totalProbes}`);
|
|
48064
|
+
}
|
|
48065
|
+
const denominator = manifest.seeds.length * manifest.counts.users * manifest.counts.epochs;
|
|
48066
|
+
const stats = {
|
|
48067
|
+
users: manifest.counts.users,
|
|
48068
|
+
epochs: manifest.counts.epochs,
|
|
48069
|
+
facts: totalFacts,
|
|
48070
|
+
probes: totalProbes,
|
|
48071
|
+
sessions: totalSessions,
|
|
48072
|
+
factsPerEpochMean: denominator === 0 ? 0 : totalFacts / denominator,
|
|
48073
|
+
driftingRatio: totalFacts === 0 ? 0 : driftingCount / totalFacts,
|
|
48074
|
+
contradictedRatio: totalFacts === 0 ? 0 : contradictedCount / totalFacts,
|
|
48075
|
+
probesByCategory,
|
|
48076
|
+
maxQuestionAnswerLeakage: maxLeakage
|
|
48077
|
+
};
|
|
48078
|
+
errors.sort();
|
|
48079
|
+
warnings.sort();
|
|
48080
|
+
return { ok: errors.length === 0, errors, warnings, stats };
|
|
48081
|
+
}
|
|
48082
|
+
|
|
48083
|
+
// src/generators/drift-gen/index.ts
|
|
48084
|
+
var DRIFT_GEN_VERSION = "1.0.0";
|
|
48085
|
+
var MANIFEST_TIMESTAMP = "1970-01-01T00:00:00.000Z";
|
|
48086
|
+
var DRIFT_GEN_DEFAULTS = Object.freeze({
|
|
48087
|
+
users: 5,
|
|
48088
|
+
epochs: 12,
|
|
48089
|
+
seed: 11,
|
|
48090
|
+
factsPerEpoch: 8,
|
|
48091
|
+
driftingRatio: 0.2,
|
|
48092
|
+
contradictedRatio: 0.1
|
|
48093
|
+
});
|
|
48094
|
+
function buildDriftCorpus(options) {
|
|
48095
|
+
const factsPerEpoch = options.factsPerEpoch ?? DRIFT_GEN_DEFAULTS.factsPerEpoch;
|
|
48096
|
+
const driftingRatio = options.driftingRatio ?? DRIFT_GEN_DEFAULTS.driftingRatio;
|
|
48097
|
+
const contradictedRatio = options.contradictedRatio ?? DRIFT_GEN_DEFAULTS.contradictedRatio;
|
|
48098
|
+
const schedule = buildCorpusSchedule({
|
|
48099
|
+
users: options.users,
|
|
48100
|
+
epochs: options.epochs,
|
|
48101
|
+
seed: options.seed,
|
|
48102
|
+
factsPerEpoch,
|
|
48103
|
+
driftingRatio,
|
|
48104
|
+
contradictedRatio
|
|
48105
|
+
});
|
|
48106
|
+
const renderRng = createSeededRandom2((options.seed ^ 6240089) >>> 0);
|
|
48107
|
+
const sessions = [];
|
|
48108
|
+
for (const user of schedule.users) {
|
|
48109
|
+
sessions.push(...renderUserSessions(renderRng, user, options.epochs));
|
|
48110
|
+
}
|
|
48111
|
+
return { facts: schedule.facts, probes: schedule.probes, sessions };
|
|
48112
|
+
}
|
|
48113
|
+
function toJsonl(rows) {
|
|
48114
|
+
return `${rows.map((row) => JSON.stringify(row)).join("\n")}
|
|
48115
|
+
`;
|
|
48116
|
+
}
|
|
48117
|
+
async function generateDriftCorpus(options) {
|
|
48118
|
+
const corpus = buildDriftCorpus(options);
|
|
48119
|
+
const seedDir = String(options.seed);
|
|
48120
|
+
const written = /* @__PURE__ */ new Map();
|
|
48121
|
+
written.set(
|
|
48122
|
+
path42.posix.join(seedDir, "gold", "facts.jsonl"),
|
|
48123
|
+
toJsonl(corpus.facts)
|
|
48124
|
+
);
|
|
48125
|
+
written.set(
|
|
48126
|
+
path42.posix.join(seedDir, "gold", "probes.jsonl"),
|
|
48127
|
+
toJsonl(corpus.probes)
|
|
48128
|
+
);
|
|
48129
|
+
const sessionsByUser = /* @__PURE__ */ new Map();
|
|
48130
|
+
for (const session of corpus.sessions) {
|
|
48131
|
+
const list = sessionsByUser.get(session.userId) ?? [];
|
|
48132
|
+
list.push(session);
|
|
48133
|
+
sessionsByUser.set(session.userId, list);
|
|
48134
|
+
}
|
|
48135
|
+
for (const [userId, sessions] of [...sessionsByUser.entries()].sort()) {
|
|
48136
|
+
written.set(
|
|
48137
|
+
path42.posix.join(seedDir, "users", userId, "sessions.jsonl"),
|
|
48138
|
+
toJsonl(sessions)
|
|
48139
|
+
);
|
|
48140
|
+
}
|
|
48141
|
+
const files = {};
|
|
48142
|
+
for (const relPath of [...written.keys()].sort()) {
|
|
48143
|
+
files[relPath] = createHash21("sha256").update(written.get(relPath)).digest("hex");
|
|
48144
|
+
}
|
|
48145
|
+
const manifest = {
|
|
48146
|
+
name: "drift-gen-core",
|
|
48147
|
+
version: DRIFT_GEN_VERSION,
|
|
48148
|
+
generatorVersion: DRIFT_GEN_VERSION,
|
|
48149
|
+
seeds: [options.seed],
|
|
48150
|
+
counts: {
|
|
48151
|
+
users: options.users,
|
|
48152
|
+
epochs: options.epochs,
|
|
48153
|
+
facts: corpus.facts.length,
|
|
48154
|
+
probes: corpus.probes.length
|
|
48155
|
+
},
|
|
48156
|
+
generator: {
|
|
48157
|
+
factsPerEpoch: options.factsPerEpoch ?? DRIFT_GEN_DEFAULTS.factsPerEpoch,
|
|
48158
|
+
driftingRatio: options.driftingRatio ?? DRIFT_GEN_DEFAULTS.driftingRatio,
|
|
48159
|
+
contradictedRatio: options.contradictedRatio ?? DRIFT_GEN_DEFAULTS.contradictedRatio
|
|
48160
|
+
},
|
|
48161
|
+
files,
|
|
48162
|
+
createdAt: MANIFEST_TIMESTAMP,
|
|
48163
|
+
licenses: [{ source: "synthetic", license: "MIT (repo)" }],
|
|
48164
|
+
...options.audit ? { audit: options.audit } : {}
|
|
48165
|
+
};
|
|
48166
|
+
const stagingDir = path42.join(options.outDir, `.staging-${options.seed}`);
|
|
48167
|
+
await rm16(stagingDir, { recursive: true, force: true });
|
|
48168
|
+
for (const [relPath, content] of written) {
|
|
48169
|
+
const absPath = path42.join(stagingDir, path42.relative(seedDir, relPath));
|
|
48170
|
+
await mkdir20(path42.dirname(absPath), { recursive: true });
|
|
48171
|
+
await writeFile19(absPath, content, "utf8");
|
|
48172
|
+
}
|
|
48173
|
+
const finalSeedDir = path42.join(options.outDir, seedDir);
|
|
48174
|
+
const backupDir = path42.join(options.outDir, `.backup-${options.seed}-${process.pid}`);
|
|
48175
|
+
const staleSeedDirs = (await readdir10(options.outDir, { withFileTypes: true })).filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name) && entry.name !== seedDir).map((entry) => ({
|
|
48176
|
+
source: path42.join(options.outDir, entry.name),
|
|
48177
|
+
backup: path42.join(options.outDir, `.backup-stale-${entry.name}-${process.pid}`)
|
|
48178
|
+
}));
|
|
48179
|
+
const quarantinedStaleDirs = [];
|
|
48180
|
+
const manifestPath = path42.join(options.outDir, "dataset.manifest.json");
|
|
48181
|
+
const manifestStaging = path42.join(options.outDir, ".staging-manifest.json");
|
|
48182
|
+
const manifestBackup = path42.join(options.outDir, `.backup-manifest-${process.pid}.json`);
|
|
48183
|
+
let hadPrevious = false;
|
|
48184
|
+
let replacementInstalled = false;
|
|
48185
|
+
let hadPreviousManifest = false;
|
|
48186
|
+
try {
|
|
48187
|
+
try {
|
|
48188
|
+
await rename5(manifestPath, manifestBackup);
|
|
48189
|
+
hadPreviousManifest = true;
|
|
48190
|
+
} catch (error) {
|
|
48191
|
+
if (error.code !== "ENOENT") throw error;
|
|
48192
|
+
}
|
|
48193
|
+
for (const stale of staleSeedDirs) {
|
|
48194
|
+
await rename5(stale.source, stale.backup);
|
|
48195
|
+
quarantinedStaleDirs.push(stale);
|
|
48196
|
+
}
|
|
48197
|
+
try {
|
|
48198
|
+
await rename5(finalSeedDir, backupDir);
|
|
48199
|
+
hadPrevious = true;
|
|
48200
|
+
} catch (error) {
|
|
48201
|
+
if (error.code !== "ENOENT") throw error;
|
|
48202
|
+
}
|
|
48203
|
+
await rename5(stagingDir, finalSeedDir);
|
|
48204
|
+
replacementInstalled = true;
|
|
48205
|
+
await writeFile19(manifestStaging, `${JSON.stringify(manifest, null, 2)}
|
|
48206
|
+
`, "utf8");
|
|
48207
|
+
await rename5(manifestStaging, manifestPath);
|
|
48208
|
+
} catch (error) {
|
|
48209
|
+
await rm16(manifestStaging, { force: true });
|
|
48210
|
+
await rm16(stagingDir, { recursive: true, force: true });
|
|
48211
|
+
if (hadPreviousManifest) {
|
|
48212
|
+
await rm16(manifestPath, { force: true });
|
|
48213
|
+
await rename5(manifestBackup, manifestPath);
|
|
48214
|
+
}
|
|
48215
|
+
if (replacementInstalled) {
|
|
48216
|
+
await rm16(finalSeedDir, { recursive: true, force: true });
|
|
48217
|
+
}
|
|
48218
|
+
if (hadPrevious) {
|
|
48219
|
+
try {
|
|
48220
|
+
await rename5(backupDir, finalSeedDir);
|
|
48221
|
+
} catch {
|
|
48222
|
+
console.error(
|
|
48223
|
+
`drift-gen: failed to restore the previous corpus; it is preserved at ${backupDir}`
|
|
48224
|
+
);
|
|
48225
|
+
}
|
|
48226
|
+
}
|
|
48227
|
+
for (let index = quarantinedStaleDirs.length - 1; index >= 0; index -= 1) {
|
|
48228
|
+
const stale = quarantinedStaleDirs[index];
|
|
48229
|
+
try {
|
|
48230
|
+
await rename5(stale.backup, stale.source);
|
|
48231
|
+
} catch {
|
|
48232
|
+
console.error(
|
|
48233
|
+
`drift-gen: failed to restore a stale corpus; it is preserved at ${stale.backup}`
|
|
48234
|
+
);
|
|
48235
|
+
}
|
|
48236
|
+
}
|
|
48237
|
+
throw error;
|
|
48238
|
+
}
|
|
48239
|
+
await Promise.all([
|
|
48240
|
+
...hadPrevious ? [rm16(backupDir, { recursive: true, force: true })] : [],
|
|
48241
|
+
...quarantinedStaleDirs.map((stale) => rm16(stale.backup, { recursive: true, force: true })),
|
|
48242
|
+
...hadPreviousManifest ? [rm16(manifestBackup, { force: true })] : []
|
|
48243
|
+
]);
|
|
48244
|
+
return { manifest, files: [...written.keys()].sort() };
|
|
48245
|
+
}
|
|
48246
|
+
function renderValidationReport(report) {
|
|
48247
|
+
const lines = [];
|
|
48248
|
+
lines.push(report.ok ? "drift-gen corpus: VALID" : "drift-gen corpus: INVALID");
|
|
48249
|
+
const { stats } = report;
|
|
48250
|
+
lines.push(
|
|
48251
|
+
` facts=${stats.facts} probes=${stats.probes} sessions=${stats.sessions} users=${stats.users} epochs=${stats.epochs}`
|
|
48252
|
+
);
|
|
48253
|
+
lines.push(
|
|
48254
|
+
` factsPerEpochMean=${stats.factsPerEpochMean.toFixed(2)} drifting=${stats.driftingRatio.toFixed(3)} contradicted=${stats.contradictedRatio.toFixed(3)}`
|
|
48255
|
+
);
|
|
48256
|
+
lines.push(
|
|
48257
|
+
` probes by category: current=${stats.probesByCategory.current} historical=${stats.probesByCategory.historical} transition=${stats.probesByCategory.transition} aggregation=${stats.probesByCategory.aggregation}`
|
|
48258
|
+
);
|
|
48259
|
+
lines.push(
|
|
48260
|
+
` max question/answer leakage: ${(stats.maxQuestionAnswerLeakage * 100).toFixed(0)}%`
|
|
48261
|
+
);
|
|
48262
|
+
for (const warning of report.warnings) lines.push(` warning: ${warning}`);
|
|
48263
|
+
for (const error of report.errors) lines.push(` error: ${error}`);
|
|
48264
|
+
return lines.join("\n");
|
|
48265
|
+
}
|
|
48266
|
+
async function runDriftGenCliCommand(options) {
|
|
48267
|
+
if (options.action === "validate") {
|
|
48268
|
+
if (!options.dir) {
|
|
48269
|
+
return {
|
|
48270
|
+
exitCode: 1,
|
|
48271
|
+
output: "drift-gen validate requires a corpus directory: remnic bench drift-gen validate <dir>"
|
|
48272
|
+
};
|
|
48273
|
+
}
|
|
48274
|
+
const report = await validateDriftCorpus(options.dir);
|
|
48275
|
+
return {
|
|
48276
|
+
exitCode: report.ok ? 0 : 1,
|
|
48277
|
+
output: options.json ? JSON.stringify(report, null, 2) : renderValidationReport(report)
|
|
48278
|
+
};
|
|
48279
|
+
}
|
|
48280
|
+
if (!options.out) {
|
|
48281
|
+
return {
|
|
48282
|
+
exitCode: 1,
|
|
48283
|
+
output: "drift-gen requires --out <dir> to write the corpus"
|
|
48284
|
+
};
|
|
48285
|
+
}
|
|
48286
|
+
const result = await generateDriftCorpus({
|
|
48287
|
+
users: options.users ?? DRIFT_GEN_DEFAULTS.users,
|
|
48288
|
+
epochs: options.epochs ?? DRIFT_GEN_DEFAULTS.epochs,
|
|
48289
|
+
seed: options.seed ?? DRIFT_GEN_DEFAULTS.seed,
|
|
48290
|
+
outDir: options.out,
|
|
48291
|
+
factsPerEpoch: options.factsPerEpoch,
|
|
48292
|
+
driftingRatio: options.driftingRatio,
|
|
48293
|
+
contradictedRatio: options.contradictedRatio
|
|
48294
|
+
});
|
|
48295
|
+
if (options.json) {
|
|
48296
|
+
return { exitCode: 0, output: JSON.stringify(result.manifest, null, 2) };
|
|
48297
|
+
}
|
|
48298
|
+
const { counts: counts2 } = result.manifest;
|
|
48299
|
+
return {
|
|
48300
|
+
exitCode: 0,
|
|
48301
|
+
output: [
|
|
48302
|
+
`drift-gen v${DRIFT_GEN_VERSION}: wrote ${result.files.length + 1} files to ${options.out}`,
|
|
48303
|
+
` users=${counts2.users} epochs=${counts2.epochs} facts=${counts2.facts} probes=${counts2.probes} seed=${result.manifest.seeds[0]}`,
|
|
48304
|
+
` validate with: remnic bench drift-gen validate ${options.out}`
|
|
48305
|
+
].join("\n")
|
|
48306
|
+
};
|
|
48307
|
+
}
|
|
45979
48308
|
export {
|
|
45980
48309
|
AMA_BENCH_DIAGNOSTIC_VARIANTS,
|
|
45981
48310
|
ASSISTANT_AGENT_CONFIG_KEY,
|
|
@@ -46018,6 +48347,8 @@ export {
|
|
|
46018
48347
|
DEFAULT_KAPPA_BOOTSTRAP_SAMPLES,
|
|
46019
48348
|
DEFAULT_KAPPA_CONFIDENCE_LEVEL,
|
|
46020
48349
|
DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL,
|
|
48350
|
+
DRIFT_GEN_DEFAULTS,
|
|
48351
|
+
DRIFT_GEN_VERSION,
|
|
46021
48352
|
EMPTY_CONTAMINATION_MANIFEST,
|
|
46022
48353
|
GENERAL_ANSWER_JUDGE_RUBRIC,
|
|
46023
48354
|
INTEGRITY_CIPHER_ALGORITHM,
|
|
@@ -46074,6 +48405,9 @@ export {
|
|
|
46074
48405
|
assistantMorningBriefDefinition,
|
|
46075
48406
|
assistantNextBestActionDefinition,
|
|
46076
48407
|
assistantSynthesisDefinition,
|
|
48408
|
+
attributeGoldMemory,
|
|
48409
|
+
attributeRun,
|
|
48410
|
+
attributeTask,
|
|
46077
48411
|
backlinkF1,
|
|
46078
48412
|
binarizeJudgeScore,
|
|
46079
48413
|
bootstrapCohensKappaConfidenceInterval,
|
|
@@ -46089,6 +48423,7 @@ export {
|
|
|
46089
48423
|
buildBenchmarkRunSeeds,
|
|
46090
48424
|
buildBuildWeekEvidenceReceipt,
|
|
46091
48425
|
buildCodexCreditReceipt,
|
|
48426
|
+
buildDriftCorpus,
|
|
46092
48427
|
buildJudgePayload,
|
|
46093
48428
|
buildOracleTrajectoryRecall,
|
|
46094
48429
|
buildProviderFreeLoCoMoRetrievalConfig,
|
|
@@ -46138,6 +48473,7 @@ export {
|
|
|
46138
48473
|
createProviderBackedStructuredJudge,
|
|
46139
48474
|
createRemnicAdapter,
|
|
46140
48475
|
createResponderFromProvider,
|
|
48476
|
+
createSeededRandom2 as createSeededRandom,
|
|
46141
48477
|
createSeededRng,
|
|
46142
48478
|
createSpotCheckFileLogger,
|
|
46143
48479
|
createStructuredBenchJudge,
|
|
@@ -46157,11 +48493,13 @@ export {
|
|
|
46157
48493
|
entityRecall,
|
|
46158
48494
|
exactMatch,
|
|
46159
48495
|
extractMetrics as extractCodingGraphMetrics,
|
|
48496
|
+
extractContentWords,
|
|
46160
48497
|
extractMarkdownSectionsByTitle,
|
|
46161
48498
|
f1Score,
|
|
46162
48499
|
fixtureToAblationScenarios,
|
|
46163
48500
|
formatHandoffNote,
|
|
46164
48501
|
formatMissingDatasetError,
|
|
48502
|
+
generateDriftCorpus,
|
|
46165
48503
|
generateReport,
|
|
46166
48504
|
generateSyntheticRepo,
|
|
46167
48505
|
getAblationCell,
|
|
@@ -46184,8 +48522,10 @@ export {
|
|
|
46184
48522
|
isSealedQrelsArtifact,
|
|
46185
48523
|
isSha256Hex,
|
|
46186
48524
|
isStructuredJudgeProvider,
|
|
48525
|
+
isTaskFailed,
|
|
46187
48526
|
judgeMemCorrectCorrectionAcceptance,
|
|
46188
48527
|
judgeMemCorrectStaleMemoryHarm,
|
|
48528
|
+
lexicalSimilarity,
|
|
46189
48529
|
linkMatches,
|
|
46190
48530
|
listBenchmarkBaselines,
|
|
46191
48531
|
listBenchmarkResults,
|
|
@@ -46220,14 +48560,17 @@ export {
|
|
|
46220
48560
|
parseLocalLabManifest,
|
|
46221
48561
|
parseRubricResponse,
|
|
46222
48562
|
parseSealedQrels,
|
|
48563
|
+
pickOne,
|
|
46223
48564
|
pickStableQualifiedName,
|
|
46224
48565
|
precisionAtK,
|
|
46225
48566
|
preflightLoCoMoRetrievalTraceCapture,
|
|
46226
48567
|
preflightLocalLabRole,
|
|
46227
48568
|
projectFolderFixture,
|
|
48569
|
+
randomInt,
|
|
46228
48570
|
recallAtK,
|
|
46229
48571
|
reconcileCodexCreditLedger,
|
|
46230
48572
|
redactBenchmarkResultSecrets,
|
|
48573
|
+
renderAttributionReportTable,
|
|
46231
48574
|
renderBaselineMarkdown,
|
|
46232
48575
|
renderBenchmarkResultExport,
|
|
46233
48576
|
renderLoComoProfileDeltaMarkdown,
|
|
@@ -46255,11 +48598,13 @@ export {
|
|
|
46255
48598
|
runAssistantMorningBriefBenchmark,
|
|
46256
48599
|
runAssistantNextBestActionBenchmark,
|
|
46257
48600
|
runAssistantSynthesisBenchmark,
|
|
48601
|
+
runAttributeCliCommand,
|
|
46258
48602
|
runBaseline,
|
|
46259
48603
|
runBenchSuite,
|
|
46260
48604
|
runBenchmark,
|
|
46261
48605
|
runCodingGraphBenchmark,
|
|
46262
48606
|
runCustomBenchmarkFile,
|
|
48607
|
+
runDriftGenCliCommand,
|
|
46263
48608
|
runExplain,
|
|
46264
48609
|
runExtractionAttack,
|
|
46265
48610
|
runJudgeCalibration,
|
|
@@ -46278,6 +48623,7 @@ export {
|
|
|
46278
48623
|
selectAmaBenchDiagnosticVariants,
|
|
46279
48624
|
selectCalibrationSlice,
|
|
46280
48625
|
selectFixtureVariant,
|
|
48626
|
+
serializeAttributionReport,
|
|
46281
48627
|
serializeBenchmarkArtifact,
|
|
46282
48628
|
serializeBuildWeekEvidenceReceipt,
|
|
46283
48629
|
serializeJsonl,
|
|
@@ -46285,7 +48631,9 @@ export {
|
|
|
46285
48631
|
serializeLoCoMoRetrievalTraceReceipt,
|
|
46286
48632
|
serializeSealedQrels,
|
|
46287
48633
|
shuffleTasks,
|
|
48634
|
+
shuffled,
|
|
46288
48635
|
timed,
|
|
48636
|
+
validateDriftCorpus,
|
|
46289
48637
|
verifyRubricDigest,
|
|
46290
48638
|
writeBenchmarkArtifact,
|
|
46291
48639
|
writeBenchmarkPublishFeed,
|