@tangle-network/agent-knowledge 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +250 -74
- package/dist/benchmarks/index.d.ts +5 -0
- package/dist/benchmarks/index.js +52 -0
- package/dist/chunk-DQ3PDMDP.js +115 -0
- package/dist/chunk-DQ3PDMDP.js.map +1 -0
- package/dist/chunk-ULXZI235.js +1855 -0
- package/dist/chunk-ULXZI235.js.map +1 -0
- package/dist/{chunk-U6KQ4FS3.js → chunk-VN2OGUUP.js} +306 -40
- package/dist/chunk-VN2OGUUP.js.map +1 -0
- package/dist/{chunk-WWY5FTKQ.js → chunk-W6VWYTNH.js} +10 -115
- package/dist/chunk-W6VWYTNH.js.map +1 -0
- package/dist/chunk-XVU5FFQA.js +49 -0
- package/dist/chunk-XVU5FFQA.js.map +1 -0
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/index-Cj5jRXOz.d.ts +426 -0
- package/dist/index.d.ts +1036 -169
- package/dist/index.js +3372 -558
- package/dist/index.js.map +1 -1
- package/dist/memory/index.d.ts +5 -70
- package/dist/memory/index.js +20 -4
- package/dist/types-CjqPcTTK.d.ts +319 -0
- package/docs/architecture.md +7 -5
- package/docs/eval/investment-material-facts.md +252 -0
- package/docs/eval/rag-eval-roadmap.md +119 -0
- package/docs/results/investment-thesis.md +306 -0
- package/docs/results/research-driving.md +265 -0
- package/docs/two-agent-research-ab.md +121 -10
- package/package.json +8 -9
- package/dist/chunk-MU5CEOGE.js +0 -387
- package/dist/chunk-MU5CEOGE.js.map +0 -1
- package/dist/chunk-U6KQ4FS3.js.map +0 -1
- package/dist/chunk-WWY5FTKQ.js.map +0 -1
- package/dist/profiles/index.d.ts +0 -194
- package/dist/profiles/index.js +0 -11
- package/docs/recursive-research-leaf.md +0 -86
- package/docs/supervisor-profiles.md +0 -295
- /package/dist/{profiles → benchmarks}/index.js.map +0 -0
package/dist/index.js
CHANGED
|
@@ -30,28 +30,30 @@ import {
|
|
|
30
30
|
normalizeLinkTarget,
|
|
31
31
|
parseFrontmatter,
|
|
32
32
|
parseKnowledgeWriteBlocks,
|
|
33
|
-
reciprocalRankFusion,
|
|
34
|
-
searchKnowledge,
|
|
35
33
|
sourceRegistryPath,
|
|
36
34
|
stringMetadata,
|
|
37
35
|
textSourceAdapter,
|
|
38
|
-
tokenizeQuery,
|
|
39
36
|
validateKnowledgeIndex,
|
|
40
37
|
writeJson,
|
|
41
38
|
writeKnowledgeIndex,
|
|
42
39
|
writeSourceRegistry
|
|
43
|
-
} from "./chunk-
|
|
40
|
+
} from "./chunk-W6VWYTNH.js";
|
|
44
41
|
import {
|
|
45
42
|
AgentMemoryHitSchema,
|
|
46
43
|
AgentMemoryKindSchema,
|
|
47
44
|
AgentMemoryScopeSchema,
|
|
48
45
|
AgentMemoryWriteInputSchema,
|
|
46
|
+
applyRetrievalHoldout,
|
|
47
|
+
applySessionStickyRetrievalHoldout,
|
|
49
48
|
createNeo4jAgentMemoryAdapter,
|
|
50
49
|
defaultGetMemoryContext,
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
renderMemoryContext
|
|
54
|
-
|
|
50
|
+
deterministicRng,
|
|
51
|
+
emitRetrievalHoldoutBypass,
|
|
52
|
+
renderMemoryContext,
|
|
53
|
+
resetRetrievalHoldoutRegistry,
|
|
54
|
+
retrievalHoldoutConfigHash,
|
|
55
|
+
toOffPolicyTrajectory
|
|
56
|
+
} from "./chunk-VN2OGUUP.js";
|
|
55
57
|
import {
|
|
56
58
|
IRS_DIMENSION_HINTS,
|
|
57
59
|
MAX_RESPONSE_BYTES,
|
|
@@ -68,19 +70,74 @@ import {
|
|
|
68
70
|
looksLikeBlockPage,
|
|
69
71
|
politeFetch
|
|
70
72
|
} from "./chunk-WCYW2GDA.js";
|
|
73
|
+
import {
|
|
74
|
+
INDUSTRY_MEMORY_BENCHMARKS,
|
|
75
|
+
INDUSTRY_RAG_BENCHMARKS,
|
|
76
|
+
buildFirstPartyMemoryLifecycleBenchmarkCases,
|
|
77
|
+
buildIndustryMemoryBenchmarkSmokeCases,
|
|
78
|
+
buildIndustryRagBenchmarkSmokeCases,
|
|
79
|
+
buildKnowledgeBenchmarkScenarios,
|
|
80
|
+
buildRetrievalBenchmarkCasesFromQrels,
|
|
81
|
+
buildRetrievalEvalDispatch,
|
|
82
|
+
buildRetrievalParameterCandidates,
|
|
83
|
+
createInMemoryBenchmarkAdapter,
|
|
84
|
+
createMemoryAdapterBenchmarkResponder,
|
|
85
|
+
createNoopMemoryBenchmarkAdapter,
|
|
86
|
+
isKnowledgeMemoryBenchmarkCase,
|
|
87
|
+
knowledgeBenchmarkJudge,
|
|
88
|
+
parseKnowledgeBenchmarkJsonl,
|
|
89
|
+
parseKnowledgeBenchmarkQrels,
|
|
90
|
+
renderKnowledgeBenchmarkReportMarkdown,
|
|
91
|
+
respondToIndustryMemoryBenchmarkSmokeCase,
|
|
92
|
+
respondToIndustryRagBenchmarkSmokeCase,
|
|
93
|
+
retrievalConfigFromSurface,
|
|
94
|
+
retrievalConfigSurface,
|
|
95
|
+
retrievalParameterSweepProposer,
|
|
96
|
+
retrievalRecallJudge,
|
|
97
|
+
runKnowledgeBenchmarkSuite,
|
|
98
|
+
runMemoryAdapterBenchmark,
|
|
99
|
+
runRetrievalImprovementLoop,
|
|
100
|
+
scoreKnowledgeBenchmarkArtifact,
|
|
101
|
+
scoreMemoryBenchmarkArtifact,
|
|
102
|
+
scoreRetrievalArtifact,
|
|
103
|
+
summarizeKnowledgeBenchmarkCampaign
|
|
104
|
+
} from "./chunk-ULXZI235.js";
|
|
105
|
+
import {
|
|
106
|
+
reciprocalRankFusion,
|
|
107
|
+
searchKnowledge,
|
|
108
|
+
tokenizeQuery
|
|
109
|
+
} from "./chunk-DQ3PDMDP.js";
|
|
110
|
+
import {
|
|
111
|
+
memoryHitToSourceRecord,
|
|
112
|
+
memoryWriteResultToSourceRecord
|
|
113
|
+
} from "./chunk-XVU5FFQA.js";
|
|
71
114
|
import {
|
|
72
115
|
sha256,
|
|
73
116
|
slugify,
|
|
74
117
|
stableId
|
|
75
118
|
} from "./chunk-YMKHCTS2.js";
|
|
76
|
-
import {
|
|
77
|
-
researcherProfile
|
|
78
|
-
} from "./chunk-MU5CEOGE.js";
|
|
79
119
|
|
|
80
120
|
// src/web-research-worker.ts
|
|
81
121
|
var DEFAULT_MODEL = "glm-5.2";
|
|
82
122
|
var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
|
|
83
123
|
var MIN_MAX_TOKENS = 1200;
|
|
124
|
+
var transientStatuses = /* @__PURE__ */ new Set([429, 502, 503, 504]);
|
|
125
|
+
async function fetchWithRetry(url, init, opts) {
|
|
126
|
+
let lastRes;
|
|
127
|
+
for (let attempt = 0; attempt <= opts.maxRetries; attempt += 1) {
|
|
128
|
+
if (opts.signal?.aborted) throw new RouterError(0, "aborted");
|
|
129
|
+
const res = await fetch(url, init);
|
|
130
|
+
if (res.ok || !transientStatuses.has(res.status)) return res;
|
|
131
|
+
lastRes = res;
|
|
132
|
+
if (attempt === opts.maxRetries) break;
|
|
133
|
+
await res.text().catch(() => "");
|
|
134
|
+
const backoff = opts.retryBaseMs * 2 ** attempt;
|
|
135
|
+
const jitter = backoff * (0.75 + Math.random() * 0.5);
|
|
136
|
+
await new Promise((resolve) => setTimeout(resolve, jitter));
|
|
137
|
+
}
|
|
138
|
+
if (lastRes) return lastRes;
|
|
139
|
+
throw new RouterError(0, "fetchWithRetry produced no response");
|
|
140
|
+
}
|
|
84
141
|
var RouterError = class extends Error {
|
|
85
142
|
constructor(status, message) {
|
|
86
143
|
super(`router ${status}: ${message}`);
|
|
@@ -96,6 +153,8 @@ function createTangleRouterClient(options = {}) {
|
|
|
96
153
|
throw new RouterError(401, "no TANGLE_API_KEY (pass apiKey or set the env var)");
|
|
97
154
|
}
|
|
98
155
|
const model = options.model ?? DEFAULT_MODEL;
|
|
156
|
+
const maxRetries = Math.max(0, options.maxRetries ?? 4);
|
|
157
|
+
const retryBaseMs = Math.max(1, options.retryBaseMs ?? 1500);
|
|
99
158
|
const headers = {
|
|
100
159
|
"Content-Type": "application/json",
|
|
101
160
|
Authorization: `Bearer ${apiKey}`
|
|
@@ -112,16 +171,20 @@ function createTangleRouterClient(options = {}) {
|
|
|
112
171
|
return {
|
|
113
172
|
async search(query, opts) {
|
|
114
173
|
const t0 = Date.now();
|
|
115
|
-
const res = await
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
174
|
+
const res = await fetchWithRetry(
|
|
175
|
+
`${baseUrl}/search`,
|
|
176
|
+
{
|
|
177
|
+
method: "POST",
|
|
178
|
+
headers,
|
|
179
|
+
signal: options.signal,
|
|
180
|
+
body: JSON.stringify({
|
|
181
|
+
query,
|
|
182
|
+
...options.searchProvider ? { provider: options.searchProvider } : {},
|
|
183
|
+
...opts?.maxResults != null ? { maxResults: opts.maxResults } : {}
|
|
184
|
+
})
|
|
185
|
+
},
|
|
186
|
+
{ maxRetries, retryBaseMs, signal: options.signal }
|
|
187
|
+
);
|
|
125
188
|
acc.searchCalls += 1;
|
|
126
189
|
acc.wallMs += Date.now() - t0;
|
|
127
190
|
if (!res.ok) {
|
|
@@ -133,12 +196,16 @@ function createTangleRouterClient(options = {}) {
|
|
|
133
196
|
async chat(messages, maxTokens) {
|
|
134
197
|
const max_tokens = Math.max(MIN_MAX_TOKENS, maxTokens ?? MIN_MAX_TOKENS);
|
|
135
198
|
const t0 = Date.now();
|
|
136
|
-
const res = await
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
199
|
+
const res = await fetchWithRetry(
|
|
200
|
+
`${baseUrl}/chat/completions`,
|
|
201
|
+
{
|
|
202
|
+
method: "POST",
|
|
203
|
+
headers,
|
|
204
|
+
signal: options.signal,
|
|
205
|
+
body: JSON.stringify({ model, messages, max_tokens, temperature: 0.2, stream: false })
|
|
206
|
+
},
|
|
207
|
+
{ maxRetries, retryBaseMs, signal: options.signal }
|
|
208
|
+
);
|
|
142
209
|
if (!res.ok) {
|
|
143
210
|
throw new RouterError(res.status, await res.text().catch(() => res.statusText));
|
|
144
211
|
}
|
|
@@ -929,6 +996,15 @@ ${excerpt}`,
|
|
|
929
996
|
};
|
|
930
997
|
}
|
|
931
998
|
|
|
999
|
+
// src/collection-research-driver.ts
|
|
1000
|
+
function createCollectionResearchDriver() {
|
|
1001
|
+
return {
|
|
1002
|
+
verifySource(_source) {
|
|
1003
|
+
return { accept: true };
|
|
1004
|
+
}
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
|
|
932
1008
|
// src/discovery.ts
|
|
933
1009
|
function createLocalDiscoveryDispatcher(worker) {
|
|
934
1010
|
return {
|
|
@@ -1179,331 +1255,739 @@ function buildKey(key) {
|
|
|
1179
1255
|
return `${key.workspaceId}::${key.sourceId}`;
|
|
1180
1256
|
}
|
|
1181
1257
|
|
|
1182
|
-
// src/
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1258
|
+
// src/investment-thesis-set.ts
|
|
1259
|
+
var investmentThesisSet = [
|
|
1260
|
+
{
|
|
1261
|
+
ticker: "SIVB",
|
|
1262
|
+
company: "SVB Financial Group",
|
|
1263
|
+
cik: "719739",
|
|
1264
|
+
cutoff: "2023-02-24",
|
|
1265
|
+
sector: "Banking",
|
|
1266
|
+
knownOutcome: "Failed in a deposit run and was placed in FDIC receivership on March 10, 2023; the holding company filed Chapter 11 on March 17, 2023.",
|
|
1267
|
+
facts: [
|
|
1268
|
+
{
|
|
1269
|
+
id: "SIVB/f1",
|
|
1270
|
+
lens: "off-balance-sheet",
|
|
1271
|
+
fact: "Held-to-maturity (HTM) securities carried at $91.3B amortized cost had a fair value of only $76.2B \u2014 a ~$15.1B unrealized loss that, because the portfolio is HTM, never touched earnings or equity and sat only in the footnotes.",
|
|
1272
|
+
expected: [
|
|
1273
|
+
{
|
|
1274
|
+
label: "HTM securities",
|
|
1275
|
+
anyOf: ["held-to-maturity", "held to maturity", "htm"]
|
|
1276
|
+
},
|
|
1277
|
+
{
|
|
1278
|
+
label: "large unrealized loss (~$15B) / fair value gap",
|
|
1279
|
+
anyOf: [
|
|
1280
|
+
"15.1",
|
|
1281
|
+
"15.2",
|
|
1282
|
+
"$15 billion",
|
|
1283
|
+
"15 billion",
|
|
1284
|
+
"76,169",
|
|
1285
|
+
"76.2 billion",
|
|
1286
|
+
"91,321",
|
|
1287
|
+
"unrealized loss",
|
|
1288
|
+
"below amortized cost",
|
|
1289
|
+
"fair value"
|
|
1290
|
+
]
|
|
1291
|
+
}
|
|
1292
|
+
],
|
|
1293
|
+
minGroups: 2,
|
|
1294
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
1295
|
+
evidence: 'Balance sheet (Dec 31, 2022): "Held-to-maturity securities, at amortized cost ... 91,321" with parenthetical "(fair value of $ 76,169 ...)". The gap = $91,321M - $76,169M = ~$15,152M unrealized loss, disclosed only in the notes.'
|
|
1296
|
+
},
|
|
1297
|
+
{
|
|
1298
|
+
id: "SIVB/f2",
|
|
1299
|
+
lens: "off-balance-sheet",
|
|
1300
|
+
fact: "The HTM unrealized loss (~$15.1B) was roughly equal to the company's entire $16.0B total stockholders' equity \u2014 a mark-to-market wipeout hidden by HTM accounting.",
|
|
1301
|
+
expected: [
|
|
1302
|
+
{
|
|
1303
|
+
label: "loss near/exceeds total equity",
|
|
1304
|
+
anyOf: [
|
|
1305
|
+
"equity",
|
|
1306
|
+
"capital",
|
|
1307
|
+
"insolvent",
|
|
1308
|
+
"wipe out",
|
|
1309
|
+
"exceeds",
|
|
1310
|
+
"nearly all",
|
|
1311
|
+
"tangible book",
|
|
1312
|
+
"16,004",
|
|
1313
|
+
"16 billion"
|
|
1314
|
+
]
|
|
1315
|
+
}
|
|
1316
|
+
],
|
|
1317
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
1318
|
+
evidence: `"Total SVBFG stockholders' equity 16,004" (Dec 31, 2022). The ~$15.15B HTM unrealized loss is ~95% of the $16.0B reported equity.`
|
|
1319
|
+
},
|
|
1320
|
+
{
|
|
1321
|
+
id: "SIVB/f3",
|
|
1322
|
+
lens: "concentration",
|
|
1323
|
+
fact: "Estimated uninsured deposits in U.S. offices were $151.5B at year-end 2022 \u2014 the run-prone funding base; a high share of total deposits exceeded the FDIC limit.",
|
|
1324
|
+
expected: [
|
|
1325
|
+
{
|
|
1326
|
+
label: "large uninsured deposit base",
|
|
1327
|
+
anyOf: [
|
|
1328
|
+
"uninsured deposit",
|
|
1329
|
+
"above the fdic",
|
|
1330
|
+
"exceed the fdic",
|
|
1331
|
+
"exceeds the fdic",
|
|
1332
|
+
"151.5",
|
|
1333
|
+
"$151 billion",
|
|
1334
|
+
"fdic insurance limit"
|
|
1335
|
+
]
|
|
1336
|
+
}
|
|
1337
|
+
],
|
|
1338
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
1339
|
+
evidence: '"As of December 31, 2022 ... the amount of estimated uninsured deposits in U.S. offices that exceed the FDIC insurance limit were $151.5 billion".'
|
|
1340
|
+
},
|
|
1341
|
+
{
|
|
1342
|
+
id: "SIVB/f4",
|
|
1343
|
+
lens: "margin-trend",
|
|
1344
|
+
fact: "Cheap noninterest-bearing demand deposits fell 20 percentage points in one year \u2014 to 47% of total deposits from 67% \u2014 meaning funding costs were set to rise sharply as clients moved to interest-bearing accounts.",
|
|
1345
|
+
expected: [
|
|
1346
|
+
{
|
|
1347
|
+
label: "deposit mix shift to costlier funding",
|
|
1348
|
+
anyOf: [
|
|
1349
|
+
"noninterest-bearing",
|
|
1350
|
+
"non-interest-bearing",
|
|
1351
|
+
"noninterest bearing",
|
|
1352
|
+
"deposit mix",
|
|
1353
|
+
"funding cost",
|
|
1354
|
+
"cost of deposits",
|
|
1355
|
+
"interest-bearing",
|
|
1356
|
+
"47 percent",
|
|
1357
|
+
"20 percentage"
|
|
1358
|
+
]
|
|
1359
|
+
}
|
|
1360
|
+
],
|
|
1361
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
1362
|
+
evidence: '"Noninterest-bearing demand deposits to total deposits decreased by 20 percentage points to 47 percent as of December 31, 2022, compared to ... 2021."'
|
|
1363
|
+
},
|
|
1364
|
+
{
|
|
1365
|
+
id: "SIVB/f5",
|
|
1366
|
+
lens: "concentration",
|
|
1367
|
+
fact: 'The deposit and loan base was concentrated in a single client type \u2014 the "innovation economy" (venture-backed technology and life-science startups) \u2014 so a downturn in venture funding would hit deposits and credit simultaneously.',
|
|
1368
|
+
expected: [
|
|
1369
|
+
{
|
|
1370
|
+
label: "concentration in tech / startups / innovation economy",
|
|
1371
|
+
anyOf: [
|
|
1372
|
+
"innovation economy",
|
|
1373
|
+
"technology",
|
|
1374
|
+
"life science",
|
|
1375
|
+
"venture",
|
|
1376
|
+
"startup",
|
|
1377
|
+
"early-stage",
|
|
1378
|
+
"concentrat",
|
|
1379
|
+
"single industry",
|
|
1380
|
+
"sector concentration"
|
|
1381
|
+
]
|
|
1382
|
+
}
|
|
1383
|
+
],
|
|
1384
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
1385
|
+
evidence: 'The 10-K repeatedly frames the franchise around clients in "the innovation economy" (technology, life science / healthcare, and the venture firms that back them) \u2014 a single-sector deposit + credit concentration.'
|
|
1386
|
+
},
|
|
1387
|
+
{
|
|
1388
|
+
id: "SIVB/f6",
|
|
1389
|
+
lens: "off-balance-sheet",
|
|
1390
|
+
fact: "Available-for-sale (AFS) securities of $28.6B amortized cost were marked to a $26.1B fair value \u2014 a ~$2.5B loss that DID flow through equity (AOCI), the visible tip of a much larger unrealized-loss iceberg dominated by the footnote-only HTM book.",
|
|
1391
|
+
expected: [
|
|
1392
|
+
{
|
|
1393
|
+
label: "AFS unrealized loss / AOCI",
|
|
1394
|
+
anyOf: [
|
|
1395
|
+
"available-for-sale",
|
|
1396
|
+
"available for sale",
|
|
1397
|
+
"afs",
|
|
1398
|
+
"aoci",
|
|
1399
|
+
"accumulated other comprehensive",
|
|
1400
|
+
"28,602",
|
|
1401
|
+
"26,069",
|
|
1402
|
+
"2.5 billion"
|
|
1403
|
+
]
|
|
1404
|
+
}
|
|
1405
|
+
],
|
|
1406
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
1407
|
+
evidence: '"Available-for-sale securities, at fair value (cost of $ 28,602 ...) 26,069" \u2014 a ~$2.5B AFS unrealized loss carried in AOCI, separate from and much smaller than the HTM gap.'
|
|
1408
|
+
}
|
|
1409
|
+
]
|
|
1410
|
+
},
|
|
1411
|
+
{
|
|
1412
|
+
ticker: "BBBY",
|
|
1413
|
+
company: "Bed Bath & Beyond Inc.",
|
|
1414
|
+
cik: "886158",
|
|
1415
|
+
cutoff: "2022-04-21",
|
|
1416
|
+
sector: "Specialty retail",
|
|
1417
|
+
knownOutcome: "Filed for Chapter 11 bankruptcy on April 23, 2023; shareholders were wiped out.",
|
|
1418
|
+
facts: [
|
|
1419
|
+
{
|
|
1420
|
+
id: "BBBY/f1",
|
|
1421
|
+
lens: "capital-return",
|
|
1422
|
+
fact: 'The company had repurchased ~$11.685B of its own stock since 2004 \u2014 including $574.9M in fiscal 2021 alone, "two years ahead of schedule" \u2014 draining the balance sheet of a business that was losing money.',
|
|
1423
|
+
expected: [
|
|
1424
|
+
{
|
|
1425
|
+
label: "massive buyback program",
|
|
1426
|
+
anyOf: [
|
|
1427
|
+
"repurchas",
|
|
1428
|
+
"buyback",
|
|
1429
|
+
"buy back",
|
|
1430
|
+
"share repurchase",
|
|
1431
|
+
"11.685",
|
|
1432
|
+
"$11.7 billion",
|
|
1433
|
+
"574.9",
|
|
1434
|
+
"$575 million"
|
|
1435
|
+
]
|
|
1436
|
+
}
|
|
1437
|
+
],
|
|
1438
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
1439
|
+
evidence: '"Since 2004 through the end of Fiscal 2021, we have repurchased approximately $11.685 billion of our common stock"; FY2021 alone "completed share repurchases of $574.9 million ... two years ahead of schedule."'
|
|
1440
|
+
},
|
|
1441
|
+
{
|
|
1442
|
+
id: "BBBY/f2",
|
|
1443
|
+
lens: "liquidity",
|
|
1444
|
+
fact: "Operating cash flow collapsed to just $17.9M in FY2021, down from $268.1M and $590.9M in the two prior years \u2014 a near-total loss of internally generated cash while it kept buying back stock.",
|
|
1445
|
+
expected: [
|
|
1446
|
+
{
|
|
1447
|
+
label: "operating cash flow collapse",
|
|
1448
|
+
anyOf: [
|
|
1449
|
+
"operating cash flow",
|
|
1450
|
+
"cash from operations",
|
|
1451
|
+
"cash provided by operating",
|
|
1452
|
+
"cash flow from operations",
|
|
1453
|
+
"17.9",
|
|
1454
|
+
"17,854",
|
|
1455
|
+
"declining cash flow",
|
|
1456
|
+
"cash generation"
|
|
1457
|
+
]
|
|
1458
|
+
}
|
|
1459
|
+
],
|
|
1460
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
1461
|
+
evidence: 'Statement of cash flows: "Net cash provided by operating activities 17,854 268,108 590,941" (FY2021 / FY2020 / FY2019, $ thousands).'
|
|
1462
|
+
},
|
|
1463
|
+
{
|
|
1464
|
+
id: "BBBY/f3",
|
|
1465
|
+
lens: "liquidity",
|
|
1466
|
+
fact: "Total shareholders' equity fell ~86% in one year \u2014 from $1.277B to $174.1M \u2014 as losses plus buybacks ate the equity cushion.",
|
|
1467
|
+
expected: [
|
|
1468
|
+
{
|
|
1469
|
+
label: "equity erosion",
|
|
1470
|
+
anyOf: [
|
|
1471
|
+
"shareholders\u2019 equity",
|
|
1472
|
+
"shareholders' equity",
|
|
1473
|
+
"stockholders equity",
|
|
1474
|
+
"book value",
|
|
1475
|
+
"equity",
|
|
1476
|
+
"net worth",
|
|
1477
|
+
"174.1",
|
|
1478
|
+
"174,145",
|
|
1479
|
+
"eroded"
|
|
1480
|
+
]
|
|
1481
|
+
}
|
|
1482
|
+
],
|
|
1483
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
1484
|
+
evidence: `"Total shareholders' equity 174,145 1,276,936" (FY2021 vs FY2020, $ thousands) \u2014 an ~86% decline in one year.`
|
|
1485
|
+
},
|
|
1486
|
+
{
|
|
1487
|
+
id: "BBBY/f4",
|
|
1488
|
+
lens: "liquidity",
|
|
1489
|
+
fact: "The company posted a $559.6M net loss in FY2021 \u2014 yet spent $574.9M on buybacks the same year, i.e. it returned more cash to shareholders than it had, let alone earned.",
|
|
1490
|
+
expected: [
|
|
1491
|
+
{
|
|
1492
|
+
label: "net loss FY2021",
|
|
1493
|
+
anyOf: [
|
|
1494
|
+
"net loss",
|
|
1495
|
+
"unprofitable",
|
|
1496
|
+
"lost money",
|
|
1497
|
+
"losing money",
|
|
1498
|
+
"559.6",
|
|
1499
|
+
"559,623",
|
|
1500
|
+
"$560 million"
|
|
1501
|
+
]
|
|
1502
|
+
}
|
|
1503
|
+
],
|
|
1504
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
1505
|
+
evidence: '"Net loss $ ( 559,623 )" for fiscal 2021 ($ thousands).'
|
|
1506
|
+
},
|
|
1507
|
+
{
|
|
1508
|
+
id: "BBBY/f5",
|
|
1509
|
+
lens: "margin-trend",
|
|
1510
|
+
fact: "Merchandise inventories rose to $1.725B even as sales fell \u2014 inventory building into a demand decline, a classic markdown-risk and cash-trap signal.",
|
|
1511
|
+
expected: [
|
|
1512
|
+
{
|
|
1513
|
+
label: "inventory building into falling demand",
|
|
1514
|
+
anyOf: [
|
|
1515
|
+
"inventor",
|
|
1516
|
+
"merchandise inventories",
|
|
1517
|
+
"overstock",
|
|
1518
|
+
"markdown",
|
|
1519
|
+
"1,725",
|
|
1520
|
+
"1.7 billion",
|
|
1521
|
+
"stockpile"
|
|
1522
|
+
]
|
|
1523
|
+
}
|
|
1524
|
+
],
|
|
1525
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
1526
|
+
evidence: '"Merchandise inventories 1,725,410 1,671,909" ($ thousands) \u2014 inventory grew year over year while comparable sales declined.'
|
|
1527
|
+
}
|
|
1528
|
+
]
|
|
1529
|
+
},
|
|
1530
|
+
{
|
|
1531
|
+
ticker: "CVNA",
|
|
1532
|
+
company: "Carvana Co.",
|
|
1533
|
+
cik: "1690820",
|
|
1534
|
+
cutoff: "2023-02-23",
|
|
1535
|
+
sector: "Auto e-commerce",
|
|
1536
|
+
knownOutcome: "The stock fell ~98% from its 2021 peak; the company narrowly avoided bankruptcy via a 2023 debt-exchange that cut and extended its obligations.",
|
|
1537
|
+
facts: [
|
|
1538
|
+
{
|
|
1539
|
+
id: "CVNA/f1",
|
|
1540
|
+
lens: "leverage",
|
|
1541
|
+
fact: "Total debt had grown to $8.39B by year-end 2022 (from $5.45B) \u2014 a debt load far larger than the equity base, built up funding growth and the ADESA deal.",
|
|
1542
|
+
expected: [
|
|
1543
|
+
{
|
|
1544
|
+
label: "large/growing debt load",
|
|
1545
|
+
anyOf: [
|
|
1546
|
+
"total debt",
|
|
1547
|
+
"long-term debt",
|
|
1548
|
+
"leverage",
|
|
1549
|
+
"highly leveraged",
|
|
1550
|
+
"debt load",
|
|
1551
|
+
"8,391",
|
|
1552
|
+
"8.4 billion",
|
|
1553
|
+
"$8.4 billion"
|
|
1554
|
+
]
|
|
1555
|
+
}
|
|
1556
|
+
],
|
|
1557
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
1558
|
+
evidence: '"Total debt 8,391 5,447" (Dec 31, 2022 vs 2021, $ millions).'
|
|
1559
|
+
},
|
|
1560
|
+
{
|
|
1561
|
+
id: "CVNA/f2",
|
|
1562
|
+
lens: "leverage",
|
|
1563
|
+
fact: "Interest expense nearly tripled to $486M in 2022 (from $176M) \u2014 debt-service was consuming cash a still-unprofitable company did not have.",
|
|
1564
|
+
expected: [
|
|
1565
|
+
{
|
|
1566
|
+
label: "rising interest burden",
|
|
1567
|
+
anyOf: [
|
|
1568
|
+
"interest expense",
|
|
1569
|
+
"interest cost",
|
|
1570
|
+
"debt service",
|
|
1571
|
+
"cost of debt",
|
|
1572
|
+
"486",
|
|
1573
|
+
"interest burden"
|
|
1574
|
+
]
|
|
1575
|
+
}
|
|
1576
|
+
],
|
|
1577
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
1578
|
+
evidence: '"Interest expense 486 176" (FY2022 vs FY2021, $ millions).'
|
|
1579
|
+
},
|
|
1580
|
+
{
|
|
1581
|
+
id: "CVNA/f3",
|
|
1582
|
+
lens: "leverage",
|
|
1583
|
+
fact: "In May 2022 Carvana bought ADESA's U.S. physical auction business for ~$2.2B in cash \u2014 a debt-funded acquisition that stretched the balance sheet right as used-car demand turned.",
|
|
1584
|
+
expected: [
|
|
1585
|
+
{
|
|
1586
|
+
label: "ADESA acquisition ~$2.2B",
|
|
1587
|
+
anyOf: ["adesa", "2.2 billion", "$2.2 billion", "physical auction", "acquisition"]
|
|
1588
|
+
}
|
|
1589
|
+
],
|
|
1590
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
1591
|
+
evidence: `"physical auction business of ADESA US Auction, LLC for approximately $2.2 billion in cash (the 'ADESA Acquisition')", closed 2022-05-09.`
|
|
1592
|
+
},
|
|
1593
|
+
{
|
|
1594
|
+
id: "CVNA/f4",
|
|
1595
|
+
lens: "governance",
|
|
1596
|
+
fact: "Carvana leases hubs and properties from DriveTime \u2014 a company controlled by founder/CEO Ernest Garcia III and his father Ernest Garcia II \u2014 a recurring related-party arrangement with the controlling family.",
|
|
1597
|
+
expected: [
|
|
1598
|
+
{
|
|
1599
|
+
label: "related-party with founder family / DriveTime",
|
|
1600
|
+
anyOf: [
|
|
1601
|
+
"related party",
|
|
1602
|
+
"related-party",
|
|
1603
|
+
"drivetime",
|
|
1604
|
+
"garcia",
|
|
1605
|
+
"controlled by",
|
|
1606
|
+
"affiliate of",
|
|
1607
|
+
"conflict of interest"
|
|
1608
|
+
]
|
|
1609
|
+
}
|
|
1610
|
+
],
|
|
1611
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
1612
|
+
evidence: 'Related Party Transactions note: lease agreements with "DriveTime Automotive Group", a related party "due to Ernest Garcia II, Ernest Garcia III, and entities controlled by one or both of them".'
|
|
1613
|
+
},
|
|
1614
|
+
{
|
|
1615
|
+
id: "CVNA/f5",
|
|
1616
|
+
lens: "liquidity",
|
|
1617
|
+
fact: "The 2022 net loss was $2.894B \u2014 a loss far wider than prior years, showing the unit economics had not turned even at scale.",
|
|
1618
|
+
expected: [
|
|
1619
|
+
{
|
|
1620
|
+
label: "large net loss FY2022",
|
|
1621
|
+
anyOf: [
|
|
1622
|
+
"net loss",
|
|
1623
|
+
"unprofitable",
|
|
1624
|
+
"losing money",
|
|
1625
|
+
"cash burn",
|
|
1626
|
+
"2,894",
|
|
1627
|
+
"2.9 billion",
|
|
1628
|
+
"$2.9 billion"
|
|
1629
|
+
]
|
|
1630
|
+
}
|
|
1631
|
+
],
|
|
1632
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
1633
|
+
evidence: '"Net loss $ (2,894 ..." for fiscal 2022 ($ millions).'
|
|
1634
|
+
}
|
|
1635
|
+
]
|
|
1636
|
+
},
|
|
1637
|
+
{
|
|
1638
|
+
ticker: "PTON",
|
|
1639
|
+
company: "Peloton Interactive, Inc.",
|
|
1640
|
+
cik: "1639825",
|
|
1641
|
+
cutoff: "2022-09-07",
|
|
1642
|
+
sector: "Consumer fitness hardware",
|
|
1643
|
+
knownOutcome: "The stock fell ~95% from its 2021 peak; the founder-CEO departed, the company underwent mass layoffs and a multi-year turnaround through fiscal 2024.",
|
|
1644
|
+
facts: [
|
|
1645
|
+
{
|
|
1646
|
+
id: "PTON/f1",
|
|
1647
|
+
lens: "margin-trend",
|
|
1648
|
+
fact: "Connected Fitness (hardware) gross margin turned NEGATIVE \u2014 to (11)% in FY2022 \u2014 meaning Peloton lost money on every bike/tread it sold before any operating cost; revenue growth was masking a broken unit economics.",
|
|
1649
|
+
expected: [
|
|
1650
|
+
{
|
|
1651
|
+
label: "negative / collapsing hardware gross margin",
|
|
1652
|
+
anyOf: [
|
|
1653
|
+
"gross margin",
|
|
1654
|
+
"negative margin",
|
|
1655
|
+
"gross profit",
|
|
1656
|
+
"margin compression",
|
|
1657
|
+
"losing money on each",
|
|
1658
|
+
"below cost",
|
|
1659
|
+
"(11)",
|
|
1660
|
+
"-11",
|
|
1661
|
+
"negative gross"
|
|
1662
|
+
]
|
|
1663
|
+
}
|
|
1664
|
+
],
|
|
1665
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
1666
|
+
evidence: 'MD&A "Gross Profit, and Gross Margin" table: Connected Fitness "Gross Margin decreased to (11)" percent in fiscal 2022 \u2014 a negative hardware gross margin.'
|
|
1667
|
+
},
|
|
1668
|
+
{
|
|
1669
|
+
id: "PTON/f2",
|
|
1670
|
+
lens: "liquidity",
|
|
1671
|
+
fact: "Inventories climbed to $1.105B as pandemic demand normalized \u2014 a glut of unsold equipment that tied up cash and risked markdowns.",
|
|
1672
|
+
expected: [
|
|
1673
|
+
{
|
|
1674
|
+
label: "inventory glut",
|
|
1675
|
+
anyOf: [
|
|
1676
|
+
"inventor",
|
|
1677
|
+
"overstock",
|
|
1678
|
+
"excess inventory",
|
|
1679
|
+
"glut",
|
|
1680
|
+
"markdown",
|
|
1681
|
+
"unsold",
|
|
1682
|
+
"1,104",
|
|
1683
|
+
"1.1 billion"
|
|
1684
|
+
]
|
|
1685
|
+
}
|
|
1686
|
+
],
|
|
1687
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
1688
|
+
evidence: '"Inventories, net 1,104.5 937" (FY2022 vs FY2021, $ millions).'
|
|
1689
|
+
},
|
|
1690
|
+
{
|
|
1691
|
+
id: "PTON/f3",
|
|
1692
|
+
lens: "governance",
|
|
1693
|
+
fact: "A dual-class structure gives Class B holders 20 votes per share vs 1 for Class A \u2014 concentrating control with insiders/founders and limiting public shareholders' say.",
|
|
1694
|
+
expected: [
|
|
1695
|
+
{
|
|
1696
|
+
label: "dual-class super-voting control",
|
|
1697
|
+
anyOf: [
|
|
1698
|
+
"dual-class",
|
|
1699
|
+
"dual class",
|
|
1700
|
+
"class b",
|
|
1701
|
+
"20 votes",
|
|
1702
|
+
"super-voting",
|
|
1703
|
+
"supervoting",
|
|
1704
|
+
"voting control",
|
|
1705
|
+
"multiple votes per share"
|
|
1706
|
+
]
|
|
1707
|
+
}
|
|
1708
|
+
],
|
|
1709
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
1710
|
+
evidence: '"Class B common stock has 20 votes per share and our Class A common stock has one vote per share."'
|
|
1711
|
+
},
|
|
1712
|
+
{
|
|
1713
|
+
id: "PTON/f4",
|
|
1714
|
+
lens: "liquidity",
|
|
1715
|
+
fact: "Peloton reported a $2.827B net loss in FY2022 \u2014 an order-of-magnitude wider loss than the prior year, signaling the demand normalization had broken the model, not just dented it.",
|
|
1716
|
+
expected: [
|
|
1717
|
+
{
|
|
1718
|
+
label: "large net loss FY2022",
|
|
1719
|
+
anyOf: [
|
|
1720
|
+
"net loss",
|
|
1721
|
+
"unprofitable",
|
|
1722
|
+
"losing money",
|
|
1723
|
+
"cash burn",
|
|
1724
|
+
"2,827",
|
|
1725
|
+
"2.8 billion",
|
|
1726
|
+
"$2.8 billion"
|
|
1727
|
+
]
|
|
1728
|
+
}
|
|
1729
|
+
],
|
|
1730
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
1731
|
+
evidence: '"Net loss $ (2,827 ..." for fiscal 2022 ($ millions).'
|
|
1732
|
+
},
|
|
1733
|
+
{
|
|
1734
|
+
id: "PTON/f5",
|
|
1735
|
+
lens: "regulatory",
|
|
1736
|
+
fact: "Peloton was running a CPSC recall of its Tread+ treadmill (tied to injuries and a child death) \u2014 an open product-safety and legal exposure beyond the demand story.",
|
|
1737
|
+
expected: [
|
|
1738
|
+
{
|
|
1739
|
+
label: "Tread+ / CPSC recall exposure",
|
|
1740
|
+
anyOf: [
|
|
1741
|
+
"recall",
|
|
1742
|
+
"cpsc",
|
|
1743
|
+
"consumer product safety",
|
|
1744
|
+
"tread+",
|
|
1745
|
+
"tread plus",
|
|
1746
|
+
"product safety",
|
|
1747
|
+
"injuries",
|
|
1748
|
+
"safety"
|
|
1749
|
+
]
|
|
1750
|
+
}
|
|
1751
|
+
],
|
|
1752
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
1753
|
+
evidence: `"recall on Tread+, which we are conducting in collaboration with the Consumer Product Safety Commission ('CPSC')"; the Tread product recalls "in the fourth quarter of fiscal 2021 continued to impact" results.`
|
|
1754
|
+
},
|
|
1755
|
+
{
|
|
1756
|
+
id: "PTON/f6",
|
|
1757
|
+
lens: "leverage",
|
|
1758
|
+
fact: "Peloton was locked into ~$334M of manufacturing purchase commitments even as demand fell \u2014 contractual inventory it had to take on regardless of whether it could sell it.",
|
|
1759
|
+
expected: [
|
|
1760
|
+
{
|
|
1761
|
+
label: "locked-in purchase commitments",
|
|
1762
|
+
anyOf: [
|
|
1763
|
+
"purchase commitment",
|
|
1764
|
+
"purchase obligation",
|
|
1765
|
+
"minimum purchase",
|
|
1766
|
+
"take-or-pay",
|
|
1767
|
+
"committed to purchase",
|
|
1768
|
+
"334",
|
|
1769
|
+
"manufacturing commitment"
|
|
1770
|
+
]
|
|
1771
|
+
}
|
|
1772
|
+
],
|
|
1773
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
1774
|
+
evidence: '"purchase commitments related to the manufacture of Peloton products were estimated to be approximately $334" million.'
|
|
1775
|
+
}
|
|
1776
|
+
]
|
|
1777
|
+
},
|
|
1778
|
+
{
|
|
1779
|
+
ticker: "SI",
|
|
1780
|
+
company: "Silvergate Capital Corporation",
|
|
1781
|
+
cik: "1312109",
|
|
1782
|
+
cutoff: "2022-02-28",
|
|
1783
|
+
sector: "Banking (digital-asset)",
|
|
1784
|
+
knownOutcome: "After the FTX collapse triggered a deposit run, Silvergate announced a voluntary wind-down of Silvergate Bank and liquidation in March 2023.",
|
|
1785
|
+
facts: [
|
|
1786
|
+
{
|
|
1787
|
+
id: "SI/f1",
|
|
1788
|
+
lens: "concentration",
|
|
1789
|
+
fact: "About 99.5% of total deposits were noninterest-bearing \u2014 essentially all funding was non-term money that could leave on demand, an extreme run-risk masked by very low funding cost.",
|
|
1790
|
+
expected: [
|
|
1791
|
+
{
|
|
1792
|
+
label: "almost all deposits noninterest-bearing / on-demand",
|
|
1793
|
+
anyOf: [
|
|
1794
|
+
"noninterest bearing",
|
|
1795
|
+
"noninterest-bearing",
|
|
1796
|
+
"non-interest-bearing",
|
|
1797
|
+
"demand deposit",
|
|
1798
|
+
"no term",
|
|
1799
|
+
"on demand",
|
|
1800
|
+
"99.5",
|
|
1801
|
+
"99 percent",
|
|
1802
|
+
"leave at any time"
|
|
1803
|
+
]
|
|
1804
|
+
}
|
|
1805
|
+
],
|
|
1806
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
1807
|
+
evidence: '"noninterest bearing deposits as a percentage of total deposits was 99.5% as of December 31, 2021."'
|
|
1808
|
+
},
|
|
1809
|
+
{
|
|
1810
|
+
id: "SI/f2",
|
|
1811
|
+
lens: "concentration",
|
|
1812
|
+
fact: "Roughly 58% of deposits came from digital-currency EXCHANGES alone \u2014 a handful of correlated crypto counterparties whose own troubles would pull deposits out together.",
|
|
1813
|
+
expected: [
|
|
1814
|
+
{
|
|
1815
|
+
label: "deposits concentrated in crypto exchanges",
|
|
1816
|
+
anyOf: [
|
|
1817
|
+
"digital currency exchange",
|
|
1818
|
+
"crypto exchange",
|
|
1819
|
+
"exchanges represent",
|
|
1820
|
+
"counterpart",
|
|
1821
|
+
"concentrat",
|
|
1822
|
+
"58%",
|
|
1823
|
+
"58 percent",
|
|
1824
|
+
"approximately 58"
|
|
1825
|
+
]
|
|
1826
|
+
}
|
|
1827
|
+
],
|
|
1828
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
1829
|
+
evidence: '"Deposits from digital currency exchanges represent approximately 58%" of deposits.'
|
|
1830
|
+
},
|
|
1831
|
+
{
|
|
1832
|
+
id: "SI/f3",
|
|
1833
|
+
lens: "concentration",
|
|
1834
|
+
fact: "The entire deposit franchise was tied to a single, volatile industry \u2014 digital-currency (crypto) customers \u2014 so a crypto downturn was a direct, undiversified funding shock.",
|
|
1835
|
+
expected: [
|
|
1836
|
+
{
|
|
1837
|
+
// The buried, depth signal is the CONCENTRATION framing — that the
|
|
1838
|
+
// whole deposit base is one undiversified industry bet — NOT the bare
|
|
1839
|
+
// fact that it banks crypto (a one-line ticker summary has that). So
|
|
1840
|
+
// bare "crypto" / "digital asset" are excluded; the load-bearing
|
|
1841
|
+
// tokens are the concentration / single-industry / undiversified
|
|
1842
|
+
// characterization or the filing's own "digital currency customers".
|
|
1843
|
+
label: 'single-industry deposit CONCENTRATION (not just "it banks crypto")',
|
|
1844
|
+
anyOf: [
|
|
1845
|
+
"digital currency customers",
|
|
1846
|
+
"single industry",
|
|
1847
|
+
"one industry",
|
|
1848
|
+
"single volatile industry",
|
|
1849
|
+
"sector concentration",
|
|
1850
|
+
"undiversified",
|
|
1851
|
+
"concentrat"
|
|
1852
|
+
]
|
|
1853
|
+
}
|
|
1854
|
+
],
|
|
1855
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
1856
|
+
evidence: `The 10-K's strategy and risk factors center the bank on "digital currency customers" and "the concentration of our deposits" in that single industry.`
|
|
1857
|
+
},
|
|
1858
|
+
{
|
|
1859
|
+
id: "SI/f4",
|
|
1860
|
+
lens: "liquidity",
|
|
1861
|
+
fact: "Total deposits had ballooned to $14.3B (from $10.4B) \u2014 fast, hot-money growth from the crypto boom that could reverse just as fast.",
|
|
1862
|
+
expected: [
|
|
1863
|
+
{
|
|
1864
|
+
// The depth signal is the SIZE/character of the deposit base — the
|
|
1865
|
+
// specific $14.3B figure or the explicit hot-money / volatile-deposit
|
|
1866
|
+
// characterization — NOT bare "total deposits" / "grew rapidly", which
|
|
1867
|
+
// any growth-story summary trips. Those generic phrases are excluded.
|
|
1868
|
+
label: "specific hot-money deposit base ($14.3B / volatile)",
|
|
1869
|
+
anyOf: [
|
|
1870
|
+
"hot money",
|
|
1871
|
+
"hot-money",
|
|
1872
|
+
"volatile deposit",
|
|
1873
|
+
"could reverse",
|
|
1874
|
+
"14.3 billion",
|
|
1875
|
+
"14,290",
|
|
1876
|
+
"$14.3b",
|
|
1877
|
+
"$14 billion"
|
|
1878
|
+
]
|
|
1879
|
+
}
|
|
1880
|
+
],
|
|
1881
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
1882
|
+
evidence: '"Total deposits $ 14,290,628" ... prior year "$ 10,411,278" ($ thousands).'
|
|
1883
|
+
},
|
|
1884
|
+
{
|
|
1885
|
+
id: "SI/f5",
|
|
1886
|
+
lens: "concentration",
|
|
1887
|
+
fact: "The franchise hinged on a single proprietary product \u2014 the Silvergate Exchange Network (SEN), a payment network built exclusively for the digital-currency industry \u2014 so its competitive moat and its deposit base were the SAME crypto-dependent bet, not two diversified ones.",
|
|
1888
|
+
expected: [
|
|
1889
|
+
{
|
|
1890
|
+
// The depth signal is naming the SPECIFIC proprietary product (the
|
|
1891
|
+
// Silvergate Exchange Network / SEN) and that the moat and the deposit
|
|
1892
|
+
// base are the same single bet — NOT bare "proprietary" / "payment
|
|
1893
|
+
// network", which a generic crypto-bank summary mentions. Those bare
|
|
1894
|
+
// terms are excluded; the SEN name or the single-product framing is
|
|
1895
|
+
// load-bearing.
|
|
1896
|
+
label: 'names the SEN single-product dependence (not generic "payment network")',
|
|
1897
|
+
anyOf: [
|
|
1898
|
+
"silvergate exchange network",
|
|
1899
|
+
"the sen",
|
|
1900
|
+
"sen)",
|
|
1901
|
+
"sen'",
|
|
1902
|
+
"single product",
|
|
1903
|
+
"single-product",
|
|
1904
|
+
"core product",
|
|
1905
|
+
"one product",
|
|
1906
|
+
"same bet"
|
|
1907
|
+
]
|
|
1908
|
+
}
|
|
1909
|
+
],
|
|
1910
|
+
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
1911
|
+
evidence: `"Silvergate Exchange Network ('SEN'), our proprietary, virtually instantaneous payment network for participants in the digital currency industry" \u2014 the bank's differentiator and its deposit magnet are the same crypto-only product.`
|
|
1912
|
+
}
|
|
1913
|
+
]
|
|
1291
1914
|
}
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1915
|
+
];
|
|
1916
|
+
function gradeFactAgainstText(fact, thesisText) {
|
|
1917
|
+
const haystack = thesisText.toLowerCase();
|
|
1918
|
+
const found = fact.expected.filter(
|
|
1919
|
+
(group) => group.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase()))
|
|
1920
|
+
);
|
|
1921
|
+
const minGroups = fact.minGroups ?? fact.expected.length;
|
|
1922
|
+
return {
|
|
1923
|
+
surfaced: found.length >= minGroups,
|
|
1924
|
+
groupsFound: found.length,
|
|
1925
|
+
groupsTotal: fact.expected.length,
|
|
1926
|
+
foundLabels: found.map((group) => group.label)
|
|
1297
1927
|
};
|
|
1298
|
-
switch (kindPart) {
|
|
1299
|
-
case "wiki":
|
|
1300
|
-
return wikiProposal(finding, locus, baseMeta);
|
|
1301
|
-
case "claim":
|
|
1302
|
-
return claimProposal(finding, locus, baseMeta);
|
|
1303
|
-
case "raw":
|
|
1304
|
-
return liftRawProposal(finding, locus, baseMeta);
|
|
1305
|
-
case "stale":
|
|
1306
|
-
return markStaleProposal(finding, locus, baseMeta);
|
|
1307
|
-
default:
|
|
1308
|
-
throw new KnowledgeProposalParseError(
|
|
1309
|
-
finding.finding_id,
|
|
1310
|
-
finding.subject,
|
|
1311
|
-
`unknown kind "${kindPart}" (expected one of: wiki | claim | raw | stale)`
|
|
1312
|
-
);
|
|
1313
|
-
}
|
|
1314
1928
|
}
|
|
1315
|
-
function
|
|
1316
|
-
const
|
|
1317
|
-
const pageSlug = hashIdx >= 0 ? locus.slice(0, hashIdx) : locus;
|
|
1318
|
-
const heading = hashIdx >= 0 ? locus.slice(hashIdx + 1) : null;
|
|
1319
|
-
const path = `knowledge/${ensureSlug(pageSlug)}.md`;
|
|
1320
|
-
const body = renderWikiBody(finding, pageSlug, heading);
|
|
1929
|
+
function gradeCompanyAgainstText(company, thesisText) {
|
|
1930
|
+
const perFact = company.facts.map((fact) => gradeFactAgainstText(fact, thesisText));
|
|
1321
1931
|
return {
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
locus: pageSlug,
|
|
1326
|
-
writeBlocks: [{ path, content: body }],
|
|
1327
|
-
metadata
|
|
1932
|
+
surfaced: perFact.filter((result) => result.surfaced).length,
|
|
1933
|
+
total: company.facts.length,
|
|
1934
|
+
perFact
|
|
1328
1935
|
};
|
|
1329
1936
|
}
|
|
1330
|
-
function
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
text: finding.recommended_action ?? finding.claim,
|
|
1339
|
-
refs,
|
|
1340
|
-
confidence: finding.confidence,
|
|
1341
|
-
status: "draft",
|
|
1342
|
-
metadata: {
|
|
1343
|
-
analyst_id: finding.analyst_id,
|
|
1344
|
-
source_finding_id: finding.finding_id,
|
|
1345
|
-
topic: locus
|
|
1937
|
+
function totalMaterialFacts(set = investmentThesisSet) {
|
|
1938
|
+
return set.reduce((sum, company) => sum + company.facts.length, 0);
|
|
1939
|
+
}
|
|
1940
|
+
function lensDistribution(set = investmentThesisSet) {
|
|
1941
|
+
const dist = {};
|
|
1942
|
+
for (const company of set) {
|
|
1943
|
+
for (const fact of company.facts) {
|
|
1944
|
+
dist[fact.lens] = (dist[fact.lens] ?? 0) + 1;
|
|
1346
1945
|
}
|
|
1347
|
-
}
|
|
1946
|
+
}
|
|
1947
|
+
return dist;
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
// src/investment-thesis-task.ts
|
|
1951
|
+
import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
|
|
1952
|
+
import { join as join2 } from "path";
|
|
1953
|
+
|
|
1954
|
+
// src/material-facts-metric.ts
|
|
1955
|
+
function kbIndexToText(index) {
|
|
1956
|
+
const pageText = index.pages.map((page) => `${page.title}
|
|
1957
|
+
${page.text}`).join("\n\n");
|
|
1958
|
+
const sourceText = index.sources.map((source) => `${source.title ?? ""}
|
|
1959
|
+
${source.text ?? ""}`).join("\n\n");
|
|
1960
|
+
return `${pageText}
|
|
1961
|
+
|
|
1962
|
+
${sourceText}`;
|
|
1963
|
+
}
|
|
1964
|
+
function materialFactsSurfacedInText(company, kbText) {
|
|
1965
|
+
const grade = gradeCompanyAgainstText(company, kbText);
|
|
1966
|
+
const perFact = company.facts.map((fact) => {
|
|
1967
|
+
const r = gradeFactAgainstText(fact, kbText);
|
|
1968
|
+
return {
|
|
1969
|
+
id: fact.id,
|
|
1970
|
+
lens: fact.lens,
|
|
1971
|
+
surfaced: r.surfaced,
|
|
1972
|
+
groupsFound: r.groupsFound,
|
|
1973
|
+
groupsTotal: r.groupsTotal,
|
|
1974
|
+
foundLabels: r.foundLabels
|
|
1975
|
+
};
|
|
1976
|
+
});
|
|
1348
1977
|
return {
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
metadata
|
|
1978
|
+
ticker: company.ticker,
|
|
1979
|
+
company: company.company,
|
|
1980
|
+
surfaced: grade.surfaced,
|
|
1981
|
+
total: grade.total,
|
|
1982
|
+
fraction: grade.total === 0 ? 0 : grade.surfaced / grade.total,
|
|
1983
|
+
perFact
|
|
1356
1984
|
};
|
|
1357
1985
|
}
|
|
1358
|
-
function
|
|
1359
|
-
const
|
|
1360
|
-
|
|
1361
|
-
"---",
|
|
1362
|
-
`title: ${sourceId}`,
|
|
1363
|
-
`source: ${sourceId}`,
|
|
1364
|
-
`status: draft`,
|
|
1365
|
-
`lifted_from_finding: ${finding.finding_id}`,
|
|
1366
|
-
"---",
|
|
1367
|
-
"",
|
|
1368
|
-
"## Why this page exists",
|
|
1369
|
-
"",
|
|
1370
|
-
finding.claim,
|
|
1371
|
-
"",
|
|
1372
|
-
...finding.rationale ? ["## Rationale", "", finding.rationale, ""] : [],
|
|
1373
|
-
...finding.recommended_action ? ["## Recommended action", "", finding.recommended_action, ""] : []
|
|
1374
|
-
].join("\n");
|
|
1375
|
-
return {
|
|
1376
|
-
id: `prop-${finding.finding_id}`,
|
|
1377
|
-
sourceFindingId: finding.finding_id,
|
|
1378
|
-
kind: "lift-raw",
|
|
1379
|
-
locus: sourceId,
|
|
1380
|
-
writeBlocks: [{ path, content: body }],
|
|
1381
|
-
metadata
|
|
1382
|
-
};
|
|
1383
|
-
}
|
|
1384
|
-
function markStaleProposal(finding, pageSlug, metadata) {
|
|
1385
|
-
const path = `knowledge/${ensureSlug(pageSlug)}.stale.md`;
|
|
1386
|
-
const body = [
|
|
1387
|
-
"---",
|
|
1388
|
-
`title: ${pageSlug} (marked stale)`,
|
|
1389
|
-
`status: superseded`,
|
|
1390
|
-
`superseded_by_finding: ${finding.finding_id}`,
|
|
1391
|
-
`confidence: ${finding.confidence}`,
|
|
1392
|
-
"---",
|
|
1393
|
-
"",
|
|
1394
|
-
"## Why marked stale",
|
|
1395
|
-
"",
|
|
1396
|
-
finding.claim,
|
|
1397
|
-
"",
|
|
1398
|
-
...finding.rationale ? ["## Evidence", "", finding.rationale, ""] : [],
|
|
1399
|
-
...finding.recommended_action ? ["## Action", "", finding.recommended_action, ""] : []
|
|
1400
|
-
].join("\n");
|
|
1401
|
-
return {
|
|
1402
|
-
id: `prop-${finding.finding_id}`,
|
|
1403
|
-
sourceFindingId: finding.finding_id,
|
|
1404
|
-
kind: "mark-stale",
|
|
1405
|
-
locus: pageSlug,
|
|
1406
|
-
writeBlocks: [{ path, content: body }],
|
|
1407
|
-
metadata
|
|
1408
|
-
};
|
|
1409
|
-
}
|
|
1410
|
-
function proposeFromFindings(findings) {
|
|
1411
|
-
const proposals = [];
|
|
1412
|
-
const errors = [];
|
|
1413
|
-
let skipped = 0;
|
|
1414
|
-
for (const f of findings) {
|
|
1415
|
-
try {
|
|
1416
|
-
const p = proposeFromFinding(f);
|
|
1417
|
-
if (p) proposals.push(p);
|
|
1418
|
-
else skipped += 1;
|
|
1419
|
-
} catch (err) {
|
|
1420
|
-
if (err instanceof KnowledgeProposalParseError) errors.push(err);
|
|
1421
|
-
else throw err;
|
|
1422
|
-
}
|
|
1423
|
-
}
|
|
1424
|
-
return { proposals, skipped, errors };
|
|
1425
|
-
}
|
|
1426
|
-
function ensureSlug(s) {
|
|
1427
|
-
return s.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 200) || "untitled";
|
|
1428
|
-
}
|
|
1429
|
-
function renderWikiBody(finding, slug, heading) {
|
|
1430
|
-
const title = humanize(slug);
|
|
1431
|
-
if (heading) {
|
|
1432
|
-
return [
|
|
1433
|
-
`## ${heading}`,
|
|
1434
|
-
"",
|
|
1435
|
-
finding.claim,
|
|
1436
|
-
"",
|
|
1437
|
-
...finding.rationale ? ["### Rationale", "", finding.rationale, ""] : [],
|
|
1438
|
-
...finding.recommended_action ? ["### Action", "", finding.recommended_action, ""] : [],
|
|
1439
|
-
`_Drafted from finding ${finding.finding_id} (confidence ${finding.confidence.toFixed(2)})._`
|
|
1440
|
-
].join("\n");
|
|
1441
|
-
}
|
|
1442
|
-
return [
|
|
1443
|
-
"---",
|
|
1444
|
-
`title: ${title}`,
|
|
1445
|
-
`status: draft`,
|
|
1446
|
-
`drafted_from_finding: ${finding.finding_id}`,
|
|
1447
|
-
`confidence: ${finding.confidence}`,
|
|
1448
|
-
"---",
|
|
1449
|
-
"",
|
|
1450
|
-
`# ${title}`,
|
|
1451
|
-
"",
|
|
1452
|
-
finding.claim,
|
|
1453
|
-
"",
|
|
1454
|
-
...finding.rationale ? ["## Rationale", "", finding.rationale, ""] : [],
|
|
1455
|
-
...finding.recommended_action ? ["## Recommended action", "", finding.recommended_action, ""] : []
|
|
1456
|
-
].join("\n");
|
|
1457
|
-
}
|
|
1458
|
-
function humanize(slug) {
|
|
1459
|
-
return slug.split("-").filter(Boolean).map((w) => w[0]?.toUpperCase() + w.slice(1)).join(" ") || slug;
|
|
1460
|
-
}
|
|
1461
|
-
|
|
1462
|
-
// src/release.ts
|
|
1463
|
-
import {
|
|
1464
|
-
evaluateReleaseConfidence,
|
|
1465
|
-
validateRunRecord
|
|
1466
|
-
} from "@tangle-network/agent-eval";
|
|
1467
|
-
function knowledgeReleaseReport(input) {
|
|
1468
|
-
const baselineRuns = input.baselineRuns ?? [];
|
|
1469
|
-
const runRecords = [...input.candidateRuns, ...baselineRuns].map(validateRunRecord);
|
|
1470
|
-
const scorecard = evaluateReleaseConfidence({
|
|
1471
|
-
target: "agent-knowledge-base",
|
|
1472
|
-
candidateId: input.candidateId,
|
|
1473
|
-
baselineId: input.baselineId ?? "baseline",
|
|
1474
|
-
traces: input.traces ?? [],
|
|
1475
|
-
runs: runRecords,
|
|
1476
|
-
gateDecision: input.gateDecision ?? null,
|
|
1477
|
-
thresholds: {
|
|
1478
|
-
requireCorpus: false,
|
|
1479
|
-
// The report gates on RunRecord-level outcomes, not on a separate scenario
|
|
1480
|
-
// corpus (KnowledgeReleaseInput carries no dataset/scenarios). The scenario
|
|
1481
|
-
// floor must therefore be 0 — otherwise the corpus axis fails closed on a
|
|
1482
|
-
// dimension the report has no way to satisfy, masking the run-based gate.
|
|
1483
|
-
minScenarioCount: 0,
|
|
1484
|
-
requireHoldout: input.hasHoldout ?? false,
|
|
1485
|
-
minHoldoutRuns: input.hasHoldout ? 1 : 0,
|
|
1486
|
-
minSearchRuns: 1,
|
|
1487
|
-
minMeanScore: input.minScore ?? 0.7
|
|
1488
|
-
}
|
|
1489
|
-
});
|
|
1490
|
-
const release = {
|
|
1491
|
-
id: stableId("krel", `${input.candidateId}:${input.createdAt ?? (/* @__PURE__ */ new Date()).toISOString()}`),
|
|
1492
|
-
candidateId: input.candidateId,
|
|
1493
|
-
createdAt: input.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
1494
|
-
promoted: scorecard.status !== "fail" && (input.promotedIsBest ?? true),
|
|
1495
|
-
scorecard,
|
|
1496
|
-
runRecordIds: runRecords.map((record) => record.runId)
|
|
1497
|
-
};
|
|
1498
|
-
return { release, scorecard, candidateRuns: input.candidateRuns, baselineRuns };
|
|
1986
|
+
async function materialFactsSurfaced(kb, checklist) {
|
|
1987
|
+
const index = typeof kb === "string" ? await buildKnowledgeIndex(kb) : kb;
|
|
1988
|
+
return materialFactsSurfacedInText(checklist, kbIndexToText(index));
|
|
1499
1989
|
}
|
|
1500
1990
|
|
|
1501
|
-
// src/research-loop.ts
|
|
1502
|
-
import {
|
|
1503
|
-
blockingKnowledgeEval,
|
|
1504
|
-
objectiveEval
|
|
1505
|
-
} from "@tangle-network/agent-eval";
|
|
1506
|
-
|
|
1507
1991
|
// src/readiness-helpers.ts
|
|
1508
1992
|
function readinessFor(options, index) {
|
|
1509
1993
|
if (!options.readinessSpecs?.length) return void 0;
|
|
@@ -1515,228 +1999,26 @@ function readinessFor(options, index) {
|
|
|
1515
1999
|
});
|
|
1516
2000
|
}
|
|
1517
2001
|
|
|
1518
|
-
// src/research-loop.ts
|
|
1519
|
-
function
|
|
1520
|
-
|
|
1521
|
-
const appliedSteps = [];
|
|
1522
|
-
return {
|
|
1523
|
-
intent: options.goal,
|
|
1524
|
-
async observe({ history, abortSignal }) {
|
|
1525
|
-
if (abortSignal.aborted) throw new Error("Knowledge control loop aborted");
|
|
1526
|
-
if (!initialized) {
|
|
1527
|
-
await initKnowledgeBase(options.root);
|
|
1528
|
-
initialized = true;
|
|
1529
|
-
}
|
|
1530
|
-
const index = await buildKnowledgeIndex(options.root);
|
|
1531
|
-
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
1532
|
-
const lintFindings = lintKnowledgeIndex(index);
|
|
1533
|
-
const readiness = readinessFor(options, index);
|
|
1534
|
-
return {
|
|
1535
|
-
root: options.root,
|
|
1536
|
-
goal: options.goal,
|
|
1537
|
-
iteration: history.length + 1,
|
|
1538
|
-
index,
|
|
1539
|
-
lintFindings,
|
|
1540
|
-
validation,
|
|
1541
|
-
readiness,
|
|
1542
|
-
previousSteps: [...appliedSteps],
|
|
1543
|
-
signal: abortSignal
|
|
1544
|
-
};
|
|
1545
|
-
},
|
|
1546
|
-
validate({ state }) {
|
|
1547
|
-
const errorFindings = state.validation.findings.filter(
|
|
1548
|
-
(finding) => finding.severity === "error"
|
|
1549
|
-
);
|
|
1550
|
-
const evals = [
|
|
1551
|
-
objectiveEval({
|
|
1552
|
-
id: "knowledge-valid",
|
|
1553
|
-
passed: state.validation.ok,
|
|
1554
|
-
severity: "critical",
|
|
1555
|
-
detail: state.validation.ok ? "Knowledge index is valid." : "Knowledge index has validation errors.",
|
|
1556
|
-
metadata: { findings: state.validation.findings }
|
|
1557
|
-
}),
|
|
1558
|
-
objectiveEval({
|
|
1559
|
-
id: "knowledge-lint-errors",
|
|
1560
|
-
passed: errorFindings.length === 0,
|
|
1561
|
-
severity: "error",
|
|
1562
|
-
detail: errorFindings.length === 0 ? "No lint errors." : `${errorFindings.length} lint error(s).`,
|
|
1563
|
-
metadata: { findings: errorFindings }
|
|
1564
|
-
})
|
|
1565
|
-
];
|
|
1566
|
-
if (state.readiness) evals.push(blockingKnowledgeEval(state.readiness.report));
|
|
1567
|
-
return evals;
|
|
1568
|
-
},
|
|
1569
|
-
shouldStop() {
|
|
1570
|
-
return { stop: false, pass: false, reason: "knowledge driver owns stop decisions" };
|
|
1571
|
-
},
|
|
1572
|
-
async act(action, ctx) {
|
|
1573
|
-
const step = await applyKnowledgeResearchDecision(options, action, ctx.state.iteration);
|
|
1574
|
-
appliedSteps.push(step);
|
|
1575
|
-
return step;
|
|
1576
|
-
}
|
|
1577
|
-
};
|
|
1578
|
-
}
|
|
1579
|
-
async function runKnowledgeResearchLoop(options) {
|
|
1580
|
-
const maxIterations = Math.max(1, options.maxIterations ?? 3);
|
|
2002
|
+
// src/two-agent-research-loop.ts
|
|
2003
|
+
async function runVerifiedResearchLoop(options) {
|
|
2004
|
+
const maxRounds = Math.max(1, options.maxRounds ?? 3);
|
|
1581
2005
|
await initKnowledgeBase(options.root);
|
|
1582
2006
|
const steps = [];
|
|
1583
2007
|
let index = await buildKnowledgeIndex(options.root);
|
|
1584
|
-
let validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
1585
|
-
let lintFindings = lintKnowledgeIndex(index);
|
|
1586
2008
|
let readiness = readinessFor(options, index);
|
|
1587
|
-
let
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
2009
|
+
let ready = isReady(readiness?.report);
|
|
2010
|
+
let steer;
|
|
2011
|
+
for (let round2 = 1; round2 <= maxRounds && !ready; round2++) {
|
|
2012
|
+
if (options.signal?.aborted) throw new Error("Two-agent research loop aborted");
|
|
2013
|
+
const gaps = gapsFromReadiness(readiness);
|
|
2014
|
+
const workerContribution = await options.worker({
|
|
1591
2015
|
root: options.root,
|
|
1592
2016
|
goal: options.goal,
|
|
1593
|
-
|
|
2017
|
+
round: round2,
|
|
1594
2018
|
index,
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
readiness,
|
|
1598
|
-
previousSteps: steps,
|
|
1599
|
-
signal: options.signal
|
|
1600
|
-
});
|
|
1601
|
-
done = Boolean(decision.done);
|
|
1602
|
-
const step = await applyKnowledgeResearchDecision(options, decision, iteration);
|
|
1603
|
-
index = await buildKnowledgeIndex(options.root);
|
|
1604
|
-
validation = step.validation;
|
|
1605
|
-
lintFindings = step.lintFindings;
|
|
1606
|
-
readiness = step.readiness;
|
|
1607
|
-
steps.push(step);
|
|
1608
|
-
await options.onStep?.(step);
|
|
1609
|
-
if (done) break;
|
|
1610
|
-
if (!step.applied && step.addedSources.length === 0) break;
|
|
1611
|
-
}
|
|
1612
|
-
return {
|
|
1613
|
-
root: options.root,
|
|
1614
|
-
goal: options.goal,
|
|
1615
|
-
iterations: steps.length,
|
|
1616
|
-
done,
|
|
1617
|
-
index,
|
|
1618
|
-
lintFindings,
|
|
1619
|
-
validation,
|
|
1620
|
-
readiness,
|
|
1621
|
-
steps
|
|
1622
|
-
};
|
|
1623
|
-
}
|
|
1624
|
-
async function applyKnowledgeResearchDecision(options, decision, iteration = 1) {
|
|
1625
|
-
const addedSources = [];
|
|
1626
|
-
for (const sourcePath of decision.sourcePaths ?? []) {
|
|
1627
|
-
addedSources.push(...await addSourcePath(options.root, sourcePath, options.sourceOptions));
|
|
1628
|
-
}
|
|
1629
|
-
for (const sourceText of decision.sourceTexts ?? []) {
|
|
1630
|
-
addedSources.push(await addSourceText(options.root, sourceText, options.sourceOptions));
|
|
1631
|
-
}
|
|
1632
|
-
const applied = decision.proposalText ? await applyKnowledgeWriteBlocks(options.root, decision.proposalText) : void 0;
|
|
1633
|
-
const index = await buildKnowledgeIndex(options.root);
|
|
1634
|
-
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
1635
|
-
const lintFindings = lintKnowledgeIndex(index);
|
|
1636
|
-
const readiness = readinessFor(options, index);
|
|
1637
|
-
const done = Boolean(decision.done);
|
|
1638
|
-
const event = createKnowledgeEvent({
|
|
1639
|
-
type: "research.iteration",
|
|
1640
|
-
actor: options.actor,
|
|
1641
|
-
target: options.root,
|
|
1642
|
-
metadata: {
|
|
1643
|
-
goal: options.goal,
|
|
1644
|
-
iteration,
|
|
1645
|
-
done,
|
|
1646
|
-
addedSourceCount: addedSources.length,
|
|
1647
|
-
written: applied?.written,
|
|
1648
|
-
warningCount: applied?.warnings.length ?? 0,
|
|
1649
|
-
errorCount: validation.findings.filter((finding) => finding.severity === "error").length
|
|
1650
|
-
}
|
|
1651
|
-
});
|
|
1652
|
-
return {
|
|
1653
|
-
iteration,
|
|
1654
|
-
notes: decision.notes,
|
|
1655
|
-
addedSources,
|
|
1656
|
-
applied,
|
|
1657
|
-
lintFindings,
|
|
1658
|
-
validation,
|
|
1659
|
-
readiness,
|
|
1660
|
-
event,
|
|
1661
|
-
done,
|
|
1662
|
-
metadata: decision.metadata
|
|
1663
|
-
};
|
|
1664
|
-
}
|
|
1665
|
-
|
|
1666
|
-
// src/research-supervisor.ts
|
|
1667
|
-
import {
|
|
1668
|
-
supervise
|
|
1669
|
-
} from "@tangle-network/agent-runtime/loops";
|
|
1670
|
-
var RESEARCH_SUPERVISOR_SYSTEM_PROMPT = [
|
|
1671
|
-
"You are a research SUPERVISOR. You do not answer the research question yourself \u2014",
|
|
1672
|
-
"you create and manage a team of researcher workers that grow ONE shared",
|
|
1673
|
-
"knowledge base until it is ready.",
|
|
1674
|
-
"",
|
|
1675
|
-
"Each round:",
|
|
1676
|
-
" 1. Read the goal and the gaps the readiness gate still reports.",
|
|
1677
|
-
" 2. DECOMPOSE the open work into independent sub-topics. Spawn ONE researcher",
|
|
1678
|
-
" per sub-topic (spawn_agent) \u2014 split by sub-topic, never duplicate work.",
|
|
1679
|
-
" Spawn more researchers for a broad goal, fewer for a narrow one.",
|
|
1680
|
-
" 3. Wait for the researchers to settle (await_event). Steer or re-spawn for",
|
|
1681
|
-
" the sub-topics that came back thin.",
|
|
1682
|
-
" 4. STOP as soon as the deliverable check reports the knowledge base is ready.",
|
|
1683
|
-
"",
|
|
1684
|
-
"Conserve your budget: every spawn spends from one shared pool. Prefer a small",
|
|
1685
|
-
"number of well-scoped researchers over a large fanout that re-covers the same",
|
|
1686
|
-
"ground."
|
|
1687
|
-
].join("\n");
|
|
1688
|
-
function knowledgeReadinessDeliverable(options) {
|
|
1689
|
-
return {
|
|
1690
|
-
describe: `knowledge base at ${options.root} is ready for: ${options.goal}`,
|
|
1691
|
-
async check() {
|
|
1692
|
-
const index = await buildKnowledgeIndex(options.root);
|
|
1693
|
-
const bundle = readinessFor(options, index);
|
|
1694
|
-
return (bundle?.report.blockingMissingRequirements.length ?? 0) === 0;
|
|
1695
|
-
}
|
|
1696
|
-
};
|
|
1697
|
-
}
|
|
1698
|
-
async function runResearchSupervisor(options) {
|
|
1699
|
-
await initKnowledgeBase(options.root);
|
|
1700
|
-
const { profile: workerProfile } = researcherProfile({ harness: options.harness });
|
|
1701
|
-
const baseInstructions = options.supervisorSystemPrompt ?? RESEARCH_SUPERVISOR_SYSTEM_PROMPT;
|
|
1702
|
-
const workerContract = workerProfile.prompt?.systemPrompt;
|
|
1703
|
-
const systemPrompt = workerContract ? `${baseInstructions}
|
|
1704
|
-
|
|
1705
|
-
Each researcher worker you spawn follows this contract:
|
|
1706
|
-
${workerContract}` : baseInstructions;
|
|
1707
|
-
const profile = {
|
|
1708
|
-
name: "research-supervisor",
|
|
1709
|
-
model: options.supervisorModel,
|
|
1710
|
-
systemPrompt
|
|
1711
|
-
};
|
|
1712
|
-
return supervise(profile, options.goal, {
|
|
1713
|
-
...options.superviseOptions,
|
|
1714
|
-
budget: options.budget,
|
|
1715
|
-
backend: options.backend,
|
|
1716
|
-
deliverable: knowledgeReadinessDeliverable(options)
|
|
1717
|
-
});
|
|
1718
|
-
}
|
|
1719
|
-
|
|
1720
|
-
// src/two-agent-research-loop.ts
|
|
1721
|
-
async function runTwoAgentResearchLoop(options) {
|
|
1722
|
-
const maxRounds = Math.max(1, options.maxRounds ?? 3);
|
|
1723
|
-
await initKnowledgeBase(options.root);
|
|
1724
|
-
const steps = [];
|
|
1725
|
-
let index = await buildKnowledgeIndex(options.root);
|
|
1726
|
-
let readiness = readinessFor(options, index);
|
|
1727
|
-
let ready = isReady(readiness?.report);
|
|
1728
|
-
let steer;
|
|
1729
|
-
for (let round2 = 1; round2 <= maxRounds && !ready; round2++) {
|
|
1730
|
-
if (options.signal?.aborted) throw new Error("Two-agent research loop aborted");
|
|
1731
|
-
const gaps = gapsFromReadiness(readiness);
|
|
1732
|
-
const workerContribution = await options.worker({
|
|
1733
|
-
root: options.root,
|
|
1734
|
-
goal: options.goal,
|
|
1735
|
-
round: round2,
|
|
1736
|
-
index,
|
|
1737
|
-
gaps,
|
|
1738
|
-
steer,
|
|
1739
|
-
readiness: requireReadiness(readiness, options),
|
|
2019
|
+
gaps,
|
|
2020
|
+
steer,
|
|
2021
|
+
readiness: requireReadiness(readiness, options),
|
|
1740
2022
|
signal: options.signal
|
|
1741
2023
|
});
|
|
1742
2024
|
const accepted = [];
|
|
@@ -1912,12 +2194,2481 @@ ${source.text}`.toLowerCase();
|
|
|
1912
2194
|
}
|
|
1913
2195
|
return hits;
|
|
1914
2196
|
}
|
|
2197
|
+
var runTwoAgentResearchLoop = runVerifiedResearchLoop;
|
|
2198
|
+
|
|
2199
|
+
// src/investment-thesis-task.ts
|
|
2200
|
+
function thesisReadinessSpecs(input) {
|
|
2201
|
+
const c = input.company;
|
|
2202
|
+
const t = input.ticker;
|
|
2203
|
+
const filing = `${c} ${t} SEC 10-K annual report SEC.gov EDGAR filing`;
|
|
2204
|
+
return [
|
|
2205
|
+
defineReadinessSpec({
|
|
2206
|
+
id: "thesis/filing",
|
|
2207
|
+
description: `the most recent SEC 10-K annual report for ${c} (${t}) filed on or before ${input.cutoff}, from SEC EDGAR`,
|
|
2208
|
+
query: filing,
|
|
2209
|
+
requiredFor: ["ResearchAgent"],
|
|
2210
|
+
importance: "blocking",
|
|
2211
|
+
minSources: 2,
|
|
2212
|
+
minHits: 1
|
|
2213
|
+
}),
|
|
2214
|
+
defineReadinessSpec({
|
|
2215
|
+
id: "thesis/balance-sheet",
|
|
2216
|
+
description: `${c} balance-sheet risks: securities marked below cost, unrealized losses, leverage / total debt, debt maturities, interest expense`,
|
|
2217
|
+
query: `${c} ${t} 10-K balance sheet total debt unrealized losses interest expense leverage`,
|
|
2218
|
+
requiredFor: ["ResearchAgent"],
|
|
2219
|
+
importance: "blocking",
|
|
2220
|
+
minSources: 2,
|
|
2221
|
+
minHits: 1
|
|
2222
|
+
}),
|
|
2223
|
+
defineReadinessSpec({
|
|
2224
|
+
id: "thesis/concentration-liquidity",
|
|
2225
|
+
description: `${c} concentration + liquidity: customer / deposit / revenue concentration, uninsured deposits, operating cash flow, net loss, inventory, equity erosion`,
|
|
2226
|
+
query: `${c} ${t} 10-K customer deposit concentration operating cash flow net loss inventory`,
|
|
2227
|
+
requiredFor: ["ResearchAgent"],
|
|
2228
|
+
importance: "blocking",
|
|
2229
|
+
minSources: 2,
|
|
2230
|
+
minHits: 1
|
|
2231
|
+
}),
|
|
2232
|
+
defineReadinessSpec({
|
|
2233
|
+
id: "thesis/governance-regulatory",
|
|
2234
|
+
description: `${c} governance + regulatory: related-party transactions, dual-class / super-voting control, buybacks / dividends, recalls, regulatory or legal exposure, margin trends`,
|
|
2235
|
+
query: `${c} ${t} 10-K related party dual class share repurchase recall regulatory gross margin`,
|
|
2236
|
+
requiredFor: ["ResearchAgent"],
|
|
2237
|
+
importance: "blocking",
|
|
2238
|
+
minSources: 2,
|
|
2239
|
+
minHits: 1
|
|
2240
|
+
})
|
|
2241
|
+
];
|
|
2242
|
+
}
|
|
2243
|
+
function thesisSynthesisMessages(input, kbText) {
|
|
2244
|
+
const system = "You are a buy-side investment analyst writing a thesis memo. You are given the raw research your team gathered from public filings (SEC 10-K) and the web. Write a thesis that a thorough analyst would write: lead with your JUDGMENT, then the KEY DRIVERS, then the MATERIAL RISKS. Be specific and quantitative \u2014 name the actual figures, balance-sheet items, concentrations, leverage, margin trends, governance items, and regulatory exposures that appear in the research. Surface the buried, non-obvious drivers a one-line ticker summary misses. Ground every claim in the research provided; do NOT invent figures. If the research does not contain a figure, do not state it.";
|
|
2245
|
+
const user = [
|
|
2246
|
+
`Company: ${input.company} (${input.ticker})`,
|
|
2247
|
+
`As-of date (reason as if it is this date): ${input.cutoff}`,
|
|
2248
|
+
input.sector ? `Sector: ${input.sector}` : "",
|
|
2249
|
+
"",
|
|
2250
|
+
"Research gathered (filings + web excerpts):",
|
|
2251
|
+
'"""',
|
|
2252
|
+
kbText.slice(0, 24e3),
|
|
2253
|
+
'"""',
|
|
2254
|
+
"",
|
|
2255
|
+
"Write the investment thesis now. Structure:",
|
|
2256
|
+
"## Judgment",
|
|
2257
|
+
"## Key drivers",
|
|
2258
|
+
"## Material risks"
|
|
2259
|
+
].filter(Boolean).join("\n");
|
|
2260
|
+
return [
|
|
2261
|
+
{ role: "system", content: system },
|
|
2262
|
+
{ role: "user", content: user }
|
|
2263
|
+
];
|
|
2264
|
+
}
|
|
2265
|
+
async function writeThesisPage(root, input, thesis) {
|
|
2266
|
+
const { knowledgeDir } = layoutFor(root);
|
|
2267
|
+
await mkdir2(knowledgeDir, { recursive: true });
|
|
2268
|
+
const path = join2(knowledgeDir, `thesis-${input.ticker.toLowerCase()}.md`);
|
|
2269
|
+
const body = [
|
|
2270
|
+
"---",
|
|
2271
|
+
`title: Investment thesis \u2014 ${input.company} (${input.ticker})`,
|
|
2272
|
+
`ticker: ${input.ticker}`,
|
|
2273
|
+
`cutoff: ${input.cutoff}`,
|
|
2274
|
+
"kind: investment-thesis",
|
|
2275
|
+
"---",
|
|
2276
|
+
`# Investment thesis \u2014 ${input.company} (${input.ticker}), as of ${input.cutoff}`,
|
|
2277
|
+
"",
|
|
2278
|
+
thesis.trim(),
|
|
2279
|
+
""
|
|
2280
|
+
].join("\n");
|
|
2281
|
+
await writeFile2(path, body, "utf8");
|
|
2282
|
+
return path;
|
|
2283
|
+
}
|
|
2284
|
+
async function runInvestmentThesisTask(input, options) {
|
|
2285
|
+
const worker = createWebResearchWorker({ ...options.workerOptions, router: options.router });
|
|
2286
|
+
const goal = `${input.company} (${input.ticker}) investment thesis as of ${input.cutoff}`;
|
|
2287
|
+
const loop = await runVerifiedResearchLoop({
|
|
2288
|
+
root: options.root,
|
|
2289
|
+
goal,
|
|
2290
|
+
worker,
|
|
2291
|
+
driver: options.driver,
|
|
2292
|
+
readinessSpecs: thesisReadinessSpecs(input),
|
|
2293
|
+
maxRounds: options.maxRounds ?? 3,
|
|
2294
|
+
signal: options.signal
|
|
2295
|
+
});
|
|
2296
|
+
const index = await buildKnowledgeIndex(options.root);
|
|
2297
|
+
const kbText = kbIndexToText(index);
|
|
2298
|
+
const messages = thesisSynthesisMessages(input, kbText);
|
|
2299
|
+
const thesis = await options.router.chat(messages, options.synthesisMaxTokens ?? 1600);
|
|
2300
|
+
const thesisPath = await writeThesisPage(options.root, input, thesis);
|
|
2301
|
+
return { loop, thesis, thesisPath };
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
// src/kb-improvement.ts
|
|
2305
|
+
import { cp, mkdir as mkdir3, open, readdir, readFile as readFile2, rename, rm, stat, writeFile as writeFile3 } from "fs/promises";
|
|
2306
|
+
import { dirname as dirname2, join as join3, relative } from "path";
|
|
2307
|
+
|
|
2308
|
+
// src/rag-eval.ts
|
|
2309
|
+
var defaultThresholds = {
|
|
2310
|
+
context_precision: 0.5,
|
|
2311
|
+
context_recall: 0.8,
|
|
2312
|
+
context_relevance: 0.5,
|
|
2313
|
+
context_sufficiency: 0.8,
|
|
2314
|
+
faithfulness: 0.9,
|
|
2315
|
+
answer_relevance: 0.7,
|
|
2316
|
+
citation_support: 0.9,
|
|
2317
|
+
abstention: 1,
|
|
2318
|
+
unsupported_answer_rate: 0
|
|
2319
|
+
};
|
|
2320
|
+
var defaultWeights = {
|
|
2321
|
+
context_relevance: 1,
|
|
2322
|
+
context_sufficiency: 1,
|
|
2323
|
+
faithfulness: 2,
|
|
2324
|
+
answer_relevance: 1,
|
|
2325
|
+
citation_support: 1,
|
|
2326
|
+
abstention: 1
|
|
2327
|
+
};
|
|
2328
|
+
var metricAliases = {
|
|
2329
|
+
answer_correctness: "answer_correctness",
|
|
2330
|
+
answer_relevance: "answer_relevance",
|
|
2331
|
+
answer_relevancy: "answer_relevance",
|
|
2332
|
+
answerrelevancy: "answer_relevance",
|
|
2333
|
+
context_precision: "context_precision",
|
|
2334
|
+
context_recall: "context_recall",
|
|
2335
|
+
context_relevance: "context_relevance",
|
|
2336
|
+
context_relevancy: "context_relevance",
|
|
2337
|
+
context_sufficiency: "context_sufficiency",
|
|
2338
|
+
contextual_precision: "context_precision",
|
|
2339
|
+
contextual_recall: "context_recall",
|
|
2340
|
+
contextual_relevancy: "context_relevance",
|
|
2341
|
+
faithfulness: "faithfulness",
|
|
2342
|
+
groundedness: "groundedness",
|
|
2343
|
+
claim_recall: "context_recall",
|
|
2344
|
+
claim_precision: "faithfulness",
|
|
2345
|
+
citation_support: "citation_support",
|
|
2346
|
+
abstention: "abstention",
|
|
2347
|
+
unsupported_answer_rate: "unsupported_answer_rate"
|
|
2348
|
+
};
|
|
2349
|
+
function ragAnswerQualityJudge(options = {}) {
|
|
2350
|
+
return {
|
|
2351
|
+
name: options.name ?? "rag-answer-quality",
|
|
2352
|
+
dimensions: [
|
|
2353
|
+
{ key: "context_precision", description: "share of retrieved context that is useful" },
|
|
2354
|
+
{ key: "context_recall", description: "share of required evidence present in context" },
|
|
2355
|
+
{ key: "context_relevance", description: "context relevance to the query and answer target" },
|
|
2356
|
+
{ key: "context_sufficiency", description: "whether context is enough to answer" },
|
|
2357
|
+
{ key: "faithfulness", description: "share of answer claims supported by retrieved context" },
|
|
2358
|
+
{ key: "answer_relevance", description: "answer addresses the user query" },
|
|
2359
|
+
{ key: "answer_correctness", description: "answer contains expected claims" },
|
|
2360
|
+
{ key: "citation_support", description: "citations support the claims they cite" },
|
|
2361
|
+
{ key: "abstention", description: "answer abstains exactly when the case is unanswerable" }
|
|
2362
|
+
],
|
|
2363
|
+
appliesTo: (scenario) => scenario.kind === "rag-answer-eval",
|
|
2364
|
+
async score({ artifact, scenario }) {
|
|
2365
|
+
const summary = scoreRagAnswerArtifact(artifact, scenario, options);
|
|
2366
|
+
return {
|
|
2367
|
+
dimensions: summary.metrics,
|
|
2368
|
+
composite: summary.composite,
|
|
2369
|
+
notes: summary.findings.map((finding) => finding.message).join("; ")
|
|
2370
|
+
};
|
|
2371
|
+
}
|
|
2372
|
+
};
|
|
2373
|
+
}
|
|
2374
|
+
function scoreRagAnswerArtifact(artifact, scenario, options = {}) {
|
|
2375
|
+
const claims = normalizeClaims(artifact);
|
|
2376
|
+
const abstained = artifact.abstained ?? looksLikeAbstention(artifact.answer);
|
|
2377
|
+
const requiredTargets = requiredContextTargets(scenario);
|
|
2378
|
+
const matchedRequiredContextCount = requiredTargets.filter(
|
|
2379
|
+
(target) => artifact.contexts.some((context) => contextMatchesTarget(context, target))
|
|
2380
|
+
).length;
|
|
2381
|
+
const requiredContextCount = requiredTargets.length;
|
|
2382
|
+
const contextRecall = requiredContextCount === 0 ? neutralScore(Boolean(scenario.unanswerable) || artifact.contexts.length > 0) : matchedRequiredContextCount / requiredContextCount;
|
|
2383
|
+
const relevantContextCount = artifact.contexts.filter(
|
|
2384
|
+
(context) => contextIsRelevant(context, scenario)
|
|
2385
|
+
).length;
|
|
2386
|
+
const contextPrecision = artifact.contexts.length === 0 ? scenario.unanswerable ? 1 : 0 : relevantContextCount / artifact.contexts.length;
|
|
2387
|
+
const contextRelevance = artifact.contexts.length === 0 ? scenario.unanswerable ? 1 : 0 : average(artifact.contexts.map((context) => contextRelevanceScore(context, scenario)));
|
|
2388
|
+
const contextSufficiency = requiredContextCount === 0 ? contextRelevance : contextRecall;
|
|
2389
|
+
const support = claims.map((claim) => claimSupport(claim.text, artifact.contexts, options));
|
|
2390
|
+
const supportedClaimCount = support.filter(Boolean).length;
|
|
2391
|
+
const faithfulness = claims.length === 0 ? abstained ? 1 : 0 : supportedClaimCount / Math.max(1, claims.length);
|
|
2392
|
+
const citation = scoreCitations(claims, artifact, scenario, options);
|
|
2393
|
+
const answerRelevance = scoreAnswerRelevance(artifact, scenario, abstained);
|
|
2394
|
+
const answerCorrectness = scoreAnswerCorrectness(artifact, scenario, abstained);
|
|
2395
|
+
const abstention = scenario.unanswerable ? abstained ? 1 : 0 : abstained ? 0 : 1;
|
|
2396
|
+
const unsupportedAnswerRate = claims.length === 0 ? abstained ? 0 : 1 : 1 - supportedClaimCount / Math.max(1, claims.length);
|
|
2397
|
+
const deterministicMetrics = {
|
|
2398
|
+
context_precision: clamp01(contextPrecision),
|
|
2399
|
+
context_recall: clamp01(contextRecall),
|
|
2400
|
+
context_relevance: clamp01(contextRelevance),
|
|
2401
|
+
context_sufficiency: clamp01(contextSufficiency),
|
|
2402
|
+
faithfulness: clamp01(faithfulness),
|
|
2403
|
+
groundedness: clamp01(faithfulness),
|
|
2404
|
+
answer_relevance: clamp01(answerRelevance),
|
|
2405
|
+
answer_correctness: clamp01(answerCorrectness),
|
|
2406
|
+
citation_support: clamp01(citation.support),
|
|
2407
|
+
abstention: clamp01(abstention),
|
|
2408
|
+
unsupported_answer_rate: clamp01(unsupportedAnswerRate)
|
|
2409
|
+
};
|
|
2410
|
+
const providerScores = normalizeExternalRagScores(artifact.externalScores ?? []);
|
|
2411
|
+
const metrics = mergeMetrics(deterministicMetrics, providerScores, options.externalScorePolicy);
|
|
2412
|
+
const thresholds = { ...defaultThresholds, ...scenario.thresholds, ...options.thresholds };
|
|
2413
|
+
const findings = diagnoseRagAnswerFailure(metrics, scenario, thresholds);
|
|
2414
|
+
const composite = weightedComposite(metrics, options.weights ?? defaultWeights);
|
|
2415
|
+
return {
|
|
2416
|
+
metrics,
|
|
2417
|
+
composite,
|
|
2418
|
+
passed: findings.every(
|
|
2419
|
+
(finding) => finding.severity !== "error" && finding.severity !== "critical"
|
|
2420
|
+
),
|
|
2421
|
+
findings,
|
|
2422
|
+
claimCount: claims.length,
|
|
2423
|
+
supportedClaimCount,
|
|
2424
|
+
citedClaimCount: citation.citedClaimCount,
|
|
2425
|
+
supportedCitationCount: citation.supportedCitationCount,
|
|
2426
|
+
matchedRequiredContextCount,
|
|
2427
|
+
requiredContextCount,
|
|
2428
|
+
providerScores
|
|
2429
|
+
};
|
|
2430
|
+
}
|
|
2431
|
+
function diagnoseRagAnswerFailure(metrics, scenario, thresholds = defaultThresholds) {
|
|
2432
|
+
const findings = [];
|
|
2433
|
+
if (below(metrics.context_recall, thresholds.context_recall)) {
|
|
2434
|
+
findings.push({
|
|
2435
|
+
id: `${scenario.id}:context-recall`,
|
|
2436
|
+
kind: scenario.slices?.includes("multi-source") ? "missing-multihop-evidence" : "retrieval-miss",
|
|
2437
|
+
severity: "error",
|
|
2438
|
+
scenarioId: scenario.id,
|
|
2439
|
+
message: "Required evidence was not present in retrieved context.",
|
|
2440
|
+
evidence: { context_recall: metrics.context_recall }
|
|
2441
|
+
});
|
|
2442
|
+
}
|
|
2443
|
+
if (below(metrics.context_precision, thresholds.context_precision)) {
|
|
2444
|
+
findings.push({
|
|
2445
|
+
id: `${scenario.id}:context-precision`,
|
|
2446
|
+
kind: "retrieval-noise",
|
|
2447
|
+
severity: "warning",
|
|
2448
|
+
scenarioId: scenario.id,
|
|
2449
|
+
message: "Retrieved context contains too much irrelevant material.",
|
|
2450
|
+
evidence: { context_precision: metrics.context_precision }
|
|
2451
|
+
});
|
|
2452
|
+
}
|
|
2453
|
+
if (below(metrics.faithfulness, thresholds.faithfulness)) {
|
|
2454
|
+
findings.push({
|
|
2455
|
+
id: `${scenario.id}:faithfulness`,
|
|
2456
|
+
kind: "generator-unsupported-claim",
|
|
2457
|
+
severity: "error",
|
|
2458
|
+
scenarioId: scenario.id,
|
|
2459
|
+
message: "The answer contains claims not supported by retrieved context.",
|
|
2460
|
+
evidence: { faithfulness: metrics.faithfulness }
|
|
2461
|
+
});
|
|
2462
|
+
}
|
|
2463
|
+
if (below(metrics.citation_support, thresholds.citation_support)) {
|
|
2464
|
+
findings.push({
|
|
2465
|
+
id: `${scenario.id}:citation-support`,
|
|
2466
|
+
kind: "citation-mismatch",
|
|
2467
|
+
severity: "error",
|
|
2468
|
+
scenarioId: scenario.id,
|
|
2469
|
+
message: "Citations do not support the claims they are attached to.",
|
|
2470
|
+
evidence: { citation_support: metrics.citation_support }
|
|
2471
|
+
});
|
|
2472
|
+
}
|
|
2473
|
+
if (below(metrics.abstention, thresholds.abstention)) {
|
|
2474
|
+
findings.push({
|
|
2475
|
+
id: `${scenario.id}:abstention`,
|
|
2476
|
+
kind: "incorrect-abstention",
|
|
2477
|
+
severity: "error",
|
|
2478
|
+
scenarioId: scenario.id,
|
|
2479
|
+
message: scenario.unanswerable ? "The system answered a case marked unanswerable." : "The system abstained from an answerable case.",
|
|
2480
|
+
evidence: { abstention: metrics.abstention }
|
|
2481
|
+
});
|
|
2482
|
+
}
|
|
2483
|
+
return findings;
|
|
2484
|
+
}
|
|
2485
|
+
function createRagAnswerQualityHook(options) {
|
|
2486
|
+
return async () => {
|
|
2487
|
+
const summaries = [];
|
|
2488
|
+
const findings = [];
|
|
2489
|
+
for (const scenario of options.scenarios) {
|
|
2490
|
+
const initialArtifact = await options.run(scenario);
|
|
2491
|
+
const external = await options.externalEvaluator?.({ scenario, artifact: initialArtifact });
|
|
2492
|
+
const artifact = external ? {
|
|
2493
|
+
...initialArtifact,
|
|
2494
|
+
externalScores: [
|
|
2495
|
+
...initialArtifact.externalScores ?? [],
|
|
2496
|
+
...Array.isArray(external) ? external : [external]
|
|
2497
|
+
]
|
|
2498
|
+
} : initialArtifact;
|
|
2499
|
+
const summary = scoreRagAnswerArtifact(artifact, scenario, {
|
|
2500
|
+
thresholds: options.thresholds,
|
|
2501
|
+
weights: options.weights
|
|
2502
|
+
});
|
|
2503
|
+
summaries.push(summary);
|
|
2504
|
+
findings.push(...summary.findings);
|
|
2505
|
+
}
|
|
2506
|
+
const metrics = aggregateRagAnswerMetrics(summaries);
|
|
2507
|
+
return {
|
|
2508
|
+
passed: findings.length === 0,
|
|
2509
|
+
metrics,
|
|
2510
|
+
findings,
|
|
2511
|
+
metadata: { scenarioCount: options.scenarios.length }
|
|
2512
|
+
};
|
|
2513
|
+
};
|
|
2514
|
+
}
|
|
2515
|
+
async function calibrateRagAnswerJudge(options) {
|
|
2516
|
+
const judge = options.judge ?? ragAnswerQualityJudge();
|
|
2517
|
+
const strong = await judge.score({
|
|
2518
|
+
artifact: options.strong,
|
|
2519
|
+
scenario: options.scenario,
|
|
2520
|
+
signal: options.signal ?? new AbortController().signal
|
|
2521
|
+
});
|
|
2522
|
+
const weak = await judge.score({
|
|
2523
|
+
artifact: options.weak,
|
|
2524
|
+
scenario: options.scenario,
|
|
2525
|
+
signal: options.signal ?? new AbortController().signal
|
|
2526
|
+
});
|
|
2527
|
+
const strongScore = strong.composite;
|
|
2528
|
+
const weakScore = weak.composite;
|
|
2529
|
+
const minStrongScore = options.minStrongScore ?? 0.7;
|
|
2530
|
+
const maxWeakScore = options.maxWeakScore ?? 0.3;
|
|
2531
|
+
return {
|
|
2532
|
+
passed: strongScore >= minStrongScore && weakScore <= maxWeakScore,
|
|
2533
|
+
strongScore,
|
|
2534
|
+
weakScore,
|
|
2535
|
+
gap: strongScore - weakScore
|
|
2536
|
+
};
|
|
2537
|
+
}
|
|
2538
|
+
function normalizeExternalRagScores(scores) {
|
|
2539
|
+
const normalized = {};
|
|
2540
|
+
for (const item of scores) {
|
|
2541
|
+
const provider = item.provider || "custom";
|
|
2542
|
+
const providerScores = normalized[provider] ?? {};
|
|
2543
|
+
for (const [key, value] of Object.entries(item.scores)) {
|
|
2544
|
+
const canonical = metricAliases[normalizeMetricName(key)];
|
|
2545
|
+
if (!canonical) continue;
|
|
2546
|
+
if (Number.isFinite(value)) providerScores[canonical] = clamp01(value);
|
|
2547
|
+
}
|
|
2548
|
+
normalized[provider] = providerScores;
|
|
2549
|
+
}
|
|
2550
|
+
return normalized;
|
|
2551
|
+
}
|
|
2552
|
+
function toRagasEvaluationRows(cases) {
|
|
2553
|
+
return cases.map(({ scenario, artifact }) => ({
|
|
2554
|
+
user_input: scenario.query,
|
|
2555
|
+
response: artifact.answer,
|
|
2556
|
+
retrieved_contexts: artifact.contexts.map((context) => context.text),
|
|
2557
|
+
reference: scenario.referenceAnswer,
|
|
2558
|
+
reference_contexts: requiredContextTexts(scenario)
|
|
2559
|
+
}));
|
|
2560
|
+
}
|
|
2561
|
+
function toDeepEvalTestCases(cases) {
|
|
2562
|
+
return cases.map(({ scenario, artifact }) => ({
|
|
2563
|
+
input: scenario.query,
|
|
2564
|
+
actual_output: artifact.answer,
|
|
2565
|
+
expected_output: scenario.referenceAnswer,
|
|
2566
|
+
retrieval_context: artifact.contexts.map((context) => context.text),
|
|
2567
|
+
context: requiredContextTexts(scenario)
|
|
2568
|
+
}));
|
|
2569
|
+
}
|
|
2570
|
+
function toTruLensRecords(cases) {
|
|
2571
|
+
return cases.map(({ scenario, artifact }) => ({
|
|
2572
|
+
input: scenario.query,
|
|
2573
|
+
output: artifact.answer,
|
|
2574
|
+
context: artifact.contexts.map((context) => context.text).join("\n\n")
|
|
2575
|
+
}));
|
|
2576
|
+
}
|
|
2577
|
+
function toRagCheckerRecords(cases) {
|
|
2578
|
+
return cases.map(({ scenario, artifact }) => ({
|
|
2579
|
+
query_id: scenario.id,
|
|
2580
|
+
query: scenario.query,
|
|
2581
|
+
gt_answer: scenario.referenceAnswer,
|
|
2582
|
+
response: artifact.answer,
|
|
2583
|
+
retrieved_context: artifact.contexts.map((context) => ({
|
|
2584
|
+
doc_id: context.id,
|
|
2585
|
+
text: context.text
|
|
2586
|
+
})),
|
|
2587
|
+
claims: normalizeClaims(artifact).map((claim) => claim.text)
|
|
2588
|
+
}));
|
|
2589
|
+
}
|
|
2590
|
+
function scoreKnowledgeBaseIndex(index, options = {}) {
|
|
2591
|
+
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
2592
|
+
const lintFindings = lintKnowledgeIndex(index);
|
|
2593
|
+
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
2594
|
+
const staleSourceCount = index.sources.filter((source) => {
|
|
2595
|
+
return source.validUntil ? Date.parse(source.validUntil) < now.getTime() : false;
|
|
2596
|
+
}).length;
|
|
2597
|
+
const sourceHashCounts = /* @__PURE__ */ new Map();
|
|
2598
|
+
for (const source of index.sources) {
|
|
2599
|
+
sourceHashCounts.set(source.contentHash, (sourceHashCounts.get(source.contentHash) ?? 0) + 1);
|
|
2600
|
+
}
|
|
2601
|
+
const duplicateSourceHashCount = [...sourceHashCounts.values()].filter(
|
|
2602
|
+
(count) => count > 1
|
|
2603
|
+
).length;
|
|
2604
|
+
const pagesWithInlineCitations = index.pages.filter(
|
|
2605
|
+
(page) => sourceRefs(page.text).length > 0
|
|
2606
|
+
).length;
|
|
2607
|
+
const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length;
|
|
2608
|
+
const metrics = {
|
|
2609
|
+
page_count: index.pages.length,
|
|
2610
|
+
source_count: index.sources.length,
|
|
2611
|
+
citation_rate: index.pages.length === 0 ? 1 : pagesWithInlineCitations / index.pages.length,
|
|
2612
|
+
source_backed_page_rate: index.pages.length === 0 ? 1 : pagesWithSources / index.pages.length,
|
|
2613
|
+
stale_source_rate: index.sources.length === 0 ? 0 : staleSourceCount / index.sources.length,
|
|
2614
|
+
duplicate_source_hash_rate: index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
|
|
2615
|
+
lint_error_count: validation.findings.filter((finding) => finding.severity === "error").length,
|
|
2616
|
+
lint_warning_count: lintFindings.filter((finding) => finding.severity === "warning").length
|
|
2617
|
+
};
|
|
2618
|
+
const findings = [];
|
|
2619
|
+
if (metrics.lint_error_count > 0) {
|
|
2620
|
+
findings.push({
|
|
2621
|
+
id: "kb:lint-errors",
|
|
2622
|
+
kind: "unknown",
|
|
2623
|
+
severity: "critical",
|
|
2624
|
+
message: `${metrics.lint_error_count} validation error(s) in the knowledge index.`,
|
|
2625
|
+
evidence: { lint_error_count: metrics.lint_error_count }
|
|
2626
|
+
});
|
|
2627
|
+
}
|
|
2628
|
+
if (metrics.citation_rate < (options.minCitationRate ?? 0)) {
|
|
2629
|
+
findings.push({
|
|
2630
|
+
id: "kb:citation-rate",
|
|
2631
|
+
kind: "missing-source",
|
|
2632
|
+
severity: "error",
|
|
2633
|
+
message: "Inline citation coverage is below the configured minimum.",
|
|
2634
|
+
evidence: { citation_rate: metrics.citation_rate }
|
|
2635
|
+
});
|
|
2636
|
+
}
|
|
2637
|
+
if (metrics.stale_source_rate > (options.maxStaleSourceRate ?? 1)) {
|
|
2638
|
+
findings.push({
|
|
2639
|
+
id: "kb:stale-source-rate",
|
|
2640
|
+
kind: "stale-source",
|
|
2641
|
+
severity: "error",
|
|
2642
|
+
message: "Stale source rate exceeds the configured maximum.",
|
|
2643
|
+
evidence: { stale_source_rate: metrics.stale_source_rate }
|
|
2644
|
+
});
|
|
2645
|
+
}
|
|
2646
|
+
return { ok: findings.length === 0, metrics, findings };
|
|
2647
|
+
}
|
|
2648
|
+
function requiredContextTargets(scenario) {
|
|
2649
|
+
if (scenario.requiredContext?.length) return [...scenario.requiredContext];
|
|
2650
|
+
return (scenario.expectedClaims ?? []).map((text) => ({ text }));
|
|
2651
|
+
}
|
|
2652
|
+
function requiredContextTexts(scenario) {
|
|
2653
|
+
return requiredContextTargets(scenario).flatMap((target) => target.text ? [target.text] : []);
|
|
2654
|
+
}
|
|
2655
|
+
function contextMatchesTarget(context, target) {
|
|
2656
|
+
if (target.id && context.id === target.id) return true;
|
|
2657
|
+
if (target.pageId && context.pageId === target.pageId) return true;
|
|
2658
|
+
if (target.sourceId && context.sourceId === target.sourceId) return true;
|
|
2659
|
+
if (target.anchorId && context.anchorId === target.anchorId) return true;
|
|
2660
|
+
if (target.text && textSupportScore(target.text, context.text) >= 0.7) return true;
|
|
2661
|
+
return false;
|
|
2662
|
+
}
|
|
2663
|
+
function contextIsRelevant(context, scenario) {
|
|
2664
|
+
return contextRelevanceScore(context, scenario) >= 0.3;
|
|
2665
|
+
}
|
|
2666
|
+
function contextRelevanceScore(context, scenario) {
|
|
2667
|
+
const targets = [
|
|
2668
|
+
scenario.query,
|
|
2669
|
+
scenario.referenceAnswer,
|
|
2670
|
+
...scenario.expectedClaims ?? [],
|
|
2671
|
+
...requiredContextTexts(scenario)
|
|
2672
|
+
].filter((value) => Boolean(value?.trim()));
|
|
2673
|
+
if (targets.length === 0) return 1;
|
|
2674
|
+
return Math.max(...targets.map((target) => textSupportScore(target, context.text)));
|
|
2675
|
+
}
|
|
2676
|
+
function claimSupport(claim, contexts, options) {
|
|
2677
|
+
if (contexts.length === 0) return false;
|
|
2678
|
+
const combined = contexts.map((context) => context.text).join("\n\n");
|
|
2679
|
+
const deterministic = groundClaimInText(claim, combined, {
|
|
2680
|
+
minOverlap: options.minClaimSupport ?? 0.7
|
|
2681
|
+
});
|
|
2682
|
+
return deterministic.grounded || textSupportScore(claim, combined) >= (options.minClaimSupport ?? 0.7);
|
|
2683
|
+
}
|
|
2684
|
+
function scoreCitations(claims, artifact, scenario, options) {
|
|
2685
|
+
if (claims.length === 0)
|
|
2686
|
+
return { support: scenario.unanswerable ? 1 : 0, citedClaimCount: 0, supportedCitationCount: 0 };
|
|
2687
|
+
const citations = artifact.citations ?? [];
|
|
2688
|
+
const citedClaims = claims.filter((claim) => citationsForClaim(claim, citations).length > 0);
|
|
2689
|
+
if (citedClaims.length === 0) {
|
|
2690
|
+
return {
|
|
2691
|
+
support: scenario.requireCitations ? 0 : 1,
|
|
2692
|
+
citedClaimCount: 0,
|
|
2693
|
+
supportedCitationCount: 0
|
|
2694
|
+
};
|
|
2695
|
+
}
|
|
2696
|
+
let supportedCitationCount = 0;
|
|
2697
|
+
for (const claim of citedClaims) {
|
|
2698
|
+
const citedContexts = contextsForClaim(claim, citations, artifact.contexts);
|
|
2699
|
+
if (claimSupport(claim.text, citedContexts, options)) supportedCitationCount += 1;
|
|
2700
|
+
}
|
|
2701
|
+
return {
|
|
2702
|
+
support: supportedCitationCount / citedClaims.length,
|
|
2703
|
+
citedClaimCount: citedClaims.length,
|
|
2704
|
+
supportedCitationCount
|
|
2705
|
+
};
|
|
2706
|
+
}
|
|
2707
|
+
function citationsForClaim(claim, citations) {
|
|
2708
|
+
const claimCitationIds = new Set(claim.citationIds ?? []);
|
|
2709
|
+
return citations.filter((citation) => {
|
|
2710
|
+
return citation.claimId === claim.id || citation.id && claimCitationIds.has(citation.id);
|
|
2711
|
+
});
|
|
2712
|
+
}
|
|
2713
|
+
function contextsForClaim(claim, citations, contexts) {
|
|
2714
|
+
const matchedCitations = citationsForClaim(claim, citations);
|
|
2715
|
+
const matchedContexts = contexts.filter(
|
|
2716
|
+
(context) => matchedCitations.some((citation) => {
|
|
2717
|
+
if (citation.contextId && citation.contextId === context.id) return true;
|
|
2718
|
+
if (citation.pageId && citation.pageId === context.pageId) return true;
|
|
2719
|
+
if (citation.sourceId && citation.sourceId === context.sourceId) return true;
|
|
2720
|
+
if (citation.anchorId && citation.anchorId === context.anchorId) return true;
|
|
2721
|
+
return false;
|
|
2722
|
+
})
|
|
2723
|
+
);
|
|
2724
|
+
return matchedContexts.length > 0 ? matchedContexts : contexts;
|
|
2725
|
+
}
|
|
2726
|
+
function scoreAnswerRelevance(artifact, scenario, abstained) {
|
|
2727
|
+
if (scenario.unanswerable) return abstained ? 1 : 0;
|
|
2728
|
+
if (abstained) return 0;
|
|
2729
|
+
const targets = [
|
|
2730
|
+
scenario.referenceAnswer,
|
|
2731
|
+
...scenario.expectedClaims ?? [],
|
|
2732
|
+
scenario.query
|
|
2733
|
+
].filter((value) => Boolean(value?.trim()));
|
|
2734
|
+
return Math.max(...targets.map((target) => textSupportScore(target, artifact.answer)));
|
|
2735
|
+
}
|
|
2736
|
+
function scoreAnswerCorrectness(artifact, scenario, abstained) {
|
|
2737
|
+
if (scenario.unanswerable) return abstained ? 1 : 0;
|
|
2738
|
+
const expected = scenario.expectedClaims ?? [];
|
|
2739
|
+
const forbidden = scenario.forbiddenClaims ?? [];
|
|
2740
|
+
const expectedScore = expected.length === 0 ? scoreAnswerRelevance(artifact, scenario, abstained) : average(expected.map((claim) => textSupportScore(claim, artifact.answer)));
|
|
2741
|
+
const forbiddenPenalty = forbidden.length === 0 ? 0 : forbidden.filter((claim) => textSupportScore(claim, artifact.answer) >= 0.7).length / forbidden.length;
|
|
2742
|
+
return clamp01(expectedScore * (1 - forbiddenPenalty));
|
|
2743
|
+
}
|
|
2744
|
+
function normalizeClaims(artifact) {
|
|
2745
|
+
if (artifact.claims?.length) return [...artifact.claims];
|
|
2746
|
+
return splitSentences(artifact.answer).map((text, index) => ({ id: `claim-${index + 1}`, text }));
|
|
2747
|
+
}
|
|
2748
|
+
function splitSentences(text) {
|
|
2749
|
+
return text.split(/(?<=[.!?])\s+/).map((sentence) => sentence.trim()).filter((sentence) => sentence.length > 0 && !looksLikeAbstention(sentence));
|
|
2750
|
+
}
|
|
2751
|
+
function looksLikeAbstention(answer) {
|
|
2752
|
+
const normalized = answer.toLowerCase();
|
|
2753
|
+
return [
|
|
2754
|
+
"i don't know",
|
|
2755
|
+
"i do not know",
|
|
2756
|
+
"not enough information",
|
|
2757
|
+
"cannot answer",
|
|
2758
|
+
"can't answer",
|
|
2759
|
+
"insufficient information",
|
|
2760
|
+
"no reliable answer"
|
|
2761
|
+
].some((phrase) => normalized.includes(phrase));
|
|
2762
|
+
}
|
|
2763
|
+
function mergeMetrics(deterministic, providerScores, policy = "prefer-external") {
|
|
2764
|
+
if (policy === "deterministic-first") return deterministic;
|
|
2765
|
+
const merged = { ...deterministic };
|
|
2766
|
+
for (const scores of Object.values(providerScores)) {
|
|
2767
|
+
for (const [key, value] of Object.entries(scores)) {
|
|
2768
|
+
merged[key] = value;
|
|
2769
|
+
if (key === "groundedness") merged.faithfulness = value;
|
|
2770
|
+
if (key === "faithfulness") merged.groundedness = value;
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
return merged;
|
|
2774
|
+
}
|
|
2775
|
+
function aggregateRagAnswerMetrics(summaries) {
|
|
2776
|
+
const keys = /* @__PURE__ */ new Set();
|
|
2777
|
+
for (const summary of summaries) {
|
|
2778
|
+
for (const key of Object.keys(summary.metrics)) keys.add(key);
|
|
2779
|
+
}
|
|
2780
|
+
const out = {};
|
|
2781
|
+
for (const key of [...keys].sort()) {
|
|
2782
|
+
out[key] = average(summaries.map((summary) => summary.metrics[key]));
|
|
2783
|
+
}
|
|
2784
|
+
out.composite = average(summaries.map((summary) => summary.composite));
|
|
2785
|
+
return out;
|
|
2786
|
+
}
|
|
2787
|
+
function weightedComposite(metrics, weights) {
|
|
2788
|
+
let weighted = 0;
|
|
2789
|
+
let total = 0;
|
|
2790
|
+
for (const [key, weight] of Object.entries(weights)) {
|
|
2791
|
+
if (!Number.isFinite(weight) || weight <= 0) continue;
|
|
2792
|
+
const metric = metrics[key];
|
|
2793
|
+
if (!Number.isFinite(metric)) continue;
|
|
2794
|
+
const value = key === "unsupported_answer_rate" ? 1 - metric : metric;
|
|
2795
|
+
weighted += value * weight;
|
|
2796
|
+
total += weight;
|
|
2797
|
+
}
|
|
2798
|
+
return total === 0 ? 0 : weighted / total;
|
|
2799
|
+
}
|
|
2800
|
+
function normalizeMetricName(metric) {
|
|
2801
|
+
return metric.trim().toLowerCase().replace(/[@/.-]+/g, "_").replace(/\s+/g, "_");
|
|
2802
|
+
}
|
|
2803
|
+
function textSupportScore(needle, haystack) {
|
|
2804
|
+
const needleWords = contentWords2(needle);
|
|
2805
|
+
if (needleWords.length === 0) return 0;
|
|
2806
|
+
const haystackWords = new Set(contentWords2(haystack));
|
|
2807
|
+
const present = needleWords.filter((word) => haystackWords.has(word));
|
|
2808
|
+
return present.length / needleWords.length;
|
|
2809
|
+
}
|
|
2810
|
+
function contentWords2(text) {
|
|
2811
|
+
const normalized = text.toLowerCase().replace(/[^\p{L}\p{N}\s]+/gu, " ").split(/\s+/).map((word) => stem(word.trim())).filter((word) => (word.length >= 3 || /^\d+$/.test(word)) && !stopwords2.has(word));
|
|
2812
|
+
return [...new Set(normalized)];
|
|
2813
|
+
}
|
|
2814
|
+
function stem(word) {
|
|
2815
|
+
if (word.endsWith("ies") && word.length > 4) return `${word.slice(0, -3)}y`;
|
|
2816
|
+
if (word.endsWith("ing") && word.length > 5) return word.slice(0, -3);
|
|
2817
|
+
if (word.endsWith("ed") && word.length > 4) return word.slice(0, -2);
|
|
2818
|
+
if (word.endsWith("s") && word.length > 3) return word.slice(0, -1);
|
|
2819
|
+
return word;
|
|
2820
|
+
}
|
|
2821
|
+
function sourceRefs(text) {
|
|
2822
|
+
const refs = [];
|
|
2823
|
+
const regex = /\[\^([A-Za-z0-9_-]+)(?:#([A-Za-z0-9_.:-]+))?\]/g;
|
|
2824
|
+
let match = regex.exec(text);
|
|
2825
|
+
while (match !== null) {
|
|
2826
|
+
refs.push(match[0]);
|
|
2827
|
+
match = regex.exec(text);
|
|
2828
|
+
}
|
|
2829
|
+
return refs;
|
|
2830
|
+
}
|
|
2831
|
+
function below(metric, threshold) {
|
|
2832
|
+
return threshold !== void 0 && metric < threshold;
|
|
2833
|
+
}
|
|
2834
|
+
function neutralScore(condition) {
|
|
2835
|
+
return condition ? 1 : 0;
|
|
2836
|
+
}
|
|
2837
|
+
function average(values) {
|
|
2838
|
+
const finite = values.filter(Number.isFinite);
|
|
2839
|
+
return finite.length === 0 ? 0 : finite.reduce((sum, value) => sum + value, 0) / finite.length;
|
|
2840
|
+
}
|
|
2841
|
+
function clamp01(value) {
|
|
2842
|
+
if (!Number.isFinite(value)) return 0;
|
|
2843
|
+
return Math.max(0, Math.min(1, value));
|
|
2844
|
+
}
|
|
2845
|
+
var stopwords2 = /* @__PURE__ */ new Set([
|
|
2846
|
+
"the",
|
|
2847
|
+
"a",
|
|
2848
|
+
"an",
|
|
2849
|
+
"and",
|
|
2850
|
+
"or",
|
|
2851
|
+
"but",
|
|
2852
|
+
"of",
|
|
2853
|
+
"to",
|
|
2854
|
+
"in",
|
|
2855
|
+
"on",
|
|
2856
|
+
"for",
|
|
2857
|
+
"with",
|
|
2858
|
+
"as",
|
|
2859
|
+
"by",
|
|
2860
|
+
"at",
|
|
2861
|
+
"from",
|
|
2862
|
+
"that",
|
|
2863
|
+
"this",
|
|
2864
|
+
"these",
|
|
2865
|
+
"those",
|
|
2866
|
+
"it",
|
|
2867
|
+
"its",
|
|
2868
|
+
"is",
|
|
2869
|
+
"are",
|
|
2870
|
+
"was",
|
|
2871
|
+
"were",
|
|
2872
|
+
"be",
|
|
2873
|
+
"been",
|
|
2874
|
+
"being",
|
|
2875
|
+
"has",
|
|
2876
|
+
"have",
|
|
2877
|
+
"had",
|
|
2878
|
+
"can",
|
|
2879
|
+
"will",
|
|
2880
|
+
"would",
|
|
2881
|
+
"should",
|
|
2882
|
+
"may",
|
|
2883
|
+
"might",
|
|
2884
|
+
"not",
|
|
2885
|
+
"no",
|
|
2886
|
+
"than",
|
|
2887
|
+
"then",
|
|
2888
|
+
"over",
|
|
2889
|
+
"under",
|
|
2890
|
+
"about",
|
|
2891
|
+
"into",
|
|
2892
|
+
"their",
|
|
2893
|
+
"they",
|
|
2894
|
+
"them",
|
|
2895
|
+
"what",
|
|
2896
|
+
"when",
|
|
2897
|
+
"where",
|
|
2898
|
+
"who",
|
|
2899
|
+
"why",
|
|
2900
|
+
"how"
|
|
2901
|
+
]);
|
|
2902
|
+
|
|
2903
|
+
// src/research-loop.ts
|
|
2904
|
+
import {
|
|
2905
|
+
blockingKnowledgeEval,
|
|
2906
|
+
objectiveEval
|
|
2907
|
+
} from "@tangle-network/agent-eval";
|
|
2908
|
+
function createKnowledgeControlLoopAdapter(options) {
|
|
2909
|
+
let initialized = false;
|
|
2910
|
+
const appliedSteps = [];
|
|
2911
|
+
return {
|
|
2912
|
+
intent: options.goal,
|
|
2913
|
+
async observe({ history, abortSignal }) {
|
|
2914
|
+
if (abortSignal.aborted) throw new Error("Knowledge control loop aborted");
|
|
2915
|
+
if (!initialized) {
|
|
2916
|
+
await initKnowledgeBase(options.root);
|
|
2917
|
+
initialized = true;
|
|
2918
|
+
}
|
|
2919
|
+
const index = await buildKnowledgeIndex(options.root);
|
|
2920
|
+
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
2921
|
+
const lintFindings = lintKnowledgeIndex(index);
|
|
2922
|
+
const readiness = readinessFor(options, index);
|
|
2923
|
+
return {
|
|
2924
|
+
root: options.root,
|
|
2925
|
+
goal: options.goal,
|
|
2926
|
+
iteration: history.length + 1,
|
|
2927
|
+
index,
|
|
2928
|
+
lintFindings,
|
|
2929
|
+
validation,
|
|
2930
|
+
readiness,
|
|
2931
|
+
previousSteps: [...appliedSteps],
|
|
2932
|
+
signal: abortSignal
|
|
2933
|
+
};
|
|
2934
|
+
},
|
|
2935
|
+
validate({ state }) {
|
|
2936
|
+
const errorFindings = state.validation.findings.filter(
|
|
2937
|
+
(finding) => finding.severity === "error"
|
|
2938
|
+
);
|
|
2939
|
+
const evals = [
|
|
2940
|
+
objectiveEval({
|
|
2941
|
+
id: "knowledge-valid",
|
|
2942
|
+
passed: state.validation.ok,
|
|
2943
|
+
severity: "critical",
|
|
2944
|
+
detail: state.validation.ok ? "Knowledge index is valid." : "Knowledge index has validation errors.",
|
|
2945
|
+
metadata: { findings: state.validation.findings }
|
|
2946
|
+
}),
|
|
2947
|
+
objectiveEval({
|
|
2948
|
+
id: "knowledge-lint-errors",
|
|
2949
|
+
passed: errorFindings.length === 0,
|
|
2950
|
+
severity: "error",
|
|
2951
|
+
detail: errorFindings.length === 0 ? "No lint errors." : `${errorFindings.length} lint error(s).`,
|
|
2952
|
+
metadata: { findings: errorFindings }
|
|
2953
|
+
})
|
|
2954
|
+
];
|
|
2955
|
+
if (state.readiness) evals.push(blockingKnowledgeEval(state.readiness.report));
|
|
2956
|
+
return evals;
|
|
2957
|
+
},
|
|
2958
|
+
shouldStop() {
|
|
2959
|
+
return { stop: false, pass: false, reason: "knowledge driver owns stop decisions" };
|
|
2960
|
+
},
|
|
2961
|
+
async act(action, ctx) {
|
|
2962
|
+
const step = await applyKnowledgeResearchDecision(options, action, ctx.state.iteration);
|
|
2963
|
+
appliedSteps.push(step);
|
|
2964
|
+
return step;
|
|
2965
|
+
}
|
|
2966
|
+
};
|
|
2967
|
+
}
|
|
2968
|
+
async function runKnowledgeResearchLoop(options) {
|
|
2969
|
+
const maxIterations = Math.max(1, options.maxIterations ?? 3);
|
|
2970
|
+
await initKnowledgeBase(options.root);
|
|
2971
|
+
const steps = [];
|
|
2972
|
+
let index = await buildKnowledgeIndex(options.root);
|
|
2973
|
+
let validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
2974
|
+
let lintFindings = lintKnowledgeIndex(index);
|
|
2975
|
+
let readiness = readinessFor(options, index);
|
|
2976
|
+
let done = false;
|
|
2977
|
+
for (let iteration = 1; iteration <= maxIterations; iteration++) {
|
|
2978
|
+
if (options.signal?.aborted) throw new Error("Knowledge research loop aborted");
|
|
2979
|
+
const decision = await options.step({
|
|
2980
|
+
root: options.root,
|
|
2981
|
+
goal: options.goal,
|
|
2982
|
+
iteration,
|
|
2983
|
+
index,
|
|
2984
|
+
lintFindings,
|
|
2985
|
+
validation,
|
|
2986
|
+
readiness,
|
|
2987
|
+
previousSteps: steps,
|
|
2988
|
+
signal: options.signal
|
|
2989
|
+
});
|
|
2990
|
+
done = Boolean(decision.done);
|
|
2991
|
+
const step = await applyKnowledgeResearchDecision(options, decision, iteration);
|
|
2992
|
+
index = await buildKnowledgeIndex(options.root);
|
|
2993
|
+
validation = step.validation;
|
|
2994
|
+
lintFindings = step.lintFindings;
|
|
2995
|
+
readiness = step.readiness;
|
|
2996
|
+
steps.push(step);
|
|
2997
|
+
await options.onStep?.(step);
|
|
2998
|
+
if (done) break;
|
|
2999
|
+
if (!step.applied && step.addedSources.length === 0) break;
|
|
3000
|
+
}
|
|
3001
|
+
return {
|
|
3002
|
+
root: options.root,
|
|
3003
|
+
goal: options.goal,
|
|
3004
|
+
iterations: steps.length,
|
|
3005
|
+
done,
|
|
3006
|
+
index,
|
|
3007
|
+
lintFindings,
|
|
3008
|
+
validation,
|
|
3009
|
+
readiness,
|
|
3010
|
+
steps
|
|
3011
|
+
};
|
|
3012
|
+
}
|
|
3013
|
+
async function applyKnowledgeResearchDecision(options, decision, iteration = 1) {
|
|
3014
|
+
const addedSources = [];
|
|
3015
|
+
for (const sourcePath of decision.sourcePaths ?? []) {
|
|
3016
|
+
addedSources.push(...await addSourcePath(options.root, sourcePath, options.sourceOptions));
|
|
3017
|
+
}
|
|
3018
|
+
for (const sourceText of decision.sourceTexts ?? []) {
|
|
3019
|
+
addedSources.push(await addSourceText(options.root, sourceText, options.sourceOptions));
|
|
3020
|
+
}
|
|
3021
|
+
const applied = decision.proposalText ? await applyKnowledgeWriteBlocks(options.root, decision.proposalText) : void 0;
|
|
3022
|
+
const index = await buildKnowledgeIndex(options.root);
|
|
3023
|
+
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
3024
|
+
const lintFindings = lintKnowledgeIndex(index);
|
|
3025
|
+
const readiness = readinessFor(options, index);
|
|
3026
|
+
const done = Boolean(decision.done);
|
|
3027
|
+
const event = createKnowledgeEvent({
|
|
3028
|
+
type: "research.iteration",
|
|
3029
|
+
actor: options.actor,
|
|
3030
|
+
target: options.root,
|
|
3031
|
+
metadata: {
|
|
3032
|
+
goal: options.goal,
|
|
3033
|
+
iteration,
|
|
3034
|
+
done,
|
|
3035
|
+
addedSourceCount: addedSources.length,
|
|
3036
|
+
written: applied?.written,
|
|
3037
|
+
warningCount: applied?.warnings.length ?? 0,
|
|
3038
|
+
errorCount: validation.findings.filter((finding) => finding.severity === "error").length
|
|
3039
|
+
}
|
|
3040
|
+
});
|
|
3041
|
+
return {
|
|
3042
|
+
iteration,
|
|
3043
|
+
notes: decision.notes,
|
|
3044
|
+
addedSources,
|
|
3045
|
+
applied,
|
|
3046
|
+
lintFindings,
|
|
3047
|
+
validation,
|
|
3048
|
+
readiness,
|
|
3049
|
+
event,
|
|
3050
|
+
done,
|
|
3051
|
+
metadata: decision.metadata
|
|
3052
|
+
};
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
// src/rag-improvement-loop.ts
|
|
3056
|
+
async function runRagKnowledgeImprovementLoop(options) {
|
|
3057
|
+
assertConfiguredRequiredPhases(options);
|
|
3058
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
3059
|
+
const phases = [];
|
|
3060
|
+
let retrieval;
|
|
3061
|
+
let findings = [];
|
|
3062
|
+
let acquisition;
|
|
3063
|
+
let knowledgeUpdate;
|
|
3064
|
+
let answerQuality;
|
|
3065
|
+
let promotion;
|
|
3066
|
+
if (phaseEnabled(options, "retrieval-tuning")) {
|
|
3067
|
+
if (options.retrieval) {
|
|
3068
|
+
retrieval = await runPhase(
|
|
3069
|
+
phases,
|
|
3070
|
+
now,
|
|
3071
|
+
"retrieval-tuning",
|
|
3072
|
+
async () => {
|
|
3073
|
+
assertNotAborted(options.signal);
|
|
3074
|
+
return runRetrievalImprovementLoop(options.retrieval);
|
|
3075
|
+
},
|
|
3076
|
+
summarizeRetrievalResult
|
|
3077
|
+
);
|
|
3078
|
+
} else {
|
|
3079
|
+
skipPhase(phases, now, "retrieval-tuning", "no retrieval options provided");
|
|
3080
|
+
}
|
|
3081
|
+
}
|
|
3082
|
+
if (phaseEnabled(options, "gap-diagnosis")) {
|
|
3083
|
+
if (options.diagnose) {
|
|
3084
|
+
findings = [
|
|
3085
|
+
...await runPhase(
|
|
3086
|
+
phases,
|
|
3087
|
+
now,
|
|
3088
|
+
"gap-diagnosis",
|
|
3089
|
+
async () => {
|
|
3090
|
+
assertNotAborted(options.signal);
|
|
3091
|
+
return options.diagnose({
|
|
3092
|
+
goal: options.goal,
|
|
3093
|
+
phases,
|
|
3094
|
+
signal: options.signal,
|
|
3095
|
+
retrieval
|
|
3096
|
+
});
|
|
3097
|
+
},
|
|
3098
|
+
(diagnosed) => `${diagnosed.length} finding(s)`
|
|
3099
|
+
)
|
|
3100
|
+
];
|
|
3101
|
+
} else {
|
|
3102
|
+
skipPhase(phases, now, "gap-diagnosis", "no diagnosis hook provided");
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
if (phaseEnabled(options, "knowledge-acquisition")) {
|
|
3106
|
+
if (options.acquireKnowledge) {
|
|
3107
|
+
acquisition = await runPhase(
|
|
3108
|
+
phases,
|
|
3109
|
+
now,
|
|
3110
|
+
"knowledge-acquisition",
|
|
3111
|
+
async () => {
|
|
3112
|
+
assertNotAborted(options.signal);
|
|
3113
|
+
return options.acquireKnowledge({
|
|
3114
|
+
goal: options.goal,
|
|
3115
|
+
phases,
|
|
3116
|
+
signal: options.signal,
|
|
3117
|
+
retrieval,
|
|
3118
|
+
findings
|
|
3119
|
+
});
|
|
3120
|
+
},
|
|
3121
|
+
summarizeAcquisitionDecision
|
|
3122
|
+
);
|
|
3123
|
+
} else {
|
|
3124
|
+
skipPhase(phases, now, "knowledge-acquisition", "no acquisition hook provided");
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
if (phaseEnabled(options, "knowledge-update")) {
|
|
3128
|
+
if (options.updateKnowledge) {
|
|
3129
|
+
knowledgeUpdate = await runPhase(
|
|
3130
|
+
phases,
|
|
3131
|
+
now,
|
|
3132
|
+
"knowledge-update",
|
|
3133
|
+
async () => {
|
|
3134
|
+
assertNotAborted(options.signal);
|
|
3135
|
+
return options.updateKnowledge({
|
|
3136
|
+
goal: options.goal,
|
|
3137
|
+
phases,
|
|
3138
|
+
signal: options.signal,
|
|
3139
|
+
retrieval,
|
|
3140
|
+
findings,
|
|
3141
|
+
acquisition
|
|
3142
|
+
});
|
|
3143
|
+
},
|
|
3144
|
+
(result) => result.summary
|
|
3145
|
+
);
|
|
3146
|
+
} else if (options.knowledgeResearch) {
|
|
3147
|
+
knowledgeUpdate = await runPhase(
|
|
3148
|
+
phases,
|
|
3149
|
+
now,
|
|
3150
|
+
"knowledge-update",
|
|
3151
|
+
async () => {
|
|
3152
|
+
assertNotAborted(options.signal);
|
|
3153
|
+
return runKnowledgeResearchUpdate(options, acquisition);
|
|
3154
|
+
},
|
|
3155
|
+
(result) => result.summary
|
|
3156
|
+
);
|
|
3157
|
+
} else {
|
|
3158
|
+
skipPhase(phases, now, "knowledge-update", "no update hook or research loop provided");
|
|
3159
|
+
}
|
|
3160
|
+
}
|
|
3161
|
+
if (phaseEnabled(options, "answer-quality")) {
|
|
3162
|
+
if (options.evaluateAnswers) {
|
|
3163
|
+
answerQuality = await runPhase(
|
|
3164
|
+
phases,
|
|
3165
|
+
now,
|
|
3166
|
+
"answer-quality",
|
|
3167
|
+
async () => {
|
|
3168
|
+
assertNotAborted(options.signal);
|
|
3169
|
+
return options.evaluateAnswers({
|
|
3170
|
+
goal: options.goal,
|
|
3171
|
+
phases,
|
|
3172
|
+
signal: options.signal,
|
|
3173
|
+
retrieval,
|
|
3174
|
+
findings,
|
|
3175
|
+
acquisition,
|
|
3176
|
+
knowledgeUpdate
|
|
3177
|
+
});
|
|
3178
|
+
},
|
|
3179
|
+
summarizeAnswerQuality
|
|
3180
|
+
);
|
|
3181
|
+
findings = [...findings, ...answerQuality.findings ?? []];
|
|
3182
|
+
} else {
|
|
3183
|
+
skipPhase(phases, now, "answer-quality", "no answer-quality hook provided");
|
|
3184
|
+
}
|
|
3185
|
+
}
|
|
3186
|
+
if (phaseEnabled(options, "promotion")) {
|
|
3187
|
+
if (options.promote) {
|
|
3188
|
+
promotion = await runPhase(
|
|
3189
|
+
phases,
|
|
3190
|
+
now,
|
|
3191
|
+
"promotion",
|
|
3192
|
+
async () => {
|
|
3193
|
+
assertNotAborted(options.signal);
|
|
3194
|
+
return options.promote({
|
|
3195
|
+
goal: options.goal,
|
|
3196
|
+
phases,
|
|
3197
|
+
signal: options.signal,
|
|
3198
|
+
retrieval,
|
|
3199
|
+
findings,
|
|
3200
|
+
acquisition,
|
|
3201
|
+
knowledgeUpdate,
|
|
3202
|
+
answerQuality
|
|
3203
|
+
});
|
|
3204
|
+
},
|
|
3205
|
+
(result) => `${result.promoted ? "promoted" : "held"}: ${result.reason}`
|
|
3206
|
+
);
|
|
3207
|
+
} else {
|
|
3208
|
+
skipPhase(phases, now, "promotion", "no promotion hook provided");
|
|
3209
|
+
}
|
|
3210
|
+
}
|
|
3211
|
+
return {
|
|
3212
|
+
goal: options.goal,
|
|
3213
|
+
phases,
|
|
3214
|
+
retrieval,
|
|
3215
|
+
findings,
|
|
3216
|
+
acquisition,
|
|
3217
|
+
knowledgeUpdate,
|
|
3218
|
+
answerQuality,
|
|
3219
|
+
promotion
|
|
3220
|
+
};
|
|
3221
|
+
}
|
|
3222
|
+
async function runKnowledgeResearchUpdate(options, acquisition) {
|
|
3223
|
+
const research = options.knowledgeResearch;
|
|
3224
|
+
if (!research) {
|
|
3225
|
+
throw new Error("knowledgeResearch options are required to run the knowledge update phase");
|
|
3226
|
+
}
|
|
3227
|
+
const { goal, step, ...rest } = research;
|
|
3228
|
+
const researchStep = step ?? acquisitionBackedResearchStep(acquisition);
|
|
3229
|
+
const maxIterations = step ? rest.maxIterations : 1;
|
|
3230
|
+
const result = await runKnowledgeResearchLoop({
|
|
3231
|
+
...rest,
|
|
3232
|
+
goal: goal ?? options.goal,
|
|
3233
|
+
maxIterations,
|
|
3234
|
+
signal: options.signal,
|
|
3235
|
+
step: researchStep
|
|
3236
|
+
});
|
|
3237
|
+
return {
|
|
3238
|
+
applied: result.steps.some((stepResult) => {
|
|
3239
|
+
return stepResult.addedSources.length > 0 || Boolean(stepResult.applied);
|
|
3240
|
+
}),
|
|
3241
|
+
summary: `${result.iterations} research iteration(s); done=${String(result.done)}`,
|
|
3242
|
+
research: result
|
|
3243
|
+
};
|
|
3244
|
+
}
|
|
3245
|
+
function acquisitionBackedResearchStep(acquisition) {
|
|
3246
|
+
if (!acquisition) {
|
|
3247
|
+
throw new Error(
|
|
3248
|
+
"knowledgeResearch requires either a step hook or a knowledge-acquisition result to apply"
|
|
3249
|
+
);
|
|
3250
|
+
}
|
|
3251
|
+
return () => ({ ...acquisition, done: acquisition.done ?? true });
|
|
3252
|
+
}
|
|
3253
|
+
async function runPhase(phases, now, phase, action, summarize) {
|
|
3254
|
+
const startedAt = now().toISOString();
|
|
3255
|
+
try {
|
|
3256
|
+
const result = await action();
|
|
3257
|
+
phases.push({
|
|
3258
|
+
phase,
|
|
3259
|
+
status: "completed",
|
|
3260
|
+
summary: summarize(result),
|
|
3261
|
+
startedAt,
|
|
3262
|
+
finishedAt: now().toISOString()
|
|
3263
|
+
});
|
|
3264
|
+
return result;
|
|
3265
|
+
} catch (error) {
|
|
3266
|
+
phases.push({
|
|
3267
|
+
phase,
|
|
3268
|
+
status: "failed",
|
|
3269
|
+
summary: error.message,
|
|
3270
|
+
startedAt,
|
|
3271
|
+
finishedAt: now().toISOString()
|
|
3272
|
+
});
|
|
3273
|
+
throw error;
|
|
3274
|
+
}
|
|
3275
|
+
}
|
|
3276
|
+
function skipPhase(phases, now, phase, summary) {
|
|
3277
|
+
const timestamp = now().toISOString();
|
|
3278
|
+
phases.push({ phase, status: "skipped", summary, startedAt: timestamp, finishedAt: timestamp });
|
|
3279
|
+
}
|
|
3280
|
+
function summarizeRetrievalResult(result) {
|
|
3281
|
+
return `${result.candidates.length} candidate(s); winner=${JSON.stringify(result.winnerConfig)}`;
|
|
3282
|
+
}
|
|
3283
|
+
function summarizeAcquisitionDecision(decision) {
|
|
3284
|
+
const sourcePathCount = decision.sourcePaths?.length ?? 0;
|
|
3285
|
+
const sourceTextCount = decision.sourceTexts?.length ?? 0;
|
|
3286
|
+
const proposal = decision.proposalText ? "proposal" : "no proposal";
|
|
3287
|
+
return `${sourcePathCount} path source(s), ${sourceTextCount} text source(s), ${proposal}`;
|
|
3288
|
+
}
|
|
3289
|
+
function summarizeAnswerQuality(result) {
|
|
3290
|
+
const metrics = Object.entries(result.metrics).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => `${key}=${formatMetric(value)}`).join(", ");
|
|
3291
|
+
return `${result.passed ? "passed" : "failed"}${metrics ? `; ${metrics}` : ""}`;
|
|
3292
|
+
}
|
|
3293
|
+
function formatMetric(value) {
|
|
3294
|
+
return Number.isFinite(value) ? value.toFixed(3) : String(value);
|
|
3295
|
+
}
|
|
3296
|
+
function phaseEnabled(options, phase) {
|
|
3297
|
+
return !options.enabledPhases || options.enabledPhases.includes(phase);
|
|
3298
|
+
}
|
|
3299
|
+
function assertConfiguredRequiredPhases(options) {
|
|
3300
|
+
for (const phase of options.requiredPhases ?? []) {
|
|
3301
|
+
if (!phaseEnabled(options, phase)) {
|
|
3302
|
+
throw new Error(`required phase ${phase} is not enabled`);
|
|
3303
|
+
}
|
|
3304
|
+
if (!phaseConfigured(options, phase)) {
|
|
3305
|
+
throw new Error(requiredPhaseMessage(phase));
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
function phaseConfigured(options, phase) {
|
|
3310
|
+
switch (phase) {
|
|
3311
|
+
case "retrieval-tuning":
|
|
3312
|
+
return Boolean(options.retrieval);
|
|
3313
|
+
case "gap-diagnosis":
|
|
3314
|
+
return Boolean(options.diagnose);
|
|
3315
|
+
case "knowledge-acquisition":
|
|
3316
|
+
return Boolean(options.acquireKnowledge);
|
|
3317
|
+
case "knowledge-update":
|
|
3318
|
+
return Boolean(options.updateKnowledge ?? options.knowledgeResearch);
|
|
3319
|
+
case "answer-quality":
|
|
3320
|
+
return Boolean(options.evaluateAnswers);
|
|
3321
|
+
case "promotion":
|
|
3322
|
+
return Boolean(options.promote);
|
|
3323
|
+
}
|
|
3324
|
+
}
|
|
3325
|
+
function requiredPhaseMessage(phase) {
|
|
3326
|
+
switch (phase) {
|
|
3327
|
+
case "retrieval-tuning":
|
|
3328
|
+
return "required phase retrieval-tuning requires retrieval options";
|
|
3329
|
+
case "gap-diagnosis":
|
|
3330
|
+
return "required phase gap-diagnosis requires a diagnose hook";
|
|
3331
|
+
case "knowledge-acquisition":
|
|
3332
|
+
return "required phase knowledge-acquisition requires an acquireKnowledge hook";
|
|
3333
|
+
case "knowledge-update":
|
|
3334
|
+
return "required phase knowledge-update requires updateKnowledge or knowledgeResearch";
|
|
3335
|
+
case "answer-quality":
|
|
3336
|
+
return "required phase answer-quality requires an evaluateAnswers hook";
|
|
3337
|
+
case "promotion":
|
|
3338
|
+
return "required phase promotion requires a promote hook";
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3341
|
+
function assertNotAborted(signal) {
|
|
3342
|
+
if (signal?.aborted) {
|
|
3343
|
+
throw new Error("RAG knowledge improvement loop aborted");
|
|
3344
|
+
}
|
|
3345
|
+
}
|
|
3346
|
+
|
|
3347
|
+
// src/kb-improvement.ts
|
|
3348
|
+
var DEFAULT_LEASE_TTL_MS = 15 * 60 * 1e3;
|
|
3349
|
+
var UPDATE_PHASES = [
|
|
3350
|
+
"knowledge-acquisition",
|
|
3351
|
+
"knowledge-update"
|
|
3352
|
+
];
|
|
3353
|
+
var EVALUATION_PHASES = [
|
|
3354
|
+
"retrieval-tuning",
|
|
3355
|
+
"gap-diagnosis",
|
|
3356
|
+
"answer-quality",
|
|
3357
|
+
"promotion"
|
|
3358
|
+
];
|
|
3359
|
+
function knowledgeImprovementRunId(root, goal) {
|
|
3360
|
+
return stableId("kimpr", `${root}:${goal}`);
|
|
3361
|
+
}
|
|
3362
|
+
function knowledgeImprovementRunDir(root, runId) {
|
|
3363
|
+
return join3(layoutFor(root).cacheDir, "improvements", runId);
|
|
3364
|
+
}
|
|
3365
|
+
async function loadKnowledgeImprovementState(root, runId, runDir = knowledgeImprovementRunDir(root, runId)) {
|
|
3366
|
+
try {
|
|
3367
|
+
return JSON.parse(await readFile2(statePath(runDir), "utf8"));
|
|
3368
|
+
} catch {
|
|
3369
|
+
return null;
|
|
3370
|
+
}
|
|
3371
|
+
}
|
|
3372
|
+
async function improveKnowledgeBase(options) {
|
|
3373
|
+
assertKnowledgeImprovementOptions(options);
|
|
3374
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
3375
|
+
const runId = options.runId ?? knowledgeImprovementRunId(options.root, options.goal);
|
|
3376
|
+
const runDir = options.runDir ?? knowledgeImprovementRunDir(options.root, runId);
|
|
3377
|
+
await initKnowledgeBase(options.root);
|
|
3378
|
+
await mkdir3(runDir, { recursive: true });
|
|
3379
|
+
const lease = await acquireRunLease(runDir, {
|
|
3380
|
+
ownerId: options.ownerId ?? `pid-${process.pid}`,
|
|
3381
|
+
ttlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS,
|
|
3382
|
+
now
|
|
3383
|
+
});
|
|
3384
|
+
try {
|
|
3385
|
+
let state = options.resume === false ? null : await loadKnowledgeImprovementState(options.root, runId, runDir);
|
|
3386
|
+
if (!state) {
|
|
3387
|
+
const baseHash = await hashKnowledgeBase(options.root);
|
|
3388
|
+
state = {
|
|
3389
|
+
version: 1,
|
|
3390
|
+
runId,
|
|
3391
|
+
root: options.root,
|
|
3392
|
+
goal: options.goal,
|
|
3393
|
+
status: "running",
|
|
3394
|
+
baseHash,
|
|
3395
|
+
createdAt: now().toISOString(),
|
|
3396
|
+
updatedAt: now().toISOString(),
|
|
3397
|
+
ownerId: lease.ownerId,
|
|
3398
|
+
candidates: []
|
|
3399
|
+
};
|
|
3400
|
+
await saveState(runDir, state, options.onState);
|
|
3401
|
+
await appendLedger(runDir, { type: "run.created", runId, baseHash });
|
|
3402
|
+
}
|
|
3403
|
+
if (state.status === "promoted") {
|
|
3404
|
+
return { runId, runDir, state, promoted: true, blocked: false };
|
|
3405
|
+
}
|
|
3406
|
+
if (state.status === "blocked") {
|
|
3407
|
+
return { runId, runDir, state, promoted: false, blocked: true };
|
|
3408
|
+
}
|
|
3409
|
+
const maxCandidates = Math.max(1, options.maxCandidates ?? 1);
|
|
3410
|
+
let candidate = findActiveCandidate(state);
|
|
3411
|
+
let lifecycle;
|
|
3412
|
+
while (candidate?.status === "running" || !candidate && state.candidates.length < maxCandidates) {
|
|
3413
|
+
if (!candidate) {
|
|
3414
|
+
const currentHash2 = await hashKnowledgeBase(options.root);
|
|
3415
|
+
if (currentHash2 !== state.baseHash) {
|
|
3416
|
+
state = await blockRun(
|
|
3417
|
+
runDir,
|
|
3418
|
+
state,
|
|
3419
|
+
`base changed before candidate creation: expected ${state.baseHash}, got ${currentHash2}`,
|
|
3420
|
+
options.onState,
|
|
3421
|
+
now
|
|
3422
|
+
);
|
|
3423
|
+
return { runId, runDir, state, promoted: false, blocked: true };
|
|
3424
|
+
}
|
|
3425
|
+
candidate = await createCandidateWorkspace(runDir, state, options.root, now);
|
|
3426
|
+
state.candidates.push(candidate);
|
|
3427
|
+
state.status = "running";
|
|
3428
|
+
state.updatedAt = now().toISOString();
|
|
3429
|
+
await saveState(runDir, state, options.onState);
|
|
3430
|
+
await appendLedger(runDir, {
|
|
3431
|
+
type: "candidate.created",
|
|
3432
|
+
runId,
|
|
3433
|
+
candidateId: candidate.candidateId,
|
|
3434
|
+
iteration: candidate.iteration
|
|
3435
|
+
});
|
|
3436
|
+
}
|
|
3437
|
+
lifecycle = await runCandidateLifecycle(runDir, runId, candidate, options, now);
|
|
3438
|
+
candidate.status = "candidate-ready";
|
|
3439
|
+
candidate.updatedAt = now().toISOString();
|
|
3440
|
+
state.status = "candidate-ready";
|
|
3441
|
+
state.updatedAt = now().toISOString();
|
|
3442
|
+
await saveState(runDir, state, options.onState);
|
|
3443
|
+
await appendLedger(runDir, {
|
|
3444
|
+
type: "candidate.ready",
|
|
3445
|
+
runId,
|
|
3446
|
+
candidateId: candidate.candidateId
|
|
3447
|
+
});
|
|
3448
|
+
}
|
|
3449
|
+
if (!candidate) {
|
|
3450
|
+
state.status = "rejected";
|
|
3451
|
+
state.updatedAt = now().toISOString();
|
|
3452
|
+
await saveState(runDir, state, options.onState);
|
|
3453
|
+
return { runId, runDir, state, promoted: false, blocked: false };
|
|
3454
|
+
}
|
|
3455
|
+
candidate = await evaluateCandidate(runDir, state, candidate, lifecycle, options, now);
|
|
3456
|
+
state.updatedAt = now().toISOString();
|
|
3457
|
+
await saveState(runDir, state, options.onState);
|
|
3458
|
+
if (!candidate.evaluation?.passed) {
|
|
3459
|
+
candidate.status = "rejected";
|
|
3460
|
+
state.status = "rejected";
|
|
3461
|
+
state.updatedAt = now().toISOString();
|
|
3462
|
+
await saveState(runDir, state, options.onState);
|
|
3463
|
+
await appendLedger(runDir, {
|
|
3464
|
+
type: "candidate.rejected",
|
|
3465
|
+
runId,
|
|
3466
|
+
candidateId: candidate.candidateId
|
|
3467
|
+
});
|
|
3468
|
+
return { runId, runDir, state, candidate, lifecycle, promoted: false, blocked: false };
|
|
3469
|
+
}
|
|
3470
|
+
if (options.promote === false) {
|
|
3471
|
+
state.status = "candidate-ready";
|
|
3472
|
+
state.updatedAt = now().toISOString();
|
|
3473
|
+
await saveState(runDir, state, options.onState);
|
|
3474
|
+
return { runId, runDir, state, candidate, lifecycle, promoted: false, blocked: false };
|
|
3475
|
+
}
|
|
3476
|
+
const currentHash = await hashKnowledgeBase(options.root);
|
|
3477
|
+
if (currentHash !== state.baseHash) {
|
|
3478
|
+
state = await blockRun(
|
|
3479
|
+
runDir,
|
|
3480
|
+
state,
|
|
3481
|
+
`base changed before promotion: expected ${state.baseHash}, got ${currentHash}`,
|
|
3482
|
+
options.onState,
|
|
3483
|
+
now
|
|
3484
|
+
);
|
|
3485
|
+
await appendLedger(runDir, {
|
|
3486
|
+
type: "promotion.blocked",
|
|
3487
|
+
runId,
|
|
3488
|
+
candidateId: candidate.candidateId,
|
|
3489
|
+
reason: state.blockedReason
|
|
3490
|
+
});
|
|
3491
|
+
return { runId, runDir, state, candidate, lifecycle, promoted: false, blocked: true };
|
|
3492
|
+
}
|
|
3493
|
+
await promoteCandidate(options.root, candidate.candidateRoot);
|
|
3494
|
+
candidate.status = "promoted";
|
|
3495
|
+
candidate.updatedAt = now().toISOString();
|
|
3496
|
+
state.status = "promoted";
|
|
3497
|
+
state.promotedCandidateId = candidate.candidateId;
|
|
3498
|
+
state.updatedAt = now().toISOString();
|
|
3499
|
+
await saveState(runDir, state, options.onState);
|
|
3500
|
+
await appendLedger(runDir, {
|
|
3501
|
+
type: "candidate.promoted",
|
|
3502
|
+
runId,
|
|
3503
|
+
candidateId: candidate.candidateId
|
|
3504
|
+
});
|
|
3505
|
+
return { runId, runDir, state, candidate, lifecycle, promoted: true, blocked: false };
|
|
3506
|
+
} finally {
|
|
3507
|
+
await lease.release();
|
|
3508
|
+
}
|
|
3509
|
+
}
|
|
3510
|
+
function assertKnowledgeImprovementOptions(options) {
|
|
3511
|
+
if (options.step && options.knowledgeResearch?.step) {
|
|
3512
|
+
throw new Error("improveKnowledgeBase accepts either step or knowledgeResearch.step, not both");
|
|
3513
|
+
}
|
|
3514
|
+
const updateDrivers = [
|
|
3515
|
+
Boolean(options.step ?? options.knowledgeResearch),
|
|
3516
|
+
Boolean(options.updateKnowledge)
|
|
3517
|
+
].filter(Boolean).length;
|
|
3518
|
+
if (updateDrivers > 1) {
|
|
3519
|
+
throw new Error(
|
|
3520
|
+
"improveKnowledgeBase accepts only one knowledge-update driver: knowledgeResearch or updateKnowledge"
|
|
3521
|
+
);
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3524
|
+
async function runCandidateLifecycle(runDir, runId, candidate, options, now) {
|
|
3525
|
+
const lifecycles = [];
|
|
3526
|
+
if (shouldRunUpdateStage(options)) {
|
|
3527
|
+
const updateLifecycle = await runRagKnowledgeImprovementLoop({
|
|
3528
|
+
goal: options.goal,
|
|
3529
|
+
acquireKnowledge: options.acquireKnowledge,
|
|
3530
|
+
knowledgeResearch: candidateKnowledgeResearchOptions(candidate.candidateRoot, options),
|
|
3531
|
+
updateKnowledge: candidateUpdateHook(runId, candidate, options),
|
|
3532
|
+
enabledPhases: selectedStagePhases(options, UPDATE_PHASES),
|
|
3533
|
+
requiredPhases: selectedStageRequiredPhases(options, UPDATE_PHASES),
|
|
3534
|
+
signal: options.signal,
|
|
3535
|
+
now
|
|
3536
|
+
});
|
|
3537
|
+
lifecycles.push(updateLifecycle);
|
|
3538
|
+
recordLifecycle(candidate, "candidate-update", updateLifecycle);
|
|
3539
|
+
}
|
|
3540
|
+
if (shouldRunEvaluationStage(options)) {
|
|
3541
|
+
const candidateIndex = await buildKnowledgeIndex(candidate.candidateRoot);
|
|
3542
|
+
const evaluationLifecycle = await runRagKnowledgeImprovementLoop({
|
|
3543
|
+
goal: options.goal,
|
|
3544
|
+
retrieval: options.retrieval ? {
|
|
3545
|
+
...options.retrieval,
|
|
3546
|
+
index: candidateIndex,
|
|
3547
|
+
runDir: options.retrieval.runDir ?? join3(runDir, "retrieval", candidate.candidateId)
|
|
3548
|
+
} : void 0,
|
|
3549
|
+
diagnose: options.diagnose,
|
|
3550
|
+
evaluateAnswers: options.evaluateAnswers,
|
|
3551
|
+
promote: options.decidePromotion,
|
|
3552
|
+
enabledPhases: selectedStagePhases(options, EVALUATION_PHASES),
|
|
3553
|
+
requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES),
|
|
3554
|
+
signal: options.signal,
|
|
3555
|
+
now
|
|
3556
|
+
});
|
|
3557
|
+
lifecycles.push(evaluationLifecycle);
|
|
3558
|
+
recordLifecycle(candidate, "candidate-evaluation", evaluationLifecycle);
|
|
3559
|
+
}
|
|
3560
|
+
return mergeLifecycleResults(options.goal, lifecycles);
|
|
3561
|
+
}
|
|
3562
|
+
function candidateKnowledgeResearchOptions(candidateRoot, options) {
|
|
3563
|
+
if (!options.step && !options.knowledgeResearch) return void 0;
|
|
3564
|
+
const { step: researchStep, ...rest } = options.knowledgeResearch ?? {};
|
|
3565
|
+
const step = options.step ?? researchStep;
|
|
3566
|
+
return {
|
|
3567
|
+
...rest,
|
|
3568
|
+
root: candidateRoot,
|
|
3569
|
+
step,
|
|
3570
|
+
maxIterations: rest.maxIterations ?? (step ? options.candidateResearchIterations ?? 3 : void 0),
|
|
3571
|
+
strict: rest.strict ?? options.strict,
|
|
3572
|
+
readinessSpecs: rest.readinessSpecs ?? options.readinessSpecs,
|
|
3573
|
+
readinessTaskId: rest.readinessTaskId ?? options.readinessTaskId,
|
|
3574
|
+
readiness: rest.readiness ?? options.readiness
|
|
3575
|
+
};
|
|
3576
|
+
}
|
|
3577
|
+
function candidateUpdateHook(runId, candidate, options) {
|
|
3578
|
+
if (!options.updateKnowledge) return void 0;
|
|
3579
|
+
return (input) => options.updateKnowledge({
|
|
3580
|
+
...input,
|
|
3581
|
+
runId,
|
|
3582
|
+
iteration: candidate.iteration,
|
|
3583
|
+
candidateId: candidate.candidateId,
|
|
3584
|
+
root: candidate.candidateRoot,
|
|
3585
|
+
baselineRoot: options.root,
|
|
3586
|
+
candidateRoot: candidate.candidateRoot,
|
|
3587
|
+
baseHash: candidate.baseHash
|
|
3588
|
+
});
|
|
3589
|
+
}
|
|
3590
|
+
function shouldRunUpdateStage(options) {
|
|
3591
|
+
const phases = selectedStagePhases(options, UPDATE_PHASES);
|
|
3592
|
+
if (phases.length === 0) return false;
|
|
3593
|
+
return Boolean(
|
|
3594
|
+
options.acquireKnowledge || options.step || options.knowledgeResearch || options.updateKnowledge || selectedStageRequiredPhases(options, UPDATE_PHASES).length > 0
|
|
3595
|
+
);
|
|
3596
|
+
}
|
|
3597
|
+
function shouldRunEvaluationStage(options) {
|
|
3598
|
+
const phases = selectedStagePhases(options, EVALUATION_PHASES);
|
|
3599
|
+
if (phases.length === 0) return false;
|
|
3600
|
+
return Boolean(
|
|
3601
|
+
options.retrieval || options.diagnose || options.evaluateAnswers || options.decidePromotion || selectedStageRequiredPhases(options, EVALUATION_PHASES).length > 0
|
|
3602
|
+
);
|
|
3603
|
+
}
|
|
3604
|
+
function selectedStagePhases(options, stagePhases) {
|
|
3605
|
+
const requested = options.enabledPhases ?? stagePhases;
|
|
3606
|
+
return requested.filter((phase) => stagePhases.includes(phase));
|
|
3607
|
+
}
|
|
3608
|
+
function selectedStageRequiredPhases(options, stagePhases) {
|
|
3609
|
+
return (options.requiredPhases ?? []).filter((phase) => stagePhases.includes(phase));
|
|
3610
|
+
}
|
|
3611
|
+
function recordLifecycle(candidate, stage, lifecycle) {
|
|
3612
|
+
const record = {
|
|
3613
|
+
stage,
|
|
3614
|
+
phases: lifecycle.phases,
|
|
3615
|
+
findingCount: lifecycle.findings.length,
|
|
3616
|
+
retrievalWinnerConfig: lifecycle.retrieval?.winnerConfig,
|
|
3617
|
+
answerQuality: lifecycle.answerQuality,
|
|
3618
|
+
promotionDecision: lifecycle.promotion
|
|
3619
|
+
};
|
|
3620
|
+
candidate.lifecycle = [...candidate.lifecycle ?? [], record];
|
|
3621
|
+
candidate.retrievalWinnerConfig = lifecycle.retrieval?.winnerConfig ?? candidate.retrievalWinnerConfig;
|
|
3622
|
+
candidate.answerQuality = lifecycle.answerQuality ?? candidate.answerQuality;
|
|
3623
|
+
candidate.promotionDecision = lifecycle.promotion ?? candidate.promotionDecision;
|
|
3624
|
+
}
|
|
3625
|
+
function mergeLifecycleResults(goal, lifecycles) {
|
|
3626
|
+
if (lifecycles.length === 0) return void 0;
|
|
3627
|
+
return {
|
|
3628
|
+
goal,
|
|
3629
|
+
phases: lifecycles.flatMap((lifecycle) => lifecycle.phases),
|
|
3630
|
+
retrieval: lastDefined(lifecycles.map((lifecycle) => lifecycle.retrieval)),
|
|
3631
|
+
findings: lifecycles.flatMap((lifecycle) => lifecycle.findings),
|
|
3632
|
+
acquisition: lastDefined(lifecycles.map((lifecycle) => lifecycle.acquisition)),
|
|
3633
|
+
knowledgeUpdate: lastDefined(lifecycles.map((lifecycle) => lifecycle.knowledgeUpdate)),
|
|
3634
|
+
answerQuality: lastDefined(lifecycles.map((lifecycle) => lifecycle.answerQuality)),
|
|
3635
|
+
promotion: lastDefined(lifecycles.map((lifecycle) => lifecycle.promotion))
|
|
3636
|
+
};
|
|
3637
|
+
}
|
|
3638
|
+
function lastDefined(values) {
|
|
3639
|
+
for (let index = values.length - 1; index >= 0; index -= 1) {
|
|
3640
|
+
if (values[index] !== void 0) return values[index];
|
|
3641
|
+
}
|
|
3642
|
+
return void 0;
|
|
3643
|
+
}
|
|
3644
|
+
async function evaluateCandidate(runDir, state, candidate, lifecycle, options, now) {
|
|
3645
|
+
const [baselineIndex, candidateIndex] = await Promise.all([
|
|
3646
|
+
buildKnowledgeIndex(options.root),
|
|
3647
|
+
buildKnowledgeIndex(candidate.candidateRoot)
|
|
3648
|
+
]);
|
|
3649
|
+
const validation = validateKnowledgeIndex(candidateIndex, { strict: options.strict });
|
|
3650
|
+
const readiness = readinessFor(options, candidateIndex);
|
|
3651
|
+
const kbQuality = scoreKnowledgeBaseIndex(candidateIndex, {
|
|
3652
|
+
strict: options.strict,
|
|
3653
|
+
...options.kbQuality
|
|
3654
|
+
});
|
|
3655
|
+
const candidateHash = await hashKnowledgeBase(candidate.candidateRoot);
|
|
3656
|
+
const metric = options.evaluate?.({
|
|
3657
|
+
runId: state.runId,
|
|
3658
|
+
iteration: candidate.iteration,
|
|
3659
|
+
root: options.root,
|
|
3660
|
+
baselineRoot: options.root,
|
|
3661
|
+
candidateRoot: candidate.candidateRoot,
|
|
3662
|
+
baselineIndex,
|
|
3663
|
+
candidateIndex,
|
|
3664
|
+
baseHash: state.baseHash,
|
|
3665
|
+
candidateHash,
|
|
3666
|
+
validation,
|
|
3667
|
+
readiness,
|
|
3668
|
+
kbQuality,
|
|
3669
|
+
lifecycle,
|
|
3670
|
+
signal: options.signal
|
|
3671
|
+
}) ?? defaultKnowledgeImprovementMetric(
|
|
3672
|
+
validation,
|
|
3673
|
+
readiness,
|
|
3674
|
+
options.readinessSpecs,
|
|
3675
|
+
kbQuality,
|
|
3676
|
+
lifecycle
|
|
3677
|
+
);
|
|
3678
|
+
candidate.validation = validation;
|
|
3679
|
+
candidate.kbQuality = kbQuality;
|
|
3680
|
+
candidate.readinessBlockingMissing = readiness?.report.blockingMissingRequirements.length;
|
|
3681
|
+
candidate.evaluation = applyLifecycleFailures(normalizeMetric(await metric), lifecycle);
|
|
3682
|
+
candidate.candidateHash = candidateHash;
|
|
3683
|
+
candidate.updatedAt = now().toISOString();
|
|
3684
|
+
await appendLedger(runDir, {
|
|
3685
|
+
type: "candidate.evaluated",
|
|
3686
|
+
runId: state.runId,
|
|
3687
|
+
candidateId: candidate.candidateId,
|
|
3688
|
+
score: candidate.evaluation.score,
|
|
3689
|
+
passed: candidate.evaluation.passed
|
|
3690
|
+
});
|
|
3691
|
+
return candidate;
|
|
3692
|
+
}
|
|
3693
|
+
function defaultKnowledgeImprovementMetric(validation, readiness, readinessSpecs, kbQuality, lifecycle) {
|
|
3694
|
+
const blockingMissing = readiness?.report.blockingMissingRequirements.length ?? 0;
|
|
3695
|
+
const blockingTotal = readinessSpecs?.filter((spec) => spec.importance === "blocking").length ?? 0;
|
|
3696
|
+
const blockingReadiness = blockingTotal === 0 ? 1 : Math.max(0, blockingTotal - blockingMissing) / blockingTotal;
|
|
3697
|
+
const answerQuality = lifecycle?.answerQuality ? average2(Object.values(lifecycle.answerQuality.metrics).filter(Number.isFinite)) : 1;
|
|
3698
|
+
const promotionDecision = lifecycle?.promotion ? lifecycle.promotion.promoted ? 1 : 0 : 1;
|
|
3699
|
+
const dimensions = {
|
|
3700
|
+
validation: validation.ok ? 1 : 0,
|
|
3701
|
+
kb_quality: kbQuality.ok ? 1 : 0,
|
|
3702
|
+
blocking_readiness: blockingReadiness,
|
|
3703
|
+
answer_quality: answerQuality,
|
|
3704
|
+
promotion_decision: promotionDecision
|
|
3705
|
+
};
|
|
3706
|
+
const failedReasons = [
|
|
3707
|
+
validation.ok ? void 0 : "candidate validation failed",
|
|
3708
|
+
kbQuality.ok ? void 0 : "candidate KB quality check failed",
|
|
3709
|
+
blockingMissing === 0 ? void 0 : `${blockingMissing}/${blockingTotal} blocking knowledge requirements still missing`,
|
|
3710
|
+
lifecycle?.answerQuality && !lifecycle.answerQuality.passed ? "answer quality failed" : void 0,
|
|
3711
|
+
lifecycle?.promotion && !lifecycle.promotion.promoted ? `promotion decision held: ${lifecycle.promotion.reason}` : void 0
|
|
3712
|
+
].filter((reason) => Boolean(reason));
|
|
3713
|
+
return {
|
|
3714
|
+
score: average2(Object.values(dimensions)),
|
|
3715
|
+
passed: failedReasons.length === 0,
|
|
3716
|
+
dimensions,
|
|
3717
|
+
notes: failedReasons.length === 0 ? "candidate passed configured checks" : failedReasons.join("; ")
|
|
3718
|
+
};
|
|
3719
|
+
}
|
|
3720
|
+
function applyLifecycleFailures(metric, lifecycle) {
|
|
3721
|
+
const reasons = [
|
|
3722
|
+
metric.notes,
|
|
3723
|
+
lifecycle?.answerQuality && !lifecycle.answerQuality.passed ? "answer quality failed" : void 0,
|
|
3724
|
+
lifecycle?.promotion && !lifecycle.promotion.promoted ? `promotion decision held: ${lifecycle.promotion.reason}` : void 0
|
|
3725
|
+
].filter((reason) => Boolean(reason));
|
|
3726
|
+
const forcedFailure = Boolean(lifecycle?.answerQuality && !lifecycle.answerQuality.passed) || Boolean(lifecycle?.promotion && !lifecycle.promotion.promoted);
|
|
3727
|
+
return {
|
|
3728
|
+
...metric,
|
|
3729
|
+
passed: metric.passed && !forcedFailure,
|
|
3730
|
+
notes: reasons.length > 0 ? reasons.join("; ") : metric.notes
|
|
3731
|
+
};
|
|
3732
|
+
}
|
|
3733
|
+
function normalizeMetric(metric) {
|
|
3734
|
+
if (!Number.isFinite(metric.score) || metric.score < 0 || metric.score > 1) {
|
|
3735
|
+
throw new Error(`knowledge improvement score must be in [0, 1], got ${String(metric.score)}`);
|
|
3736
|
+
}
|
|
3737
|
+
return { ...metric, passed: Boolean(metric.passed) };
|
|
3738
|
+
}
|
|
3739
|
+
async function createCandidateWorkspace(runDir, state, root, now) {
|
|
3740
|
+
const iteration = state.candidates.length + 1;
|
|
3741
|
+
const candidateId = stableId("kcand", `${state.runId}:${iteration}:${now().toISOString()}`);
|
|
3742
|
+
const candidateRoot = join3(runDir, "candidates", candidateId, "workspace");
|
|
3743
|
+
await copyKnowledgeWorkspace(root, candidateRoot);
|
|
3744
|
+
const createdAt = now().toISOString();
|
|
3745
|
+
return {
|
|
3746
|
+
iteration,
|
|
3747
|
+
candidateId,
|
|
3748
|
+
candidateRoot,
|
|
3749
|
+
baseHash: state.baseHash,
|
|
3750
|
+
status: "running",
|
|
3751
|
+
createdAt,
|
|
3752
|
+
updatedAt: createdAt
|
|
3753
|
+
};
|
|
3754
|
+
}
|
|
3755
|
+
function findActiveCandidate(state) {
|
|
3756
|
+
return [...state.candidates].reverse().find((candidate) => candidate.status === "candidate-ready" || candidate.status === "running");
|
|
3757
|
+
}
|
|
3758
|
+
async function copyKnowledgeWorkspace(sourceRoot, targetRoot) {
|
|
3759
|
+
await rm(targetRoot, { recursive: true, force: true });
|
|
3760
|
+
await initKnowledgeBase(targetRoot);
|
|
3761
|
+
await copyIfExists(join3(sourceRoot, "knowledge"), join3(targetRoot, "knowledge"));
|
|
3762
|
+
await copyIfExists(join3(sourceRoot, "raw"), join3(targetRoot, "raw"));
|
|
3763
|
+
await copyIfExists(
|
|
3764
|
+
join3(layoutFor(sourceRoot).cacheDir, "sources.json"),
|
|
3765
|
+
join3(layoutFor(targetRoot).cacheDir, "sources.json")
|
|
3766
|
+
);
|
|
3767
|
+
await writeKnowledgeIndex(targetRoot);
|
|
3768
|
+
}
|
|
3769
|
+
async function promoteCandidate(root, candidateRoot) {
|
|
3770
|
+
await copyIfExists(join3(candidateRoot, "knowledge"), join3(root, "knowledge"), { replace: true });
|
|
3771
|
+
await copyIfExists(join3(candidateRoot, "raw"), join3(root, "raw"), { replace: true });
|
|
3772
|
+
await copyIfExists(
|
|
3773
|
+
join3(layoutFor(candidateRoot).cacheDir, "sources.json"),
|
|
3774
|
+
join3(layoutFor(root).cacheDir, "sources.json"),
|
|
3775
|
+
{ replace: true }
|
|
3776
|
+
);
|
|
3777
|
+
await writeKnowledgeIndex(root);
|
|
3778
|
+
}
|
|
3779
|
+
async function copyIfExists(source, target, options = {}) {
|
|
3780
|
+
try {
|
|
3781
|
+
await stat(source);
|
|
3782
|
+
} catch {
|
|
3783
|
+
return;
|
|
3784
|
+
}
|
|
3785
|
+
if (options.replace) await rm(target, { recursive: true, force: true });
|
|
3786
|
+
await mkdir3(dirname2(target), { recursive: true });
|
|
3787
|
+
await cp(source, target, { recursive: true });
|
|
3788
|
+
}
|
|
3789
|
+
async function hashKnowledgeBase(root) {
|
|
3790
|
+
const entries = [];
|
|
3791
|
+
for (const rel of ["knowledge", "raw"]) {
|
|
3792
|
+
entries.push(...await hashTreeEntries(root, rel));
|
|
3793
|
+
}
|
|
3794
|
+
const sourceRegistry = relative(root, layoutFor(root).sourceRegistryPath).replace(/\\/g, "/");
|
|
3795
|
+
entries.push(...await hashFileEntry(root, sourceRegistry));
|
|
3796
|
+
entries.sort((a, b) => a.path.localeCompare(b.path));
|
|
3797
|
+
return sha256(JSON.stringify(entries));
|
|
3798
|
+
}
|
|
3799
|
+
async function hashTreeEntries(root, relDir) {
|
|
3800
|
+
const abs = join3(root, relDir);
|
|
3801
|
+
try {
|
|
3802
|
+
const s = await stat(abs);
|
|
3803
|
+
if (!s.isDirectory()) return [];
|
|
3804
|
+
} catch {
|
|
3805
|
+
return [];
|
|
3806
|
+
}
|
|
3807
|
+
const out = [];
|
|
3808
|
+
const entries = await readdir(abs, { withFileTypes: true });
|
|
3809
|
+
for (const entry of entries) {
|
|
3810
|
+
const rel = join3(relDir, entry.name).replace(/\\/g, "/");
|
|
3811
|
+
if (entry.isDirectory()) out.push(...await hashTreeEntries(root, rel));
|
|
3812
|
+
else if (entry.isFile()) out.push(...await hashFileEntry(root, rel));
|
|
3813
|
+
}
|
|
3814
|
+
return out;
|
|
3815
|
+
}
|
|
3816
|
+
async function hashFileEntry(root, rel) {
|
|
3817
|
+
try {
|
|
3818
|
+
const bytes = await readFile2(join3(root, rel));
|
|
3819
|
+
return [{ path: rel, hash: sha256(bytes.toString("base64")) }];
|
|
3820
|
+
} catch {
|
|
3821
|
+
return [];
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3824
|
+
async function acquireRunLease(runDir, options) {
|
|
3825
|
+
const path = join3(runDir, "run.lock");
|
|
3826
|
+
const now = options.now();
|
|
3827
|
+
const expiresAt = new Date(now.getTime() + options.ttlMs);
|
|
3828
|
+
const payload = {
|
|
3829
|
+
ownerId: options.ownerId,
|
|
3830
|
+
acquiredAt: now.toISOString(),
|
|
3831
|
+
expiresAt: expiresAt.toISOString(),
|
|
3832
|
+
pid: process.pid
|
|
3833
|
+
};
|
|
3834
|
+
try {
|
|
3835
|
+
const handle = await open(path, "wx");
|
|
3836
|
+
try {
|
|
3837
|
+
await handle.writeFile(`${JSON.stringify(payload, null, 2)}
|
|
3838
|
+
`, "utf8");
|
|
3839
|
+
} finally {
|
|
3840
|
+
await handle.close();
|
|
3841
|
+
}
|
|
3842
|
+
} catch (error) {
|
|
3843
|
+
const code = error.code;
|
|
3844
|
+
if (code !== "EEXIST") throw error;
|
|
3845
|
+
const existing = await readLease(path);
|
|
3846
|
+
if (existing && new Date(existing.expiresAt).getTime() > now.getTime()) {
|
|
3847
|
+
throw new Error(
|
|
3848
|
+
`knowledge improvement run is locked by ${existing.ownerId} until ${existing.expiresAt}`
|
|
3849
|
+
);
|
|
3850
|
+
}
|
|
3851
|
+
await rm(path, { force: true });
|
|
3852
|
+
return acquireRunLease(runDir, options);
|
|
3853
|
+
}
|
|
3854
|
+
return {
|
|
3855
|
+
ownerId: options.ownerId,
|
|
3856
|
+
path,
|
|
3857
|
+
async release() {
|
|
3858
|
+
const current = await readLease(path);
|
|
3859
|
+
if (!current || current.ownerId === options.ownerId) await rm(path, { force: true });
|
|
3860
|
+
}
|
|
3861
|
+
};
|
|
3862
|
+
}
|
|
3863
|
+
async function readLease(path) {
|
|
3864
|
+
try {
|
|
3865
|
+
return JSON.parse(await readFile2(path, "utf8"));
|
|
3866
|
+
} catch {
|
|
3867
|
+
return null;
|
|
3868
|
+
}
|
|
3869
|
+
}
|
|
3870
|
+
async function blockRun(runDir, state, reason, onState, now) {
|
|
3871
|
+
state.status = "blocked";
|
|
3872
|
+
state.blockedReason = reason;
|
|
3873
|
+
state.updatedAt = now().toISOString();
|
|
3874
|
+
await saveState(runDir, state, onState);
|
|
3875
|
+
return state;
|
|
3876
|
+
}
|
|
3877
|
+
async function saveState(runDir, state, onState) {
|
|
3878
|
+
await writeJsonAtomic(statePath(runDir), state);
|
|
3879
|
+
await onState?.(state);
|
|
3880
|
+
}
|
|
3881
|
+
async function appendLedger(runDir, value) {
|
|
3882
|
+
const row = { at: (/* @__PURE__ */ new Date()).toISOString(), ...value };
|
|
3883
|
+
await mkdir3(runDir, { recursive: true });
|
|
3884
|
+
await writeFile3(join3(runDir, "events.jsonl"), `${JSON.stringify(row)}
|
|
3885
|
+
`, { flag: "a" });
|
|
3886
|
+
}
|
|
3887
|
+
async function writeJsonAtomic(path, value) {
|
|
3888
|
+
await mkdir3(dirname2(path), { recursive: true });
|
|
3889
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.tmp`;
|
|
3890
|
+
await writeFile3(tmp, `${JSON.stringify(value, null, 2)}
|
|
3891
|
+
`, "utf8");
|
|
3892
|
+
await rename(tmp, path);
|
|
3893
|
+
}
|
|
3894
|
+
function statePath(runDir) {
|
|
3895
|
+
return join3(runDir, "state.json");
|
|
3896
|
+
}
|
|
3897
|
+
function average2(values) {
|
|
3898
|
+
const finite = values.filter(Number.isFinite);
|
|
3899
|
+
if (finite.length === 0) return 0;
|
|
3900
|
+
return finite.reduce((sum, value) => sum + value, 0) / finite.length;
|
|
3901
|
+
}
|
|
3902
|
+
|
|
3903
|
+
// src/kb-store.ts
|
|
3904
|
+
import { mkdir as mkdir4, readFile as readFile3, writeFile as writeFile4 } from "fs/promises";
|
|
3905
|
+
import { dirname as dirname3, join as join4 } from "path";
|
|
3906
|
+
var MemoryKbStore = class {
|
|
3907
|
+
sources = /* @__PURE__ */ new Map();
|
|
3908
|
+
pages = /* @__PURE__ */ new Map();
|
|
3909
|
+
events = [];
|
|
3910
|
+
index = null;
|
|
3911
|
+
async putSource(source) {
|
|
3912
|
+
this.sources.set(source.id, clone(source));
|
|
3913
|
+
}
|
|
3914
|
+
async getSource(id) {
|
|
3915
|
+
return clone(this.sources.get(id) ?? null);
|
|
3916
|
+
}
|
|
3917
|
+
async listSources() {
|
|
3918
|
+
return [...this.sources.values()].map(clone);
|
|
3919
|
+
}
|
|
3920
|
+
async putPage(page) {
|
|
3921
|
+
this.pages.set(page.id, clone(page));
|
|
3922
|
+
}
|
|
3923
|
+
async getPage(idOrPath) {
|
|
3924
|
+
return clone(
|
|
3925
|
+
this.pages.get(idOrPath) ?? [...this.pages.values()].find((page) => page.path === idOrPath) ?? null
|
|
3926
|
+
);
|
|
3927
|
+
}
|
|
3928
|
+
async listPages() {
|
|
3929
|
+
return [...this.pages.values()].map(clone);
|
|
3930
|
+
}
|
|
3931
|
+
async putIndex(index) {
|
|
3932
|
+
this.index = clone(index);
|
|
3933
|
+
}
|
|
3934
|
+
async getIndex() {
|
|
3935
|
+
if (this.index) return clone(this.index);
|
|
3936
|
+
const pages = await this.listPages();
|
|
3937
|
+
const sources = await this.listSources();
|
|
3938
|
+
return {
|
|
3939
|
+
root: "memory",
|
|
3940
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3941
|
+
sources,
|
|
3942
|
+
pages,
|
|
3943
|
+
graph: buildKnowledgeGraph(pages)
|
|
3944
|
+
};
|
|
3945
|
+
}
|
|
3946
|
+
async putEvent(event) {
|
|
3947
|
+
this.events.push(clone(event));
|
|
3948
|
+
}
|
|
3949
|
+
async listEvents(query = {}) {
|
|
3950
|
+
let out = this.events;
|
|
3951
|
+
if (query.type) out = out.filter((event) => event.type === query.type);
|
|
3952
|
+
if (query.target) out = out.filter((event) => event.target === query.target);
|
|
3953
|
+
out = [...out].sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
3954
|
+
return out.slice(-(query.limit ?? out.length)).map(clone);
|
|
3955
|
+
}
|
|
3956
|
+
};
|
|
3957
|
+
var FileSystemKbStore = class extends MemoryKbStore {
|
|
3958
|
+
constructor(dir) {
|
|
3959
|
+
super();
|
|
3960
|
+
this.dir = dir;
|
|
3961
|
+
}
|
|
3962
|
+
dir;
|
|
3963
|
+
async putIndex(index) {
|
|
3964
|
+
await super.putIndex(index);
|
|
3965
|
+
await writeJson2(join4(this.dir, "index.json"), index);
|
|
3966
|
+
}
|
|
3967
|
+
async getIndex() {
|
|
3968
|
+
try {
|
|
3969
|
+
return JSON.parse(await readFile3(join4(this.dir, "index.json"), "utf8"));
|
|
3970
|
+
} catch {
|
|
3971
|
+
return await super.getIndex();
|
|
3972
|
+
}
|
|
3973
|
+
}
|
|
3974
|
+
async putEvent(event) {
|
|
3975
|
+
await super.putEvent(event);
|
|
3976
|
+
const events = await this.listEvents();
|
|
3977
|
+
await writeJson2(join4(this.dir, "events.json"), events);
|
|
3978
|
+
}
|
|
3979
|
+
};
|
|
3980
|
+
async function writeJson2(path, value) {
|
|
3981
|
+
await mkdir4(dirname3(path), { recursive: true });
|
|
3982
|
+
await writeFile4(path, `${JSON.stringify(value, null, 2)}
|
|
3983
|
+
`, "utf8");
|
|
3984
|
+
}
|
|
3985
|
+
function clone(value) {
|
|
3986
|
+
return value == null ? value : JSON.parse(JSON.stringify(value));
|
|
3987
|
+
}
|
|
3988
|
+
|
|
3989
|
+
// src/propose-from-finding.ts
|
|
3990
|
+
var KnowledgeProposalParseError = class extends Error {
|
|
3991
|
+
constructor(findingId, subject, message) {
|
|
3992
|
+
super(`proposeFromFinding(${findingId}, subject=${subject}): ${message}`);
|
|
3993
|
+
this.findingId = findingId;
|
|
3994
|
+
this.subject = subject;
|
|
3995
|
+
this.name = "KnowledgeProposalParseError";
|
|
3996
|
+
}
|
|
3997
|
+
findingId;
|
|
3998
|
+
subject;
|
|
3999
|
+
};
|
|
4000
|
+
function proposeFromFinding(finding) {
|
|
4001
|
+
if (!finding.subject) return null;
|
|
4002
|
+
if (!finding.subject.startsWith("agent-knowledge:")) return null;
|
|
4003
|
+
const rest = finding.subject.slice("agent-knowledge:".length);
|
|
4004
|
+
const [kindPart, ...locusParts] = rest.split(":");
|
|
4005
|
+
const locus = locusParts.join(":");
|
|
4006
|
+
if (!kindPart || !locus) {
|
|
4007
|
+
throw new KnowledgeProposalParseError(
|
|
4008
|
+
finding.finding_id,
|
|
4009
|
+
finding.subject,
|
|
4010
|
+
"expected `agent-knowledge:<kind>:<locus>` shape"
|
|
4011
|
+
);
|
|
4012
|
+
}
|
|
4013
|
+
const baseMeta = {
|
|
4014
|
+
severity: finding.severity,
|
|
4015
|
+
confidence: finding.confidence,
|
|
4016
|
+
evidence_uri: finding.evidence_refs[0]?.uri,
|
|
4017
|
+
analyst_id: finding.analyst_id
|
|
4018
|
+
};
|
|
4019
|
+
switch (kindPart) {
|
|
4020
|
+
case "wiki":
|
|
4021
|
+
return wikiProposal(finding, locus, baseMeta);
|
|
4022
|
+
case "claim":
|
|
4023
|
+
return claimProposal(finding, locus, baseMeta);
|
|
4024
|
+
case "raw":
|
|
4025
|
+
return liftRawProposal(finding, locus, baseMeta);
|
|
4026
|
+
case "stale":
|
|
4027
|
+
return markStaleProposal(finding, locus, baseMeta);
|
|
4028
|
+
default:
|
|
4029
|
+
throw new KnowledgeProposalParseError(
|
|
4030
|
+
finding.finding_id,
|
|
4031
|
+
finding.subject,
|
|
4032
|
+
`unknown kind "${kindPart}" (expected one of: wiki | claim | raw | stale)`
|
|
4033
|
+
);
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
function wikiProposal(finding, locus, metadata) {
|
|
4037
|
+
const hashIdx = locus.indexOf("#");
|
|
4038
|
+
const pageSlug = hashIdx >= 0 ? locus.slice(0, hashIdx) : locus;
|
|
4039
|
+
const heading = hashIdx >= 0 ? locus.slice(hashIdx + 1) : null;
|
|
4040
|
+
const path = `knowledge/${ensureSlug(pageSlug)}.md`;
|
|
4041
|
+
const body = renderWikiBody(finding, pageSlug, heading);
|
|
4042
|
+
return {
|
|
4043
|
+
id: `prop-${finding.finding_id}`,
|
|
4044
|
+
sourceFindingId: finding.finding_id,
|
|
4045
|
+
kind: heading ? "append-section" : "create-page",
|
|
4046
|
+
locus: pageSlug,
|
|
4047
|
+
writeBlocks: [{ path, content: body }],
|
|
4048
|
+
metadata
|
|
4049
|
+
};
|
|
4050
|
+
}
|
|
4051
|
+
function claimProposal(finding, locus, metadata) {
|
|
4052
|
+
const refs = finding.evidence_refs.filter((r) => r.uri).map((r) => ({
|
|
4053
|
+
sourceId: `analyst-finding:${finding.finding_id}`,
|
|
4054
|
+
anchorId: r.uri,
|
|
4055
|
+
quote: r.excerpt
|
|
4056
|
+
}));
|
|
4057
|
+
const claim = {
|
|
4058
|
+
id: `claim-${finding.finding_id}`,
|
|
4059
|
+
text: finding.recommended_action ?? finding.claim,
|
|
4060
|
+
refs,
|
|
4061
|
+
confidence: finding.confidence,
|
|
4062
|
+
status: "draft",
|
|
4063
|
+
metadata: {
|
|
4064
|
+
analyst_id: finding.analyst_id,
|
|
4065
|
+
source_finding_id: finding.finding_id,
|
|
4066
|
+
topic: locus
|
|
4067
|
+
}
|
|
4068
|
+
};
|
|
4069
|
+
return {
|
|
4070
|
+
id: `prop-${finding.finding_id}`,
|
|
4071
|
+
sourceFindingId: finding.finding_id,
|
|
4072
|
+
kind: "create-claim",
|
|
4073
|
+
locus,
|
|
4074
|
+
writeBlocks: [],
|
|
4075
|
+
claim,
|
|
4076
|
+
metadata
|
|
4077
|
+
};
|
|
4078
|
+
}
|
|
4079
|
+
function liftRawProposal(finding, sourceId, metadata) {
|
|
4080
|
+
const path = `knowledge/${ensureSlug(sourceId)}.md`;
|
|
4081
|
+
const body = [
|
|
4082
|
+
"---",
|
|
4083
|
+
`title: ${sourceId}`,
|
|
4084
|
+
`source: ${sourceId}`,
|
|
4085
|
+
`status: draft`,
|
|
4086
|
+
`lifted_from_finding: ${finding.finding_id}`,
|
|
4087
|
+
"---",
|
|
4088
|
+
"",
|
|
4089
|
+
"## Why this page exists",
|
|
4090
|
+
"",
|
|
4091
|
+
finding.claim,
|
|
4092
|
+
"",
|
|
4093
|
+
...finding.rationale ? ["## Rationale", "", finding.rationale, ""] : [],
|
|
4094
|
+
...finding.recommended_action ? ["## Recommended action", "", finding.recommended_action, ""] : []
|
|
4095
|
+
].join("\n");
|
|
4096
|
+
return {
|
|
4097
|
+
id: `prop-${finding.finding_id}`,
|
|
4098
|
+
sourceFindingId: finding.finding_id,
|
|
4099
|
+
kind: "lift-raw",
|
|
4100
|
+
locus: sourceId,
|
|
4101
|
+
writeBlocks: [{ path, content: body }],
|
|
4102
|
+
metadata
|
|
4103
|
+
};
|
|
4104
|
+
}
|
|
4105
|
+
function markStaleProposal(finding, pageSlug, metadata) {
|
|
4106
|
+
const path = `knowledge/${ensureSlug(pageSlug)}.stale.md`;
|
|
4107
|
+
const body = [
|
|
4108
|
+
"---",
|
|
4109
|
+
`title: ${pageSlug} (marked stale)`,
|
|
4110
|
+
`status: superseded`,
|
|
4111
|
+
`superseded_by_finding: ${finding.finding_id}`,
|
|
4112
|
+
`confidence: ${finding.confidence}`,
|
|
4113
|
+
"---",
|
|
4114
|
+
"",
|
|
4115
|
+
"## Why marked stale",
|
|
4116
|
+
"",
|
|
4117
|
+
finding.claim,
|
|
4118
|
+
"",
|
|
4119
|
+
...finding.rationale ? ["## Evidence", "", finding.rationale, ""] : [],
|
|
4120
|
+
...finding.recommended_action ? ["## Action", "", finding.recommended_action, ""] : []
|
|
4121
|
+
].join("\n");
|
|
4122
|
+
return {
|
|
4123
|
+
id: `prop-${finding.finding_id}`,
|
|
4124
|
+
sourceFindingId: finding.finding_id,
|
|
4125
|
+
kind: "mark-stale",
|
|
4126
|
+
locus: pageSlug,
|
|
4127
|
+
writeBlocks: [{ path, content: body }],
|
|
4128
|
+
metadata
|
|
4129
|
+
};
|
|
4130
|
+
}
|
|
4131
|
+
function proposeFromFindings(findings) {
|
|
4132
|
+
const proposals = [];
|
|
4133
|
+
const errors = [];
|
|
4134
|
+
let skipped = 0;
|
|
4135
|
+
for (const f of findings) {
|
|
4136
|
+
try {
|
|
4137
|
+
const p = proposeFromFinding(f);
|
|
4138
|
+
if (p) proposals.push(p);
|
|
4139
|
+
else skipped += 1;
|
|
4140
|
+
} catch (err) {
|
|
4141
|
+
if (err instanceof KnowledgeProposalParseError) errors.push(err);
|
|
4142
|
+
else throw err;
|
|
4143
|
+
}
|
|
4144
|
+
}
|
|
4145
|
+
return { proposals, skipped, errors };
|
|
4146
|
+
}
|
|
4147
|
+
function ensureSlug(s) {
|
|
4148
|
+
return s.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 200) || "untitled";
|
|
4149
|
+
}
|
|
4150
|
+
function renderWikiBody(finding, slug, heading) {
|
|
4151
|
+
const title = humanize(slug);
|
|
4152
|
+
if (heading) {
|
|
4153
|
+
return [
|
|
4154
|
+
`## ${heading}`,
|
|
4155
|
+
"",
|
|
4156
|
+
finding.claim,
|
|
4157
|
+
"",
|
|
4158
|
+
...finding.rationale ? ["### Rationale", "", finding.rationale, ""] : [],
|
|
4159
|
+
...finding.recommended_action ? ["### Action", "", finding.recommended_action, ""] : [],
|
|
4160
|
+
`_Drafted from finding ${finding.finding_id} (confidence ${finding.confidence.toFixed(2)})._`
|
|
4161
|
+
].join("\n");
|
|
4162
|
+
}
|
|
4163
|
+
return [
|
|
4164
|
+
"---",
|
|
4165
|
+
`title: ${title}`,
|
|
4166
|
+
`status: draft`,
|
|
4167
|
+
`drafted_from_finding: ${finding.finding_id}`,
|
|
4168
|
+
`confidence: ${finding.confidence}`,
|
|
4169
|
+
"---",
|
|
4170
|
+
"",
|
|
4171
|
+
`# ${title}`,
|
|
4172
|
+
"",
|
|
4173
|
+
finding.claim,
|
|
4174
|
+
"",
|
|
4175
|
+
...finding.rationale ? ["## Rationale", "", finding.rationale, ""] : [],
|
|
4176
|
+
...finding.recommended_action ? ["## Recommended action", "", finding.recommended_action, ""] : []
|
|
4177
|
+
].join("\n");
|
|
4178
|
+
}
|
|
4179
|
+
function humanize(slug) {
|
|
4180
|
+
return slug.split("-").filter(Boolean).map((w) => w[0]?.toUpperCase() + w.slice(1)).join(" ") || slug;
|
|
4181
|
+
}
|
|
4182
|
+
|
|
4183
|
+
// src/readiness-check.ts
|
|
4184
|
+
async function evaluateKnowledgeBaseReadiness(options) {
|
|
4185
|
+
const index = await buildKnowledgeIndex(options.root);
|
|
4186
|
+
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
4187
|
+
const readiness = readinessFor(
|
|
4188
|
+
{
|
|
4189
|
+
...options,
|
|
4190
|
+
readinessSpecs: options.readinessSpecs ? [...options.readinessSpecs] : void 0
|
|
4191
|
+
},
|
|
4192
|
+
index
|
|
4193
|
+
);
|
|
4194
|
+
const kbQuality = scoreKnowledgeBaseIndex(index, {
|
|
4195
|
+
strict: options.strict,
|
|
4196
|
+
...options.kbQuality
|
|
4197
|
+
});
|
|
4198
|
+
const blockingMissing = readiness?.report.blockingMissingRequirements.length ?? 0;
|
|
4199
|
+
const blockingTotal = options.readinessSpecs?.filter((spec) => spec.importance === "blocking").length ?? 0;
|
|
4200
|
+
const blockingReadiness = blockingTotal === 0 ? 1 : Math.max(0, blockingTotal - blockingMissing) / blockingTotal;
|
|
4201
|
+
const ready = validation.ok && kbQuality.ok && blockingMissing === 0;
|
|
4202
|
+
const failures = [
|
|
4203
|
+
validation.ok ? void 0 : `${validation.findings.length} validation finding(s)`,
|
|
4204
|
+
kbQuality.ok ? void 0 : `${kbQuality.findings.length} KB quality finding(s)`,
|
|
4205
|
+
blockingMissing === 0 ? void 0 : `${blockingMissing}/${blockingTotal} blocking readiness requirement(s) missing`
|
|
4206
|
+
].filter((failure) => Boolean(failure));
|
|
4207
|
+
return {
|
|
4208
|
+
ready,
|
|
4209
|
+
summary: ready ? "knowledge base passed readiness checks" : failures.join("; "),
|
|
4210
|
+
index,
|
|
4211
|
+
validation,
|
|
4212
|
+
readiness,
|
|
4213
|
+
kbQuality,
|
|
4214
|
+
dimensions: {
|
|
4215
|
+
validation: validation.ok ? 1 : 0,
|
|
4216
|
+
kb_quality: kbQuality.ok ? 1 : 0,
|
|
4217
|
+
blocking_readiness: blockingReadiness
|
|
4218
|
+
}
|
|
4219
|
+
};
|
|
4220
|
+
}
|
|
4221
|
+
|
|
4222
|
+
// src/release.ts
|
|
4223
|
+
import {
|
|
4224
|
+
evaluateReleaseConfidence,
|
|
4225
|
+
validateRunRecord
|
|
4226
|
+
} from "@tangle-network/agent-eval";
|
|
4227
|
+
function knowledgeReleaseReport(input) {
|
|
4228
|
+
const baselineRuns = input.baselineRuns ?? [];
|
|
4229
|
+
const runRecords = [...input.candidateRuns, ...baselineRuns].map(validateRunRecord);
|
|
4230
|
+
const scorecard = evaluateReleaseConfidence({
|
|
4231
|
+
target: "agent-knowledge-base",
|
|
4232
|
+
candidateId: input.candidateId,
|
|
4233
|
+
baselineId: input.baselineId ?? "baseline",
|
|
4234
|
+
traces: input.traces ?? [],
|
|
4235
|
+
runs: runRecords,
|
|
4236
|
+
gateDecision: input.gateDecision ?? null,
|
|
4237
|
+
thresholds: {
|
|
4238
|
+
requireCorpus: false,
|
|
4239
|
+
// The report gates on RunRecord-level outcomes, not on a separate scenario
|
|
4240
|
+
// corpus (KnowledgeReleaseInput carries no dataset/scenarios). The scenario
|
|
4241
|
+
// floor must therefore be 0 — otherwise the corpus axis fails closed on a
|
|
4242
|
+
// dimension the report has no way to satisfy, masking the run-based gate.
|
|
4243
|
+
minScenarioCount: 0,
|
|
4244
|
+
requireHoldout: input.hasHoldout ?? false,
|
|
4245
|
+
minHoldoutRuns: input.hasHoldout ? 1 : 0,
|
|
4246
|
+
minSearchRuns: 1,
|
|
4247
|
+
minMeanScore: input.minScore ?? 0.7
|
|
4248
|
+
}
|
|
4249
|
+
});
|
|
4250
|
+
const release = {
|
|
4251
|
+
id: stableId("krel", `${input.candidateId}:${input.createdAt ?? (/* @__PURE__ */ new Date()).toISOString()}`),
|
|
4252
|
+
candidateId: input.candidateId,
|
|
4253
|
+
createdAt: input.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
4254
|
+
promoted: scorecard.status !== "fail" && (input.promotedIsBest ?? true),
|
|
4255
|
+
scorecard,
|
|
4256
|
+
runRecordIds: runRecords.map((record) => record.runId)
|
|
4257
|
+
};
|
|
4258
|
+
return { release, scorecard, candidateRuns: input.candidateRuns, baselineRuns };
|
|
4259
|
+
}
|
|
4260
|
+
|
|
4261
|
+
// src/research-driving-driver.ts
|
|
4262
|
+
function createResearchDrivingDriver(options = {}) {
|
|
4263
|
+
const minIndependentSources = Math.max(2, options.minIndependentSources ?? 2);
|
|
4264
|
+
const maxQuestionsPerRound = Math.max(1, options.maxQuestionsPerRound ?? 6);
|
|
4265
|
+
const maxClaimsPerSource = Math.max(1, options.maxClaimsPerSource ?? 3);
|
|
4266
|
+
const deterministicFallback = options.deterministicFallback ?? true;
|
|
4267
|
+
const claims = /* @__PURE__ */ new Map();
|
|
4268
|
+
const questions = /* @__PURE__ */ new Map();
|
|
4269
|
+
let rounds = 0;
|
|
4270
|
+
let lastSteer;
|
|
4271
|
+
function resolveRouter2() {
|
|
4272
|
+
return options.router ?? createTangleRouterClient(options.router_options);
|
|
4273
|
+
}
|
|
4274
|
+
function recordClaim(extracted, sourceUri, round2) {
|
|
4275
|
+
const id = claimId(extracted.text);
|
|
4276
|
+
const host = hostOf2(sourceUri);
|
|
4277
|
+
const existing = claims.get(id);
|
|
4278
|
+
if (existing) {
|
|
4279
|
+
if (host) existing.supportingHosts.add(host);
|
|
4280
|
+
if (!existing.supportingUris.includes(sourceUri)) existing.supportingUris.push(sourceUri);
|
|
4281
|
+
linkContradiction(existing, extracted.contradictsExistingId);
|
|
4282
|
+
return existing;
|
|
4283
|
+
}
|
|
4284
|
+
const tracked = {
|
|
4285
|
+
id,
|
|
4286
|
+
text: extracted.text.trim(),
|
|
4287
|
+
supportingHosts: new Set(host ? [host] : []),
|
|
4288
|
+
supportingUris: [sourceUri],
|
|
4289
|
+
contradicts: /* @__PURE__ */ new Set(),
|
|
4290
|
+
contested: false,
|
|
4291
|
+
firstSeenRound: round2
|
|
4292
|
+
};
|
|
4293
|
+
linkContradiction(tracked, extracted.contradictsExistingId);
|
|
4294
|
+
claims.set(id, tracked);
|
|
4295
|
+
return tracked;
|
|
4296
|
+
}
|
|
4297
|
+
function linkContradiction(claim, otherId) {
|
|
4298
|
+
if (!otherId || otherId === claim.id) return;
|
|
4299
|
+
const other = claims.get(otherId);
|
|
4300
|
+
if (!other) return;
|
|
4301
|
+
claim.contradicts.add(otherId);
|
|
4302
|
+
other.contradicts.add(claim.id);
|
|
4303
|
+
claim.contested = true;
|
|
4304
|
+
other.contested = true;
|
|
4305
|
+
}
|
|
4306
|
+
function independentSupport(claim) {
|
|
4307
|
+
return claim.supportingHosts.size;
|
|
4308
|
+
}
|
|
4309
|
+
function isCorroborated(claim) {
|
|
4310
|
+
return independentSupport(claim) >= minIndependentSources;
|
|
4311
|
+
}
|
|
4312
|
+
function isWeak(claim) {
|
|
4313
|
+
return !isCorroborated(claim) && !claim.contested;
|
|
4314
|
+
}
|
|
4315
|
+
function snapshot() {
|
|
4316
|
+
const all = [...claims.values()];
|
|
4317
|
+
const allQuestions = [...questions.values()];
|
|
4318
|
+
return {
|
|
4319
|
+
claims: all,
|
|
4320
|
+
questions: allQuestions,
|
|
4321
|
+
weaklySupported: all.filter(isWeak),
|
|
4322
|
+
corroborated: all.filter(isCorroborated),
|
|
4323
|
+
contested: all.filter((claim) => claim.contested),
|
|
4324
|
+
openQuestions: allQuestions.filter((question) => !question.addressed),
|
|
4325
|
+
rounds
|
|
4326
|
+
};
|
|
4327
|
+
}
|
|
4328
|
+
function markAddressed(newClaimTexts) {
|
|
4329
|
+
for (const question of questions.values()) {
|
|
4330
|
+
if (question.addressed) continue;
|
|
4331
|
+
if (question.kind === "contradiction") {
|
|
4332
|
+
const settled = question.claimIds.some((id) => {
|
|
4333
|
+
const claim = claims.get(id);
|
|
4334
|
+
return claim ? isCorroborated(claim) || claim.contested : false;
|
|
4335
|
+
});
|
|
4336
|
+
if (settled) question.addressed = true;
|
|
4337
|
+
continue;
|
|
4338
|
+
}
|
|
4339
|
+
const qWords = contentWordSet(question.text);
|
|
4340
|
+
if (qWords.size === 0) continue;
|
|
4341
|
+
for (const text of newClaimTexts) {
|
|
4342
|
+
const overlap = overlapFraction(qWords, contentWordSet(text));
|
|
4343
|
+
if (overlap >= 0.5) {
|
|
4344
|
+
question.addressed = true;
|
|
4345
|
+
break;
|
|
4346
|
+
}
|
|
4347
|
+
}
|
|
4348
|
+
}
|
|
4349
|
+
}
|
|
4350
|
+
return {
|
|
4351
|
+
/**
|
|
4352
|
+
* `verifySource` — the per-source hook. This driver does NOT filter for
|
|
4353
|
+
* relevance/dedup (other drivers do that, and the loop already dedups exact
|
|
4354
|
+
* uris). Its job here is to EXTRACT the source's claims and grow the ledger.
|
|
4355
|
+
* It accepts every source that yields at least one extractable claim; it
|
|
4356
|
+
* only rejects a source with NO extractable signal at all (empty/unusable),
|
|
4357
|
+
* because such a source cannot drive the research and pollutes the KB.
|
|
4358
|
+
*/
|
|
4359
|
+
async verifySource(source, ctx) {
|
|
4360
|
+
const extracted = await extractClaims(source, ctx);
|
|
4361
|
+
if (extracted.length === 0) {
|
|
4362
|
+
return {
|
|
4363
|
+
accept: false,
|
|
4364
|
+
reason: "no extractable claim: source yields nothing to drive the research deeper"
|
|
4365
|
+
};
|
|
4366
|
+
}
|
|
4367
|
+
const newTexts = [];
|
|
4368
|
+
for (const claim of extracted) {
|
|
4369
|
+
recordClaim(claim, source.uri, ctx.round);
|
|
4370
|
+
newTexts.push(claim.text);
|
|
4371
|
+
}
|
|
4372
|
+
markAddressed(newTexts);
|
|
4373
|
+
return { accept: true };
|
|
4374
|
+
},
|
|
4375
|
+
/**
|
|
4376
|
+
* `foldGaps` — the DEPTH driver. Runs after the worker's contribution is
|
|
4377
|
+
* applied each round. It builds the next round's steer from (1) the readiness
|
|
4378
|
+
* gaps the loop still reports, (2) freshly generated DEEP sub-questions, and
|
|
4379
|
+
* (3) INVALIDATION challenges for weakly-supported / contradicted claims.
|
|
4380
|
+
*/
|
|
4381
|
+
foldGaps(gaps) {
|
|
4382
|
+
rounds += 1;
|
|
4383
|
+
const round2 = rounds;
|
|
4384
|
+
const ledger = [...claims.values()];
|
|
4385
|
+
const invalidationTargets = ledger.filter(
|
|
4386
|
+
(claim) => isWeak(claim) || claim.contradicts.size > 0
|
|
4387
|
+
);
|
|
4388
|
+
const deepQuestions = synthesizeDeepQuestions(ledger, round2).slice(0, maxQuestionsPerRound);
|
|
4389
|
+
for (const question of deepQuestions) {
|
|
4390
|
+
if (!questions.has(question.id)) questions.set(question.id, question);
|
|
4391
|
+
}
|
|
4392
|
+
const text = buildSteerText(gaps, deepQuestions, invalidationTargets, minIndependentSources);
|
|
4393
|
+
lastSteer = { round: round2, deepQuestions, invalidationTargets, gaps, text };
|
|
4394
|
+
options.onSteer?.(lastSteer);
|
|
4395
|
+
return text;
|
|
4396
|
+
},
|
|
4397
|
+
researchState: snapshot,
|
|
4398
|
+
isComplete() {
|
|
4399
|
+
const all = [...claims.values()];
|
|
4400
|
+
if (all.length === 0) return false;
|
|
4401
|
+
const everyClaimSettled = all.every((claim) => isCorroborated(claim) || claim.contested);
|
|
4402
|
+
const everyQuestionAddressed = [...questions.values()].every((question) => question.addressed);
|
|
4403
|
+
return everyClaimSettled && everyQuestionAddressed;
|
|
4404
|
+
},
|
|
4405
|
+
lastSteer() {
|
|
4406
|
+
return lastSteer;
|
|
4407
|
+
}
|
|
4408
|
+
};
|
|
4409
|
+
async function extractClaims(source, ctx) {
|
|
4410
|
+
const ledger = [...claims.values()];
|
|
4411
|
+
const fromLlm = await extractClaimsWithLlm(source, ctx, ledger);
|
|
4412
|
+
if (fromLlm.length > 0) return fromLlm.slice(0, maxClaimsPerSource);
|
|
4413
|
+
if (deterministicFallback) return deterministicClaims(source).slice(0, maxClaimsPerSource);
|
|
4414
|
+
return [];
|
|
4415
|
+
}
|
|
4416
|
+
async function extractClaimsWithLlm(source, ctx, ledger) {
|
|
4417
|
+
let router;
|
|
4418
|
+
try {
|
|
4419
|
+
router = resolveRouter2();
|
|
4420
|
+
} catch {
|
|
4421
|
+
return [];
|
|
4422
|
+
}
|
|
4423
|
+
const excerpt = source.text.slice(0, 1800);
|
|
4424
|
+
const ledgerLines = ledger.slice(0, 20).map((claim) => `- [${claim.id}] ${claim.text}`).join("\n");
|
|
4425
|
+
const system = `You extract the KEY factual claims a researcher would cite a page for, and flag CONTRADICTIONS with claims already on the ledger. A claim is one concrete, checkable assertion using the page's own terms and numbers. Return ONLY a JSON array; each item is {"claim": string, "contradicts": string|null} where contradicts is the bracketed [id] of a ledger claim this page DIRECTLY contradicts, else null. Return at most ${maxClaimsPerSource} claims. No prose.`;
|
|
4426
|
+
const user = [
|
|
4427
|
+
`Research goal: ${ctx.goal}`,
|
|
4428
|
+
`Page title: ${source.title ?? "(none)"}`,
|
|
4429
|
+
ledgerLines ? `Claims already on the ledger:
|
|
4430
|
+
${ledgerLines}` : "Ledger is empty.",
|
|
4431
|
+
`Page excerpt:
|
|
4432
|
+
${excerpt}`,
|
|
4433
|
+
'Key claims as JSON [{"claim": "...", "contradicts": "[id]"|null}]:'
|
|
4434
|
+
].join("\n\n");
|
|
4435
|
+
let raw = "";
|
|
4436
|
+
try {
|
|
4437
|
+
raw = await router.chat(
|
|
4438
|
+
[
|
|
4439
|
+
{ role: "system", content: system },
|
|
4440
|
+
{ role: "user", content: user }
|
|
4441
|
+
],
|
|
4442
|
+
1200
|
|
4443
|
+
);
|
|
4444
|
+
} catch {
|
|
4445
|
+
return [];
|
|
4446
|
+
}
|
|
4447
|
+
return parseExtractedClaims(raw, ledger);
|
|
4448
|
+
}
|
|
4449
|
+
function deterministicClaims(source) {
|
|
4450
|
+
const sentences = source.text.split(/(?<=[.!?])\s+/).map((sentence) => sentence.trim()).filter((sentence) => contentWordSet(sentence).size >= 3);
|
|
4451
|
+
return sentences.slice(0, maxClaimsPerSource).map((text) => ({ text }));
|
|
4452
|
+
}
|
|
4453
|
+
function synthesizeDeepQuestions(ledger, round2) {
|
|
4454
|
+
if (ledger.length === 0) return [];
|
|
4455
|
+
const out = [];
|
|
4456
|
+
for (const claim of ledger) {
|
|
4457
|
+
for (const otherId of claim.contradicts) {
|
|
4458
|
+
const other = ledger.find((entry) => entry.id === otherId);
|
|
4459
|
+
if (!other || other.id < claim.id) continue;
|
|
4460
|
+
out.push(
|
|
4461
|
+
makeQuestion(
|
|
4462
|
+
"contradiction",
|
|
4463
|
+
`Does any independent source resolve the contradiction between "${truncate(claim.text)}" and "${truncate(other.text)}"? Find evidence that confirms or refutes one of them.`,
|
|
4464
|
+
[claim.id, other.id],
|
|
4465
|
+
round2
|
|
4466
|
+
)
|
|
4467
|
+
);
|
|
4468
|
+
}
|
|
4469
|
+
}
|
|
4470
|
+
for (const claim of ledger.filter((entry) => !entry.contested)) {
|
|
4471
|
+
if (claim.supportingHosts.size < minIndependentSources) {
|
|
4472
|
+
out.push(
|
|
4473
|
+
makeQuestion(
|
|
4474
|
+
"gap",
|
|
4475
|
+
`Only one independent source supports "${truncate(claim.text)}". What specific corroborating result, dataset, or independent measurement is missing to confirm it?`,
|
|
4476
|
+
[claim.id],
|
|
4477
|
+
round2
|
|
4478
|
+
)
|
|
4479
|
+
);
|
|
4480
|
+
}
|
|
4481
|
+
}
|
|
4482
|
+
for (const claim of [...ledger].sort((a, b) => b.supportingHosts.size - a.supportingHosts.size).slice(0, 2)) {
|
|
4483
|
+
out.push(
|
|
4484
|
+
makeQuestion(
|
|
4485
|
+
"mechanism",
|
|
4486
|
+
`Under what precise condition does "${truncate(claim.text)}" stop holding? Find a source that states the mechanism, limit, or failure mode.`,
|
|
4487
|
+
[claim.id],
|
|
4488
|
+
round2
|
|
4489
|
+
)
|
|
4490
|
+
);
|
|
4491
|
+
}
|
|
4492
|
+
const ranked = [...ledger].sort((a, b) => b.supportingHosts.size - a.supportingHosts.size);
|
|
4493
|
+
if (ranked.length >= 2 && ranked[0] && ranked[1]) {
|
|
4494
|
+
out.push(
|
|
4495
|
+
makeQuestion(
|
|
4496
|
+
"comparative",
|
|
4497
|
+
`How does the tradeoff in "${truncate(ranked[0].text)}" differ from "${truncate(ranked[1].text)}"? Find a source that compares them directly.`,
|
|
4498
|
+
[ranked[0].id, ranked[1].id],
|
|
4499
|
+
round2
|
|
4500
|
+
)
|
|
4501
|
+
);
|
|
4502
|
+
}
|
|
4503
|
+
const priority = {
|
|
4504
|
+
contradiction: 0,
|
|
4505
|
+
gap: 1,
|
|
4506
|
+
mechanism: 2,
|
|
4507
|
+
comparative: 3
|
|
4508
|
+
};
|
|
4509
|
+
return out.sort((a, b) => priority[a.kind] - priority[b.kind]);
|
|
4510
|
+
}
|
|
4511
|
+
}
|
|
4512
|
+
function makeQuestion(kind, text, claimIds, raisedRound) {
|
|
4513
|
+
return {
|
|
4514
|
+
kind,
|
|
4515
|
+
text,
|
|
4516
|
+
id: `q_${sha256(`${kind}:${text}`).slice(0, 16)}`,
|
|
4517
|
+
claimIds,
|
|
4518
|
+
addressed: false,
|
|
4519
|
+
raisedRound
|
|
4520
|
+
};
|
|
4521
|
+
}
|
|
4522
|
+
function claimId(text) {
|
|
4523
|
+
return `c_${sha256(normalizeText(text)).slice(0, 16)}`;
|
|
4524
|
+
}
|
|
4525
|
+
function hostOf2(uri) {
|
|
4526
|
+
try {
|
|
4527
|
+
return new URL(uri.trim()).hostname.toLowerCase().replace(/^www\./, "");
|
|
4528
|
+
} catch {
|
|
4529
|
+
return canonicalizeUrl(uri);
|
|
4530
|
+
}
|
|
4531
|
+
}
|
|
4532
|
+
function normalizeText(text) {
|
|
4533
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]+/gu, " ").replace(/\s+/g, " ").trim();
|
|
4534
|
+
}
|
|
4535
|
+
var stopwords3 = /* @__PURE__ */ new Set([
|
|
4536
|
+
"the",
|
|
4537
|
+
"a",
|
|
4538
|
+
"an",
|
|
4539
|
+
"and",
|
|
4540
|
+
"or",
|
|
4541
|
+
"but",
|
|
4542
|
+
"of",
|
|
4543
|
+
"to",
|
|
4544
|
+
"in",
|
|
4545
|
+
"on",
|
|
4546
|
+
"for",
|
|
4547
|
+
"with",
|
|
4548
|
+
"as",
|
|
4549
|
+
"by",
|
|
4550
|
+
"at",
|
|
4551
|
+
"from",
|
|
4552
|
+
"that",
|
|
4553
|
+
"this",
|
|
4554
|
+
"these",
|
|
4555
|
+
"those",
|
|
4556
|
+
"it",
|
|
4557
|
+
"its",
|
|
4558
|
+
"is",
|
|
4559
|
+
"are",
|
|
4560
|
+
"was",
|
|
4561
|
+
"were",
|
|
4562
|
+
"be",
|
|
4563
|
+
"been",
|
|
4564
|
+
"being",
|
|
4565
|
+
"has",
|
|
4566
|
+
"have",
|
|
4567
|
+
"had",
|
|
4568
|
+
"can",
|
|
4569
|
+
"will",
|
|
4570
|
+
"would",
|
|
4571
|
+
"should",
|
|
4572
|
+
"may",
|
|
4573
|
+
"might",
|
|
4574
|
+
"not",
|
|
4575
|
+
"no",
|
|
4576
|
+
"than",
|
|
4577
|
+
"then",
|
|
4578
|
+
"over",
|
|
4579
|
+
"under",
|
|
4580
|
+
"about",
|
|
4581
|
+
"into",
|
|
4582
|
+
"their",
|
|
4583
|
+
"they",
|
|
4584
|
+
"them"
|
|
4585
|
+
]);
|
|
4586
|
+
function contentWordSet(text) {
|
|
4587
|
+
return new Set(
|
|
4588
|
+
normalizeText(text).split(" ").filter((word) => word.length >= 3 && !stopwords3.has(word))
|
|
4589
|
+
);
|
|
4590
|
+
}
|
|
4591
|
+
function overlapFraction(a, b) {
|
|
4592
|
+
if (a.size === 0) return 0;
|
|
4593
|
+
let hits = 0;
|
|
4594
|
+
for (const word of a) if (b.has(word)) hits += 1;
|
|
4595
|
+
return hits / a.size;
|
|
4596
|
+
}
|
|
4597
|
+
function truncate(text, max = 140) {
|
|
4598
|
+
const t = text.trim().replace(/\s+/g, " ");
|
|
4599
|
+
return t.length <= max ? t : `${t.slice(0, max - 1)}\u2026`;
|
|
4600
|
+
}
|
|
4601
|
+
function parseExtractedClaims(raw, ledger) {
|
|
4602
|
+
const text = raw.trim();
|
|
4603
|
+
if (!text) return [];
|
|
4604
|
+
const arrayMatch = text.match(/\[[\s\S]*\]/);
|
|
4605
|
+
if (!arrayMatch) return [];
|
|
4606
|
+
let parsed;
|
|
4607
|
+
try {
|
|
4608
|
+
parsed = JSON.parse(arrayMatch[0]);
|
|
4609
|
+
} catch {
|
|
4610
|
+
return [];
|
|
4611
|
+
}
|
|
4612
|
+
if (!Array.isArray(parsed)) return [];
|
|
4613
|
+
const ledgerIds = new Set(ledger.map((claim) => claim.id));
|
|
4614
|
+
const out = [];
|
|
4615
|
+
for (const item of parsed) {
|
|
4616
|
+
if (!item || typeof item !== "object") continue;
|
|
4617
|
+
const record = item;
|
|
4618
|
+
if (typeof record.claim !== "string" || !record.claim.trim()) continue;
|
|
4619
|
+
const contradictsId = extractBracketId(record.contradicts);
|
|
4620
|
+
out.push({
|
|
4621
|
+
text: record.claim.trim(),
|
|
4622
|
+
contradictsExistingId: contradictsId && ledgerIds.has(contradictsId) ? contradictsId : void 0
|
|
4623
|
+
});
|
|
4624
|
+
}
|
|
4625
|
+
return out;
|
|
4626
|
+
}
|
|
4627
|
+
function extractBracketId(value) {
|
|
4628
|
+
if (typeof value !== "string") return void 0;
|
|
4629
|
+
const trimmed = value.trim();
|
|
4630
|
+
if (!trimmed || trimmed.toLowerCase() === "null") return void 0;
|
|
4631
|
+
const bracket = trimmed.match(/\[([^\]]+)\]/);
|
|
4632
|
+
const id = (bracket?.[1] ?? trimmed).trim();
|
|
4633
|
+
return id.startsWith("c_") ? id : void 0;
|
|
4634
|
+
}
|
|
4635
|
+
function buildSteerText(gaps, deepQuestions, invalidationTargets, minIndependentSources) {
|
|
4636
|
+
const lines = [];
|
|
4637
|
+
lines.push(
|
|
4638
|
+
"Do NOT just add more sources. Go DEEPER and VALIDATE. Address the following before adding breadth:"
|
|
4639
|
+
);
|
|
4640
|
+
if (deepQuestions.length > 0) {
|
|
4641
|
+
lines.push("", "Deep sub-questions to answer this round:");
|
|
4642
|
+
for (const question of deepQuestions) {
|
|
4643
|
+
lines.push(`- (${question.kind}) ${question.text}`);
|
|
4644
|
+
}
|
|
4645
|
+
}
|
|
4646
|
+
if (invalidationTargets.length > 0) {
|
|
4647
|
+
lines.push(
|
|
4648
|
+
"",
|
|
4649
|
+
`Claims needing corroboration or refutation (each must reach >= ${minIndependentSources} INDEPENDENT sources, or be shown contested):`
|
|
4650
|
+
);
|
|
4651
|
+
for (const claim of invalidationTargets) {
|
|
4652
|
+
const reason = claim.contradicts.size > 0 ? "CONTRADICTED by another source \u2014 find evidence that resolves it" : `only ${claim.supportingHosts.size} independent source \u2014 find a SECOND, independent corroborating source`;
|
|
4653
|
+
lines.push(`- "${truncate(claim.text)}" \u2014 ${reason}`);
|
|
4654
|
+
}
|
|
4655
|
+
}
|
|
4656
|
+
if (gaps.length > 0) {
|
|
4657
|
+
lines.push("", "Readiness gaps still open:");
|
|
4658
|
+
for (const gap of gaps) {
|
|
4659
|
+
lines.push(`- (${gap.blocking ? "blocking" : "soft"}) ${gap.description} [${gap.id}]`);
|
|
4660
|
+
}
|
|
4661
|
+
}
|
|
4662
|
+
return lines.join("\n");
|
|
4663
|
+
}
|
|
1915
4664
|
export {
|
|
1916
4665
|
AgentMemoryHitSchema,
|
|
1917
4666
|
AgentMemoryKindSchema,
|
|
1918
4667
|
AgentMemoryScopeSchema,
|
|
1919
4668
|
AgentMemoryWriteInputSchema,
|
|
1920
4669
|
FileSystemKbStore,
|
|
4670
|
+
INDUSTRY_MEMORY_BENCHMARKS,
|
|
4671
|
+
INDUSTRY_RAG_BENCHMARKS,
|
|
1921
4672
|
IRS_DIMENSION_HINTS,
|
|
1922
4673
|
KnowledgeBaseCandidateSchema,
|
|
1923
4674
|
KnowledgeEventSchema,
|
|
@@ -1931,7 +4682,6 @@ export {
|
|
|
1931
4682
|
MemoryKbStore,
|
|
1932
4683
|
POLITE_USER_AGENT,
|
|
1933
4684
|
READINESS_SPEC_DEFAULTS,
|
|
1934
|
-
RESEARCH_SUPERVISOR_SYSTEM_PROMPT,
|
|
1935
4685
|
RouterError,
|
|
1936
4686
|
SCAFFOLD_PAGE_BASENAMES,
|
|
1937
4687
|
SourceAnchorSchema,
|
|
@@ -1942,9 +4692,19 @@ export {
|
|
|
1942
4692
|
addSourceText,
|
|
1943
4693
|
applyKnowledgeWriteBlocks,
|
|
1944
4694
|
applyKnowledgeWriteBlocksFile,
|
|
4695
|
+
applyRetrievalHoldout,
|
|
4696
|
+
applySessionStickyRetrievalHoldout,
|
|
1945
4697
|
buildEvalKnowledgeBundle,
|
|
4698
|
+
buildFirstPartyMemoryLifecycleBenchmarkCases,
|
|
4699
|
+
buildIndustryMemoryBenchmarkSmokeCases,
|
|
4700
|
+
buildIndustryRagBenchmarkSmokeCases,
|
|
4701
|
+
buildKnowledgeBenchmarkScenarios,
|
|
1946
4702
|
buildKnowledgeGraph,
|
|
1947
4703
|
buildKnowledgeIndex,
|
|
4704
|
+
buildRetrievalBenchmarkCasesFromQrels,
|
|
4705
|
+
buildRetrievalEvalDispatch,
|
|
4706
|
+
buildRetrievalParameterCandidates,
|
|
4707
|
+
calibrateRagAnswerJudge,
|
|
1948
4708
|
canonicalizeUrl,
|
|
1949
4709
|
chunkMarkdown,
|
|
1950
4710
|
citedClaimKey,
|
|
@@ -1953,14 +4713,20 @@ export {
|
|
|
1953
4713
|
createAdaptiveResearchDriver,
|
|
1954
4714
|
createClaimDecorator,
|
|
1955
4715
|
createClaimGroundingVerifier,
|
|
4716
|
+
createCollectionResearchDriver,
|
|
1956
4717
|
createCornellLiiSource,
|
|
1957
4718
|
createD1FreshnessStoreStub,
|
|
1958
4719
|
createFileSystemFreshnessStore,
|
|
4720
|
+
createInMemoryBenchmarkAdapter,
|
|
1959
4721
|
createIrsPublicationsSource,
|
|
1960
4722
|
createKnowledgeControlLoopAdapter,
|
|
1961
4723
|
createKnowledgeEvent,
|
|
1962
4724
|
createLocalDiscoveryDispatcher,
|
|
4725
|
+
createMemoryAdapterBenchmarkResponder,
|
|
1963
4726
|
createNeo4jAgentMemoryAdapter,
|
|
4727
|
+
createNoopMemoryBenchmarkAdapter,
|
|
4728
|
+
createRagAnswerQualityHook,
|
|
4729
|
+
createResearchDrivingDriver,
|
|
1964
4730
|
createStateSosSource,
|
|
1965
4731
|
createTangleRouterClient,
|
|
1966
4732
|
createVerifyingResearchDriver,
|
|
@@ -1968,39 +4734,79 @@ export {
|
|
|
1968
4734
|
defaultGetMemoryContext,
|
|
1969
4735
|
defineReadinessSpec,
|
|
1970
4736
|
detectChanges,
|
|
4737
|
+
deterministicRng,
|
|
4738
|
+
diagnoseRagAnswerFailure,
|
|
4739
|
+
emitRetrievalHoldoutBypass,
|
|
4740
|
+
evaluateKnowledgeBaseReadiness,
|
|
1971
4741
|
explainKnowledgeTarget,
|
|
1972
4742
|
extractLinks,
|
|
1973
4743
|
extractWikilinks,
|
|
1974
4744
|
firstMatch,
|
|
1975
4745
|
formatFrontmatter,
|
|
4746
|
+
gradeCompanyAgainstText,
|
|
4747
|
+
gradeFactAgainstText,
|
|
1976
4748
|
groundClaimInText,
|
|
4749
|
+
hashKnowledgeBase,
|
|
1977
4750
|
htmlToText,
|
|
4751
|
+
improveKnowledgeBase,
|
|
1978
4752
|
initKnowledgeBase,
|
|
1979
4753
|
innerHtmlById,
|
|
1980
4754
|
inspectKnowledgeIndex,
|
|
4755
|
+
investmentThesisSet,
|
|
4756
|
+
isKnowledgeMemoryBenchmarkCase,
|
|
1981
4757
|
isSafeKnowledgePath,
|
|
1982
4758
|
isScaffoldPath,
|
|
1983
|
-
|
|
4759
|
+
kbIndexToText,
|
|
4760
|
+
knowledgeBenchmarkJudge,
|
|
4761
|
+
knowledgeImprovementRunDir,
|
|
4762
|
+
knowledgeImprovementRunId,
|
|
1984
4763
|
knowledgeReleaseReport,
|
|
1985
4764
|
layoutFor,
|
|
4765
|
+
lensDistribution,
|
|
1986
4766
|
lintKnowledgeIndex,
|
|
4767
|
+
loadKnowledgeImprovementState,
|
|
1987
4768
|
loadKnowledgePages,
|
|
1988
4769
|
loadSourceRegistry,
|
|
1989
4770
|
looksLikeBlockPage,
|
|
4771
|
+
materialFactsSurfaced,
|
|
4772
|
+
materialFactsSurfacedInText,
|
|
1990
4773
|
mediaTypeFor,
|
|
1991
4774
|
memoryHitToSourceRecord,
|
|
1992
4775
|
memoryWriteResultToSourceRecord,
|
|
4776
|
+
normalizeExternalRagScores,
|
|
1993
4777
|
normalizeLinkTarget,
|
|
1994
4778
|
parseFrontmatter,
|
|
4779
|
+
parseKnowledgeBenchmarkJsonl,
|
|
4780
|
+
parseKnowledgeBenchmarkQrels,
|
|
1995
4781
|
parseKnowledgeWriteBlocks,
|
|
1996
4782
|
politeFetch,
|
|
1997
4783
|
proposeFromFinding,
|
|
1998
4784
|
proposeFromFindings,
|
|
4785
|
+
ragAnswerQualityJudge,
|
|
1999
4786
|
reciprocalRankFusion,
|
|
4787
|
+
renderKnowledgeBenchmarkReportMarkdown,
|
|
2000
4788
|
renderMemoryContext,
|
|
4789
|
+
resetRetrievalHoldoutRegistry,
|
|
4790
|
+
respondToIndustryMemoryBenchmarkSmokeCase,
|
|
4791
|
+
respondToIndustryRagBenchmarkSmokeCase,
|
|
4792
|
+
retrievalConfigFromSurface,
|
|
4793
|
+
retrievalConfigSurface,
|
|
4794
|
+
retrievalHoldoutConfigHash,
|
|
4795
|
+
retrievalParameterSweepProposer,
|
|
4796
|
+
retrievalRecallJudge,
|
|
4797
|
+
runInvestmentThesisTask,
|
|
4798
|
+
runKnowledgeBenchmarkSuite,
|
|
2001
4799
|
runKnowledgeResearchLoop,
|
|
2002
|
-
|
|
4800
|
+
runMemoryAdapterBenchmark,
|
|
4801
|
+
runRagKnowledgeImprovementLoop,
|
|
4802
|
+
runRetrievalImprovementLoop,
|
|
2003
4803
|
runTwoAgentResearchLoop,
|
|
4804
|
+
runVerifiedResearchLoop,
|
|
4805
|
+
scoreKnowledgeBaseIndex,
|
|
4806
|
+
scoreKnowledgeBenchmarkArtifact,
|
|
4807
|
+
scoreMemoryBenchmarkArtifact,
|
|
4808
|
+
scoreRagAnswerArtifact,
|
|
4809
|
+
scoreRetrievalArtifact,
|
|
2004
4810
|
searchKnowledge,
|
|
2005
4811
|
sha256,
|
|
2006
4812
|
slugify,
|
|
@@ -2008,8 +4814,16 @@ export {
|
|
|
2008
4814
|
sourceRegistryPath,
|
|
2009
4815
|
stableId,
|
|
2010
4816
|
stripFrontmatter,
|
|
4817
|
+
summarizeKnowledgeBenchmarkCampaign,
|
|
2011
4818
|
textSourceAdapter,
|
|
4819
|
+
thesisReadinessSpecs,
|
|
4820
|
+
toDeepEvalTestCases,
|
|
4821
|
+
toOffPolicyTrajectory,
|
|
4822
|
+
toRagCheckerRecords,
|
|
4823
|
+
toRagasEvaluationRows,
|
|
4824
|
+
toTruLensRecords,
|
|
2012
4825
|
tokenizeQuery,
|
|
4826
|
+
totalMaterialFacts,
|
|
2013
4827
|
triageSource,
|
|
2014
4828
|
validateKnowledgeIndex,
|
|
2015
4829
|
withCitedClaim,
|