@tangle-network/agent-knowledge 19.1.5 → 20.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -2
- package/README.md +17 -91
- package/dist/benchmarks/index.d.ts +2 -2
- package/dist/benchmarks/index.js +2 -2
- package/dist/benchmarks-jir3ZP-M.js +162 -0
- package/dist/benchmarks-jir3ZP-M.js.map +1 -0
- package/dist/cli.js +2 -3
- package/dist/cli.js.map +1 -1
- package/dist/index-BhvSH4E4.d.ts +512 -0
- package/dist/index-BhvSH4E4.d.ts.map +1 -0
- package/dist/index-OAxtm6is.d.ts +9 -0
- package/dist/index-OAxtm6is.d.ts.map +1 -0
- package/dist/index.d.ts +197 -1332
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +561 -3653
- package/dist/index.js.map +1 -1
- package/dist/{inspect-C6c4zuKp.js → inspect-CMigdsD5.js} +282 -24
- package/dist/inspect-CMigdsD5.js.map +1 -0
- package/dist/memory/index.d.ts +3 -3
- package/dist/memory/index.js +3 -3
- package/dist/memory-BH6oT1Qe.js +2728 -0
- package/dist/memory-BH6oT1Qe.js.map +1 -0
- package/dist/source-record-CdWvzvO6.js +39 -0
- package/dist/source-record-CdWvzvO6.js.map +1 -0
- package/dist/types-BWV_Fvp7.d.ts +224 -0
- package/dist/types-BWV_Fvp7.d.ts.map +1 -0
- package/dist/{types-m2QB86fF.d.ts → types-BekdNwGD.d.ts} +4 -19
- package/dist/types-BekdNwGD.d.ts.map +1 -0
- package/dist/viz/index.d.ts +1 -1
- package/docs/architecture.md +2 -30
- package/docs/eval/investment-material-facts.md +2 -0
- package/docs/results/research-driving.md +8 -8
- package/docs/verified-research-ab.md +12 -10
- package/package.json +6 -12
- package/skills/build-with-agent-knowledge/SKILL.md +2 -15
- package/dist/benchmarks-BjXxjAke.js +0 -2824
- package/dist/benchmarks-BjXxjAke.js.map +0 -1
- package/dist/index-Bh9wLd0_.d.ts +0 -1097
- package/dist/index-Bh9wLd0_.d.ts.map +0 -1
- package/dist/index-DZeFm-BP.d.ts +0 -680
- package/dist/index-DZeFm-BP.d.ts.map +0 -1
- package/dist/inspect-C6c4zuKp.js.map +0 -1
- package/dist/memory-CUk1mgPY.js +0 -5778
- package/dist/memory-CUk1mgPY.js.map +0 -1
- package/dist/search-BBr5eUyT.js +0 -275
- package/dist/search-BBr5eUyT.js.map +0 -1
- package/dist/sources/index.d.ts +0 -376
- package/dist/sources/index.d.ts.map +0 -1
- package/dist/sources/index.js +0 -615
- package/dist/sources/index.js.map +0 -1
- package/dist/types-m2QB86fF.d.ts.map +0 -1
- package/docs/eval/rag-eval-roadmap.md +0 -123
package/dist/index.js
CHANGED
|
@@ -1,689 +1,18 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as assertGradeableEvidence, $n as removeDurable, $t as KnowledgeIndexSchema, A as createKnowledgeVisibilitySnapshot, An as isKnowledgeMutationHeld, At as auditKnowledgeCitations, B as snapshotImmutableVisibility, Bn as knowledgeFileTransactionPlanHash, Bt as isScaffoldPath, C as createRunScopedStores, Cn as mergeTrackedClaims, Ct as KnowledgeCitationAuditError, D as KNOWLEDGE_RECEIPT_DIGEST_ALGORITHM, Dn as parseFrontmatter, Dt as assertKnowledgeCitationsResolved, E as originatedPages, En as formatFrontmatter, Et as assertKnowledgeCitationAudit, F as knowledgePageDigest, Fn as withKnowledgeRead, Ft as resolveRunScopedCitation, G as visibilityIndexOf, Gn as canonicalPathsEqual, Gt as WIKILINK_REGEX, H as validateOrigin, Hn as rollbackKnowledgeFileTransaction, Ht as layoutFor, I as knowledgeVisibilityArtifactRef, In as applyKnowledgeFileTransaction, It as resolveRunScopedCitations, J as validateKnowledgeIndex, Jn as isMissingFile, Jt as DeepQuestionSchema, K as detectNearDuplicatePages, Kn as canonicalRelativeWithinRoot, Kt as extractWikilinks, L as nonEmpty$1, Ln as assertKnowledgeMutationPath, Lt as SCAFFOLD_PAGE_BASENAMES, M as digest, Mn as recoverPendingKnowledgeMutation, Mt as parseKnowledgeCitationReference, N as encodeKnowledgeVisibilitySnapshot, Nn as runInKnowledgeMutationScope, Nt as resolveKnowledgeCitation, O as KNOWLEDGE_USE_RECEIPT_SCHEMA_VERSION, On as acquireDurableFileLock, Ot as assertRunScopedCitationsResolved, P as immutablePageDigest, Pn as withKnowledgeMutation, Pt as resolveKnowledgeCitations, Q as UncheckableClaimError, Qn as relativeWithinRoot, Qt as KnowledgeGraphNodeSchema, R as normalizeVisibilityArtifact, Rn as commitKnowledgeFileMutations, Rt as initKnowledgeBase, S as createFileRunLineageAuthority, Sn as mergeClaimLedgers, St as knowledgePageRelations, T as loadKnowledgeToolChain, Tn as researchSourceVersionKey, Tt as assertCurrentRunCitationsResolved, U as verifyKnowledgeVisibilitySnapshot, Un as DEFAULT_PAGES_DIRECTORY, Ut as loadKnowledgePages, V as validateKnowledgePage, Vn as prepareKnowledgeFileTransaction, Vt as knowledgePageFromMarkdown, W as visibilityIdentity, Wn as normalizePagesDirectory, Wt as writeJson, X as CHECKABLE_RUNG_THRESHOLD, Xn as readRegularFileNoFollow, Xt as KnowledgeEventSchema, Y as lintKnowledgeIndex, Yn as listRegularFilesWithinRoot, Yt as KnowledgeBaseCandidateSchema, Z as DEADLINE_EXIT_CODE, Zn as readRegularFileWithinRoot, Zt as KnowledgeGraphEdgeSchema, _ as isSafeKnowledgePath, _n as claimSourceHost, _t as KB_INDEX_PATH, a as searchKnowledge, an as ResearchClaimRecordSchema, ar as writeFileDurableWithinRoot, at as buildKnowledgeIndex, b as assertKnowledgeWriteIntake, bn as linkClaimContradictions, bt as assertClaimLedgerId, c as scoreBm25, cn as SourceRecordSchema, cr as mediaTypeFor, ct as addSourceText, d as DEFAULT_FIELD_BOOSTS, dn as assertDeepQuestionIntegrity, dt as sourceRegistryPath, en as KnowledgePageInvalidationSchema, er as renameDurable, et as claimCheckKey, f as assembleLexicalIndex, fn as assertResearchClaimEvidenceIntegrity, ft as writeSourceRegistry, g as applyKnowledgeWriteBlocksFile, gn as claimId, gt as KB_EVENTS_PATH, h as applyKnowledgeWriteBlocks, hn as claimEvidenceId, ht as KB_CLAIM_LEDGER_DIR, i as reciprocalRankFusion, in as ResearchClaimLedgerSchema, ir as writeFileDurable, it as verifyGradeableEvidence, j as decodeKnowledgeVisibilitySnapshot, jn as measureKnowledgeMutation, jt as formatKnowledgeCitationReference, k as assertSchemaVersion, kn as inspectPendingKnowledgeMutation, kt as auditCurrentRunCitations, l as tokenizeQuery, ln as KNOWLEDGE_EVENT_TYPES, lr as textSourceAdapter, lt as loadSourceRegistry, m as stringMetadata, mn as assertTrackedClaimIntegrity, mt as FileSystemKbStore, n as inspectKnowledgeIndex, nn as KnowledgeRelationSchema, nr as withSafeDescendant, nt as gradeFor, o as searchKnowledgePages, on as ResearchSourceVersionSchema, or as writeJsonDurable, ot as writeKnowledgeIndex, p as lexicalPageTerms, pn as assertResearchClaimLedgerIntegrity, pt as ClaimLedgerMigrationRequiredError, q as normalizePageText, qn as isKernelAnchoredPath, qt as normalizeLinkTarget, r as KNOWLEDGE_SEARCH_RETRIEVER_ID, rn as ResearchClaimEvidenceSchema, rr as withSafeDirectory, rt as verdictFor, s as buildKnowledgeLexicalIndex, sn as SourceAnchorSchema, sr as writeJsonDurableWithinRoot, st as addSourcePath, t as explainKnowledgeTarget, tn as KnowledgePageSchema, tr as syncDirectory, tt as gradeClaims, u as tokenizeText, un as ClaimLedgerGoalConflictError, ut as snapshotSourceTextInput, v as parseKnowledgeWriteBlocks, vn as deepQuestionId, vt as KB_STORE_DIR, w as isKnowledgeToolChain, wn as normalizeClaimText, wt as KnowledgeCitationResolutionError, x as RUN_LINEAGE_BASENAME, xn as materializeRegisteredClaimEvidence, xt as buildKnowledgeGraph, y as KnowledgeDuplicateIntakeError, yn as emptyClaimLedger, yt as MemoryKbStore, z as normalizeVisibilityRef, zn as finishKnowledgeFileTransaction, zt as isKnowledgePagePath } from "./inspect-CMigdsD5.js";
|
|
2
2
|
import { i as textSourceId, n as slugify, r as stableId, t as sha256 } from "./ids-Bevz_pXV.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { a as
|
|
6
|
-
import {
|
|
7
|
-
import { createHash } from "node:crypto";
|
|
8
|
-
import { TCloudClient, TCloudError } from "@tangle-network/tcloud";
|
|
9
|
-
import { cp, lstat, mkdir, mkdtemp, readFile, rm, stat } from "node:fs/promises";
|
|
10
|
-
import { dirname, join } from "node:path";
|
|
11
|
-
import { agentImprovementActivationResultSchema, agentImprovementActivationSchema, canonicalCandidateDigest, canonicalCandidateJson, omitTopLevelDigest, sha256Bytes, sha256DigestSchema } from "@tangle-network/agent-interface";
|
|
3
|
+
import { n as memoryWriteResultToSourceRecord, t as memoryHitToSourceRecord } from "./source-record-CdWvzvO6.js";
|
|
4
|
+
import { n as createNoopMemoryBenchmarkAdapter, t as createInMemoryBenchmarkAdapter } from "./benchmarks-jir3ZP-M.js";
|
|
5
|
+
import { A as resetRetrievalHoldoutRegistry, C as forkAgentMemoryBranchSnapshot, D as applySessionStickyRetrievalHoldout, E as applyRetrievalHoldout, M as toOffPolicyTrajectory, O as deterministicRng, S as createAgentMemoryBranch, T as renderMemoryContext, _ as resolveMemoryCleanupTimeoutMs, a as HindsightOperationUnknownError, b as createGraphitiMemoryAdapter, c as AgentMemoryHitSchema, d as AgentMemoryWriteInputSchema, f as AgentMemoryLifecycleTimeoutError, g as memoryRecoveryDelayMs, h as createMemoryExecutionPool, i as mem0MemoryAdapterIdentity, j as retrievalHoldoutConfigHash, k as emitRetrievalHoldoutBypass, l as AgentMemoryKindSchema, m as DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, n as createNeo4jAgentMemoryAdapter, o as createHindsightMemoryAdapter, p as AgentMemoryLifecycleUnsafeError, r as createMem0MemoryAdapter, s as hindsightMemoryBankId, t as createPlayMemoryTools, u as AgentMemoryScopeSchema, v as runBoundedMemoryLifecycle, w as defaultGetMemoryContext, x as graphitiMemoryAdapterIdentity, y as sleepForMemoryRecovery } from "./memory-BH6oT1Qe.js";
|
|
6
|
+
import { agentImprovementActivationResultSchema, agentImprovementActivationSchema, canonicalCandidateDigest, omitTopLevelDigest, sha256Bytes, sha256DigestSchema } from "@tangle-network/agent-interface";
|
|
12
7
|
import { acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, canonicalJson, contentHash, evaluateReleaseConfidence, objectiveEval, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps, validateRunRecord } from "@tangle-network/agent-eval";
|
|
13
8
|
import { z } from "zod";
|
|
14
|
-
import {
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
import { cp, lstat, mkdir, mkdtemp, rm, stat } from "node:fs/promises";
|
|
11
|
+
import { dirname, join } from "node:path";
|
|
12
|
+
import { compareOptimizationMethods, surfaceHash } from "@tangle-network/agent-eval/campaign";
|
|
15
13
|
import { tmpdir } from "node:os";
|
|
16
14
|
import { isDeepStrictEqual } from "node:util";
|
|
17
15
|
import { pathToFileURL } from "node:url";
|
|
18
|
-
//#region src/web-research-worker.ts
|
|
19
|
-
/**
|
|
20
|
-
* Real web-research worker + verifying driver for `runVerifiedResearchLoop`.
|
|
21
|
-
*
|
|
22
|
-
* This is the GENERAL, any-topic implementation behind the two-agent research
|
|
23
|
-
* loop's live arm. Given the open knowledge gaps the readiness gate surfaces,
|
|
24
|
-
* the worker:
|
|
25
|
-
*
|
|
26
|
-
* 1. asks an LLM (glm-5.2 by default) to turn each gap into focused web
|
|
27
|
-
* search queries,
|
|
28
|
-
* 2. runs a REAL web search over the Tangle router (`POST /v1/search` — the
|
|
29
|
-
* same endpoint `tcloud mcp`'s `web_search` tool forwards to), so there is
|
|
30
|
-
* no hardcoded corpus,
|
|
31
|
-
* 3. fetches the top results with the repo's polite, cached `politeFetch` and
|
|
32
|
-
* reduces each page to text with `htmlToText`,
|
|
33
|
-
* 4. proposes the readable, verifiable pages as `ResearchSourceProposal`s plus
|
|
34
|
-
* a `buildPages` that writes citing `knowledge/*.md` pages from the sources
|
|
35
|
-
* the driver accepts.
|
|
36
|
-
*
|
|
37
|
-
* The verifying DRIVER is the differentiated role from the two-agent loop: a
|
|
38
|
-
* second LLM pass that judges each fetched source's on-topic relevance to the
|
|
39
|
-
* goal + open gaps and rejects off-topic / spam / already-covered material. The
|
|
40
|
-
* worker ADDS; the driver GATES. Together they build a cleaner knowledge base
|
|
41
|
-
* than a single agent at the same compute budget.
|
|
42
|
-
*
|
|
43
|
-
* Model and search requests use the published TCloud SDK. Knowledge owns the
|
|
44
|
-
* research policy, not HTTP, bearer headers, retries or provider rate cards.
|
|
45
|
-
* Point it at any router with `baseUrl`; supply `apiKey` or `TANGLE_API_KEY`.
|
|
46
|
-
*/
|
|
47
|
-
/** Default router model. Plain id (no namespace) — see CLAUDE/creds notes. */
|
|
48
|
-
const DEFAULT_MODEL = "glm-5.2";
|
|
49
|
-
/** Default router base. */
|
|
50
|
-
const DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
|
|
51
|
-
/**
|
|
52
|
-
* glm-5.2 spends its first tokens on hidden `reasoning_content`; below ~1200
|
|
53
|
-
* output tokens it returns EMPTY visible content. Floor every call so a
|
|
54
|
-
* reasoning model never silently yields nothing. (Verified creds note.)
|
|
55
|
-
*/
|
|
56
|
-
const MIN_MAX_TOKENS = 1200;
|
|
57
|
-
/** Compatibility error facade. HTTP execution and classification belong to TCloud. */
|
|
58
|
-
var RouterError = class extends Error {
|
|
59
|
-
status;
|
|
60
|
-
constructor(status, message) {
|
|
61
|
-
super(`router ${status}: ${message}`);
|
|
62
|
-
this.status = status;
|
|
63
|
-
this.name = "RouterError";
|
|
64
|
-
}
|
|
65
|
-
};
|
|
66
|
-
/** Adapt Knowledge's public contract to the published SDK, without a second transport. */
|
|
67
|
-
function createTangleRouterClient(options = {}) {
|
|
68
|
-
const apiKey = options.apiKey ?? process.env.TANGLE_API_KEY;
|
|
69
|
-
if (!apiKey) throw new RouterError(401, "no TANGLE_API_KEY (pass apiKey or set the env var)");
|
|
70
|
-
const client = new TCloudClient({
|
|
71
|
-
baseURL: (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""),
|
|
72
|
-
apiKey,
|
|
73
|
-
model: options.model ?? DEFAULT_MODEL,
|
|
74
|
-
timeout: 0,
|
|
75
|
-
retry: {
|
|
76
|
-
maxRetries: Math.max(0, options.maxRetries ?? 4),
|
|
77
|
-
initialBackoffMs: Math.max(1, options.retryBaseMs ?? 1500),
|
|
78
|
-
retryableStatuses: [
|
|
79
|
-
429,
|
|
80
|
-
502,
|
|
81
|
-
503,
|
|
82
|
-
504
|
|
83
|
-
]
|
|
84
|
-
}
|
|
85
|
-
});
|
|
86
|
-
const acc = {
|
|
87
|
-
chatCalls: 0,
|
|
88
|
-
searchCalls: 0,
|
|
89
|
-
promptTokens: 0,
|
|
90
|
-
completionTokens: 0,
|
|
91
|
-
usd: 0,
|
|
92
|
-
wallMs: 0
|
|
93
|
-
};
|
|
94
|
-
const recordCost = (cost) => {
|
|
95
|
-
acc.usd += typeof cost === "number" && Number.isFinite(cost) && cost >= 0 ? cost : NaN;
|
|
96
|
-
};
|
|
97
|
-
const translate = (error) => {
|
|
98
|
-
options.signal?.throwIfAborted();
|
|
99
|
-
if (error instanceof TCloudError) throw new RouterError(error.status, error.message);
|
|
100
|
-
throw error;
|
|
101
|
-
};
|
|
102
|
-
return {
|
|
103
|
-
async search(query, opts) {
|
|
104
|
-
options.signal?.throwIfAborted();
|
|
105
|
-
const started = Date.now();
|
|
106
|
-
try {
|
|
107
|
-
const response = await client.search({
|
|
108
|
-
query,
|
|
109
|
-
...options.searchProvider ? { provider: options.searchProvider } : {},
|
|
110
|
-
...opts?.maxResults != null ? { maxResults: opts.maxResults } : {},
|
|
111
|
-
signal: options.signal
|
|
112
|
-
});
|
|
113
|
-
recordCost(response.usage?.billed_cost);
|
|
114
|
-
return (response.data ?? []).filter((hit) => typeof hit?.url === "string" && hit.url.length > 0).map((hit) => ({
|
|
115
|
-
title: hit.title ?? hit.url,
|
|
116
|
-
url: hit.url,
|
|
117
|
-
snippet: hit.snippet
|
|
118
|
-
}));
|
|
119
|
-
} catch (error) {
|
|
120
|
-
return translate(error);
|
|
121
|
-
} finally {
|
|
122
|
-
acc.searchCalls += 1;
|
|
123
|
-
acc.wallMs += Date.now() - started;
|
|
124
|
-
}
|
|
125
|
-
},
|
|
126
|
-
async chat(messages, maxTokens) {
|
|
127
|
-
options.signal?.throwIfAborted();
|
|
128
|
-
const started = Date.now();
|
|
129
|
-
try {
|
|
130
|
-
const response = await client.chat({
|
|
131
|
-
messages,
|
|
132
|
-
maxTokens: Math.max(MIN_MAX_TOKENS, maxTokens ?? MIN_MAX_TOKENS),
|
|
133
|
-
temperature: .2,
|
|
134
|
-
signal: options.signal
|
|
135
|
-
});
|
|
136
|
-
acc.chatCalls += 1;
|
|
137
|
-
acc.promptTokens += response.usage?.prompt_tokens ?? 0;
|
|
138
|
-
acc.completionTokens += response.usage?.completion_tokens ?? 0;
|
|
139
|
-
recordCost(response.tangle?.costSource === "receipt" ? response.tangle.costUsd : void 0);
|
|
140
|
-
return response.choices?.[0]?.message?.content ?? "";
|
|
141
|
-
} catch (error) {
|
|
142
|
-
return translate(error);
|
|
143
|
-
} finally {
|
|
144
|
-
acc.wallMs += Date.now() - started;
|
|
145
|
-
}
|
|
146
|
-
},
|
|
147
|
-
usage() {
|
|
148
|
-
return { ...acc };
|
|
149
|
-
}
|
|
150
|
-
};
|
|
151
|
-
}
|
|
152
|
-
/** Bind run cancellation to the default router without changing an injected client. */
|
|
153
|
-
function resolveRouter(opts, signal) {
|
|
154
|
-
if (opts.router) return opts.router;
|
|
155
|
-
const requestSignal = combineRouterSignals(signal, opts.router_options?.signal);
|
|
156
|
-
return createTangleRouterClient({
|
|
157
|
-
...opts.router_options,
|
|
158
|
-
signal: requestSignal
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
function combineRouterSignals(runSignal, configuredSignal) {
|
|
162
|
-
return runSignal && configuredSignal ? AbortSignal.any([runSignal, configuredSignal]) : runSignal ?? configuredSignal;
|
|
163
|
-
}
|
|
164
|
-
function throwIfRouterAborted(runSignal, opts) {
|
|
165
|
-
runSignal?.throwIfAborted();
|
|
166
|
-
if (!opts.router) opts.router_options?.signal?.throwIfAborted();
|
|
167
|
-
}
|
|
168
|
-
/**
|
|
169
|
-
* The real web-research worker. Conforms to the loop's `ResearchWorker`
|
|
170
|
-
* contract: given the open gaps, it returns the sources it found plus a
|
|
171
|
-
* `buildPages` that emits citing pages from the sources the driver accepted.
|
|
172
|
-
*/
|
|
173
|
-
function createWebResearchWorker(options = {}) {
|
|
174
|
-
const queriesPerGap = Math.max(1, options.queriesPerGap ?? 2);
|
|
175
|
-
const resultsPerQuery = Math.max(1, options.resultsPerQuery ?? 3);
|
|
176
|
-
const maxSourcesPerRound = Math.max(1, options.maxSourcesPerRound ?? 6);
|
|
177
|
-
const minTextChars = Math.max(1, options.minTextChars ?? 200);
|
|
178
|
-
const maxTextChars = Math.max(minTextChars, options.maxTextChars ?? 4e3);
|
|
179
|
-
return async (ctx) => {
|
|
180
|
-
const assertActive = () => throwIfRouterAborted(ctx.signal, options);
|
|
181
|
-
assertActive();
|
|
182
|
-
const router = resolveRouter(options, ctx.signal);
|
|
183
|
-
const fetchSignal = options.router ? ctx.signal : combineRouterSignals(ctx.signal, options.router_options?.signal);
|
|
184
|
-
const targetGaps = ctx.gaps.filter((gap) => gap.blocking);
|
|
185
|
-
const gaps = targetGaps.length > 0 ? targetGaps : ctx.gaps;
|
|
186
|
-
if (gaps.length === 0) return {
|
|
187
|
-
sources: [],
|
|
188
|
-
notes: "no open gaps to research"
|
|
189
|
-
};
|
|
190
|
-
const queries = await formSearchQueries(router, ctx, gaps, queriesPerGap, assertActive);
|
|
191
|
-
assertActive();
|
|
192
|
-
const proposals = [];
|
|
193
|
-
const seenUris = /* @__PURE__ */ new Set();
|
|
194
|
-
for (const query of queries) {
|
|
195
|
-
if (proposals.length >= maxSourcesPerRound) break;
|
|
196
|
-
assertActive();
|
|
197
|
-
let hits;
|
|
198
|
-
try {
|
|
199
|
-
hits = await router.search(query, { maxResults: resultsPerQuery });
|
|
200
|
-
} catch (error) {
|
|
201
|
-
assertActive();
|
|
202
|
-
if (error?.name === "AbortError") throw error;
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
assertActive();
|
|
206
|
-
for (const hit of hits) {
|
|
207
|
-
assertActive();
|
|
208
|
-
if (proposals.length >= maxSourcesPerRound) break;
|
|
209
|
-
if (seenUris.has(hit.url)) continue;
|
|
210
|
-
const fetched = await politeFetch(hit.url, {
|
|
211
|
-
signal: fetchSignal,
|
|
212
|
-
cacheDir: options.cacheDir
|
|
213
|
-
});
|
|
214
|
-
assertActive();
|
|
215
|
-
if (!fetched.verifiable) continue;
|
|
216
|
-
const text = htmlToText(fetched.body).slice(0, maxTextChars);
|
|
217
|
-
if (text.length < minTextChars) continue;
|
|
218
|
-
seenUris.add(hit.url);
|
|
219
|
-
proposals.push({
|
|
220
|
-
uri: hit.url,
|
|
221
|
-
title: hit.title || hit.url,
|
|
222
|
-
text,
|
|
223
|
-
lastVerifiedAt: fetched.fetchedAt,
|
|
224
|
-
metadata: {
|
|
225
|
-
discoveredVia: query,
|
|
226
|
-
snippet: hit.snippet ?? "",
|
|
227
|
-
goal: ctx.goal,
|
|
228
|
-
sourceUpdatedAt: fetched.sourceUpdatedAt
|
|
229
|
-
}
|
|
230
|
-
});
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
assertActive();
|
|
234
|
-
return {
|
|
235
|
-
sources: proposals,
|
|
236
|
-
buildPages: buildCitingPages(proposals),
|
|
237
|
-
notes: `web-research worker: ${queries.length} queries → ${proposals.length} fetched sources`
|
|
238
|
-
};
|
|
239
|
-
};
|
|
240
|
-
}
|
|
241
|
-
/**
|
|
242
|
-
* Ask the LLM to turn the open gaps into focused web search queries. Falls back
|
|
243
|
-
* to the gap's own readiness query if the model returns nothing parseable, so a
|
|
244
|
-
* model hiccup degrades to a sane search rather than an empty round.
|
|
245
|
-
*/
|
|
246
|
-
async function formSearchQueries(router, ctx, gaps, queriesPerGap, assertActive) {
|
|
247
|
-
const gapLines = gaps.map((gap, i) => `${i + 1}. ${gap.description} (readiness query: "${gap.query}")`).join("\n");
|
|
248
|
-
const want = gaps.length * queriesPerGap;
|
|
249
|
-
const system = "You are a research librarian. Turn knowledge gaps into precise web search queries that will surface authoritative primary sources (papers, docs, standards, official pages). Return ONLY a JSON array of query strings, no prose.";
|
|
250
|
-
const user = [
|
|
251
|
-
`Research goal: ${ctx.goal}`,
|
|
252
|
-
ctx.steer ? `Steer from the coordinator:\n${ctx.steer}` : "",
|
|
253
|
-
`Open knowledge gaps:\n${gapLines}`,
|
|
254
|
-
`Return up to ${want} search query strings as a JSON array (e.g. ["query one","query two"]).`
|
|
255
|
-
].filter(Boolean).join("\n\n");
|
|
256
|
-
let raw = "";
|
|
257
|
-
try {
|
|
258
|
-
raw = await router.chat([{
|
|
259
|
-
role: "system",
|
|
260
|
-
content: system
|
|
261
|
-
}, {
|
|
262
|
-
role: "user",
|
|
263
|
-
content: user
|
|
264
|
-
}], MIN_MAX_TOKENS);
|
|
265
|
-
assertActive();
|
|
266
|
-
} catch (error) {
|
|
267
|
-
assertActive();
|
|
268
|
-
if (error?.name === "AbortError") throw error;
|
|
269
|
-
raw = "";
|
|
270
|
-
}
|
|
271
|
-
const fromLlm = parseQueryList(raw).slice(0, want);
|
|
272
|
-
if (fromLlm.length > 0) return dedupeStrings(fromLlm);
|
|
273
|
-
return dedupeStrings(gaps.map((gap) => gap.query || gap.description));
|
|
274
|
-
}
|
|
275
|
-
/** Parse a JSON array of query strings, tolerant of code fences / surrounding prose. */
|
|
276
|
-
function parseQueryList(raw) {
|
|
277
|
-
const text = raw.trim();
|
|
278
|
-
if (!text) return [];
|
|
279
|
-
const candidates = [];
|
|
280
|
-
const arrayMatch = text.match(/\[[\s\S]*\]/);
|
|
281
|
-
if (arrayMatch) try {
|
|
282
|
-
const arr = JSON.parse(arrayMatch[0]);
|
|
283
|
-
for (const item of arr) if (typeof item === "string" && item.trim()) candidates.push(item.trim());
|
|
284
|
-
} catch {}
|
|
285
|
-
if (candidates.length === 0) for (const line of text.split("\n")) {
|
|
286
|
-
const cleaned = line.replace(/^\s*(?:[-*]|\d+[.)])\s*/, "").replace(/^["']|["']$/g, "").trim();
|
|
287
|
-
if (cleaned && cleaned.length > 2 && !cleaned.startsWith("{")) candidates.push(cleaned);
|
|
288
|
-
}
|
|
289
|
-
return candidates;
|
|
290
|
-
}
|
|
291
|
-
function dedupeStrings(values) {
|
|
292
|
-
const seen = /* @__PURE__ */ new Set();
|
|
293
|
-
const out = [];
|
|
294
|
-
for (const value of values) {
|
|
295
|
-
const key = value.toLowerCase().trim();
|
|
296
|
-
if (key && !seen.has(key)) {
|
|
297
|
-
seen.add(key);
|
|
298
|
-
out.push(value.trim());
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
return out;
|
|
302
|
-
}
|
|
303
|
-
/**
|
|
304
|
-
* Build the curated `knowledge/*.md` pages from the sources the driver accepted.
|
|
305
|
-
* Each page cites the registered source by its assigned `record.id` (matched back
|
|
306
|
-
* to the proposal via `metadata.originalUri`, which `addSourceText` stashes).
|
|
307
|
-
*/
|
|
308
|
-
function buildCitingPages(proposals) {
|
|
309
|
-
return (acceptedSources) => {
|
|
310
|
-
if (acceptedSources.length === 0) return void 0;
|
|
311
|
-
return acceptedSources.map((record) => {
|
|
312
|
-
const proposal = proposals.find((p) => p.uri === record.metadata?.originalUri);
|
|
313
|
-
const uri = proposal?.uri ?? record.id;
|
|
314
|
-
const slug = uri.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 120) || record.id;
|
|
315
|
-
const title = proposal?.title ?? record.id;
|
|
316
|
-
const body = proposal?.text ?? "";
|
|
317
|
-
return [
|
|
318
|
-
`---FILE: knowledge/${slug}.md---`,
|
|
319
|
-
"---",
|
|
320
|
-
`title: ${escapeYaml(title)}`,
|
|
321
|
-
`sources: ["${record.id}"]`,
|
|
322
|
-
`source_url: ${uri}`,
|
|
323
|
-
"---",
|
|
324
|
-
`# ${title}`,
|
|
325
|
-
"",
|
|
326
|
-
body,
|
|
327
|
-
"",
|
|
328
|
-
`Source: ${uri}`,
|
|
329
|
-
"---END FILE---"
|
|
330
|
-
].join("\n");
|
|
331
|
-
}).join("\n");
|
|
332
|
-
};
|
|
333
|
-
}
|
|
334
|
-
function escapeYaml(value) {
|
|
335
|
-
return value.replace(/[\r\n]+/g, " ").replace(/"/g, "'").trim();
|
|
336
|
-
}
|
|
337
|
-
/**
|
|
338
|
-
* The verifying driver: a real LLM pass that judges each candidate source's
|
|
339
|
-
* on-topic relevance to the goal + open gaps and whether it duplicates material
|
|
340
|
-
* already accepted this round. This is the differentiated coordinator role — it
|
|
341
|
-
* GATES the worker's additions; it adds nothing itself.
|
|
342
|
-
*
|
|
343
|
-
* The loop already dedups exact-uri duplicates and only calls this on genuinely
|
|
344
|
-
* new candidates, so the verifier focuses on relevance + near-duplicate
|
|
345
|
-
* judgement, not bookkeeping.
|
|
346
|
-
*/
|
|
347
|
-
function createVerifyingResearchDriver(options = {}) {
|
|
348
|
-
const acceptOnParseFailure = options.acceptOnParseFailure ?? false;
|
|
349
|
-
return { async verifySource(source, ctx) {
|
|
350
|
-
const assertActive = () => throwIfRouterAborted(ctx.signal, options);
|
|
351
|
-
assertActive();
|
|
352
|
-
const router = resolveRouter(options, ctx.signal);
|
|
353
|
-
const gapLines = ctx.gaps.map((gap) => `- ${gap.description} (query: "${gap.query}")`).join("\n");
|
|
354
|
-
const acceptedTitles = ctx.acceptedThisRound.map((accepted) => `- ${accepted.title ?? accepted.uri}`).join("\n");
|
|
355
|
-
const excerpt = source.text.slice(0, 1500);
|
|
356
|
-
const system = "You verify whether a fetched web source belongs in a curated knowledge base. Accept a source ONLY if it is genuinely on-topic for the research goal and helps close one of the open gaps, AND it is not a near-duplicate of an already-accepted source. Reject spam, listicles, off-topic pages, marketing, and near-duplicates. Respond with ONLY a JSON object: {\"accept\": true|false, \"reason\": \"<short reason>\"}.";
|
|
357
|
-
const user = [
|
|
358
|
-
`Research goal: ${ctx.goal}`,
|
|
359
|
-
`Open gaps:\n${gapLines || "(none specified)"}`,
|
|
360
|
-
acceptedTitles ? `Already accepted this round:\n${acceptedTitles}` : "Nothing accepted yet this round.",
|
|
361
|
-
`Candidate source:\nURL: ${source.uri}\nTitle: ${source.title ?? "(none)"}\nExcerpt:\n${excerpt}`,
|
|
362
|
-
"Verdict as JSON {\"accept\": boolean, \"reason\": string}:"
|
|
363
|
-
].join("\n\n");
|
|
364
|
-
let raw = "";
|
|
365
|
-
try {
|
|
366
|
-
raw = await router.chat([{
|
|
367
|
-
role: "system",
|
|
368
|
-
content: system
|
|
369
|
-
}, {
|
|
370
|
-
role: "user",
|
|
371
|
-
content: user
|
|
372
|
-
}], MIN_MAX_TOKENS);
|
|
373
|
-
assertActive();
|
|
374
|
-
} catch (error) {
|
|
375
|
-
assertActive();
|
|
376
|
-
if (error?.name === "AbortError") throw error;
|
|
377
|
-
return acceptOnParseFailure ? { accept: true } : {
|
|
378
|
-
accept: false,
|
|
379
|
-
reason: `verifier unavailable: ${error.message}`
|
|
380
|
-
};
|
|
381
|
-
}
|
|
382
|
-
const verdict = parseVerdict(raw);
|
|
383
|
-
if (verdict) return verdict;
|
|
384
|
-
return acceptOnParseFailure ? { accept: true } : {
|
|
385
|
-
accept: false,
|
|
386
|
-
reason: "verifier returned an unparseable verdict"
|
|
387
|
-
};
|
|
388
|
-
} };
|
|
389
|
-
}
|
|
390
|
-
/** Parse the verifier's `{accept, reason}` JSON, tolerant of fences / prose. */
|
|
391
|
-
function parseVerdict(raw) {
|
|
392
|
-
const text = raw.trim();
|
|
393
|
-
if (!text) return null;
|
|
394
|
-
const objMatch = text.match(/\{[\s\S]*\}/);
|
|
395
|
-
if (!objMatch) return null;
|
|
396
|
-
try {
|
|
397
|
-
const parsed = JSON.parse(objMatch[0]);
|
|
398
|
-
if (typeof parsed.accept !== "boolean") return null;
|
|
399
|
-
if (parsed.accept) return { accept: true };
|
|
400
|
-
return {
|
|
401
|
-
accept: false,
|
|
402
|
-
reason: typeof parsed.reason === "string" && parsed.reason.trim() ? parsed.reason.trim() : "rejected by verifier"
|
|
403
|
-
};
|
|
404
|
-
} catch {
|
|
405
|
-
return null;
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
//#endregion
|
|
409
|
-
//#region src/adaptive-driver.ts
|
|
410
|
-
/**
|
|
411
|
-
* Adaptive verifier mode for `runVerifiedResearchLoop`.
|
|
412
|
-
*
|
|
413
|
-
* The cost/quality A/B (`docs/results/cost-quality.md`) found the LLM relevance
|
|
414
|
-
* verifier's cleanliness win is dominated by DE-DUPLICATION — which a
|
|
415
|
-
* deterministic content-hash / canonical-URL check captures at ~none of the LLM
|
|
416
|
-
* premium — and that an LLM check only earns its dollar on the off-scope tail.
|
|
417
|
-
* The honest production move it names is: do the cheap deterministic work first,
|
|
418
|
-
* spend the LLM only where it pays. This module is that driver.
|
|
419
|
-
*
|
|
420
|
-
* Per candidate source the adaptive driver runs THREE stages, cheapest first,
|
|
421
|
-
* and stops at the first that decides:
|
|
422
|
-
*
|
|
423
|
-
* 1. DEDUP ($0, no LLM). Reject a source whose CONTENT (normalized-text hash)
|
|
424
|
-
* or whose CANONICAL URL matches one already accepted this round or already
|
|
425
|
-
* in the knowledge base. This is the de-dup the relevance judge was being
|
|
426
|
-
* paid to do; doing it deterministically is free and exact.
|
|
427
|
-
*
|
|
428
|
-
* 2. HEURISTIC TRIAGE ($0, no LLM). For a unique survivor, a cheap host /
|
|
429
|
-
* title / length signal classifies it as clearly-keep, clearly-drop, or
|
|
430
|
-
* AMBIGUOUS. Clear cases are resolved without a model: an authoritative host
|
|
431
|
-
* (arxiv, *.edu, *.gov, official docs) with a substantial readable body is
|
|
432
|
-
* kept; an obvious spam/listicle/marketing title or a too-thin body is
|
|
433
|
-
* dropped. Only genuinely ambiguous survivors fall through.
|
|
434
|
-
*
|
|
435
|
-
* 3. LLM ESCALATION ($, one call). ONLY the ambiguous survivors reach the LLM
|
|
436
|
-
* `verifySource` — the shipped `createVerifyingResearchDriver` relevance
|
|
437
|
-
* judge. This is where the verifier earns its premium: the off-scope tail a
|
|
438
|
-
* cheap rule can't adjudicate.
|
|
439
|
-
*
|
|
440
|
-
* The result is the cost/quality frontier point the doc predicted: most of the
|
|
441
|
-
* cleanliness (dedup + clear drops) at a fraction of the LLM $/calls (only the
|
|
442
|
-
* ambiguous tail pays). It is a real `ResearchDriver` — same contract the
|
|
443
|
-
* two-agent loop already gates on — and reuses `sha256`, the relevance verifier,
|
|
444
|
-
* and the index; it reinvents none of them.
|
|
445
|
-
*/
|
|
446
|
-
/**
|
|
447
|
-
* Canonicalize a URL for duplicate detection: lowercase host, strip a leading
|
|
448
|
-
* `www.`, drop the scheme, the fragment, a trailing slash, and tracking query
|
|
449
|
-
* params (`utm_*`, `ref`, `fbclid`, `gclid`, …). Two URLs that differ only by
|
|
450
|
-
* those decorations canonicalize to the same key, so the dedup stage treats them
|
|
451
|
-
* as the same source. Falls back to the lowercased raw string when the input is
|
|
452
|
-
* not a parseable absolute URL (so non-http identifiers still dedup by equality).
|
|
453
|
-
*/
|
|
454
|
-
function canonicalizeUrl(uri) {
|
|
455
|
-
const trimmed = uri.trim();
|
|
456
|
-
try {
|
|
457
|
-
const url = new URL(trimmed);
|
|
458
|
-
const host = url.hostname.toLowerCase().replace(/^www\./, "");
|
|
459
|
-
const kept = [];
|
|
460
|
-
for (const [key, value] of url.searchParams) {
|
|
461
|
-
const lower = key.toLowerCase();
|
|
462
|
-
if (lower.startsWith("utm_")) continue;
|
|
463
|
-
if (trackingParams.has(lower)) continue;
|
|
464
|
-
kept.push([key, value]);
|
|
465
|
-
}
|
|
466
|
-
kept.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
467
|
-
const query = kept.map(([k, v]) => `${k}=${v}`).join("&");
|
|
468
|
-
return `${host}${url.pathname.replace(/\/+$/, "") || "/"}${query ? `?${query}` : ""}`;
|
|
469
|
-
} catch {
|
|
470
|
-
return trimmed.toLowerCase();
|
|
471
|
-
}
|
|
472
|
-
}
|
|
473
|
-
/** Tracking / referrer query params dropped during URL canonicalization. */
|
|
474
|
-
const trackingParams = /* @__PURE__ */ new Set([
|
|
475
|
-
"ref",
|
|
476
|
-
"ref_src",
|
|
477
|
-
"source",
|
|
478
|
-
"fbclid",
|
|
479
|
-
"gclid",
|
|
480
|
-
"mc_cid",
|
|
481
|
-
"mc_eid",
|
|
482
|
-
"igshid",
|
|
483
|
-
"spm",
|
|
484
|
-
"_hsenc",
|
|
485
|
-
"_hsmi"
|
|
486
|
-
]);
|
|
487
|
-
/**
|
|
488
|
-
* A stable content key for a fetched page: the sha256 of its normalized text
|
|
489
|
-
* (lowercased, punctuation/whitespace collapsed). Two pages whose readable body
|
|
490
|
-
* is the same modulo formatting collide here, so the dedup stage rejects a
|
|
491
|
-
* mirror/syndication of an already-accepted source even when the URL differs.
|
|
492
|
-
*/
|
|
493
|
-
function contentKey(text) {
|
|
494
|
-
const normalized = text.toLowerCase().replace(/[^\p{L}\p{N}\s]+/gu, " ").replace(/\s+/g, " ").trim();
|
|
495
|
-
return sha256(normalized);
|
|
496
|
-
}
|
|
497
|
-
/** Default authoritative host suffixes — papers, official docs, standards. */
|
|
498
|
-
const defaultAuthoritativeHosts = [
|
|
499
|
-
"arxiv.org",
|
|
500
|
-
"aclanthology.org",
|
|
501
|
-
"openreview.net",
|
|
502
|
-
"dl.acm.org",
|
|
503
|
-
"ieeexplore.ieee.org",
|
|
504
|
-
"nature.com",
|
|
505
|
-
"science.org",
|
|
506
|
-
"pubmed.ncbi.nlm.nih.gov",
|
|
507
|
-
"ncbi.nlm.nih.gov",
|
|
508
|
-
".edu",
|
|
509
|
-
".gov",
|
|
510
|
-
"docs.python.org",
|
|
511
|
-
"pytorch.org",
|
|
512
|
-
"tensorflow.org",
|
|
513
|
-
"huggingface.co",
|
|
514
|
-
"github.com",
|
|
515
|
-
"developer.mozilla.org",
|
|
516
|
-
"wikipedia.org",
|
|
517
|
-
"w3.org",
|
|
518
|
-
"ietf.org",
|
|
519
|
-
"rfc-editor.org"
|
|
520
|
-
];
|
|
521
|
-
/** Default spam/listicle/marketing title patterns. */
|
|
522
|
-
const defaultSpamPatterns = [
|
|
523
|
-
/\bbuy\b.*\b(cheap|now|deal|sale|discount)\b/i,
|
|
524
|
-
/\b\d+\s+(things|ways|reasons|tips|tricks|secrets|hacks)\b.*\b(you|that|will)\b/i,
|
|
525
|
-
/\bshock(ing|ed)?\b/i,
|
|
526
|
-
/\bclickbait\b/i,
|
|
527
|
-
/!!!|\$\$\$/,
|
|
528
|
-
/\b(coupon|promo code|affiliate|sponsored)\b/i,
|
|
529
|
-
/\bbest .* (of \d{4}|in \d{4})\b/i
|
|
530
|
-
];
|
|
531
|
-
/**
|
|
532
|
-
* Classify a UNIQUE survivor (already past dedup) with cheap host/title/length
|
|
533
|
-
* signals only — no LLM. Returns `keep`, `drop`, or `ambiguous`. `ambiguous` is
|
|
534
|
-
* the residue the LLM is reserved for: on-topic-looking pages on unknown hosts
|
|
535
|
-
* with a plausible body, which a host/title rule cannot adjudicate.
|
|
536
|
-
*/
|
|
537
|
-
function triageSource(source, options) {
|
|
538
|
-
const titleAndSnippet = `${source.title ?? ""} ${typeof source.metadata?.snippet === "string" ? source.metadata.snippet : ""}`.trim();
|
|
539
|
-
const bodyLen = source.text.trim().length;
|
|
540
|
-
for (const pattern of options.spamPatterns) if (pattern.test(titleAndSnippet)) return {
|
|
541
|
-
triage: "drop",
|
|
542
|
-
reason: `spam/listicle title (${pattern.source})`
|
|
543
|
-
};
|
|
544
|
-
if (bodyLen < options.minBodyChars) return {
|
|
545
|
-
triage: "drop",
|
|
546
|
-
reason: `thin body (${bodyLen} < ${options.minBodyChars} chars)`
|
|
547
|
-
};
|
|
548
|
-
const host = hostOf(source.uri);
|
|
549
|
-
if (host.length > 0 && options.authoritativeHosts.some((suffix) => hostMatches(host, suffix)) && bodyLen >= options.substantialBodyChars) return {
|
|
550
|
-
triage: "keep",
|
|
551
|
-
reason: `authoritative host ${host} + substantial body (${bodyLen})`
|
|
552
|
-
};
|
|
553
|
-
return {
|
|
554
|
-
triage: "ambiguous",
|
|
555
|
-
reason: `unknown host ${host || "(none)"}, body ${bodyLen} chars`
|
|
556
|
-
};
|
|
557
|
-
}
|
|
558
|
-
function hostOf(uri) {
|
|
559
|
-
try {
|
|
560
|
-
return new URL(uri.trim()).hostname.toLowerCase().replace(/^www\./, "");
|
|
561
|
-
} catch {
|
|
562
|
-
return "";
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
/** Suffix host match: `.edu` matches `mit.edu`; `arxiv.org` matches `export.arxiv.org`. */
|
|
566
|
-
function hostMatches(host, suffix) {
|
|
567
|
-
if (suffix.startsWith(".")) return host.endsWith(suffix) || host === suffix.slice(1);
|
|
568
|
-
return host === suffix || host.endsWith(`.${suffix}`);
|
|
569
|
-
}
|
|
570
|
-
/**
|
|
571
|
-
* Build the adaptive verifier. The deterministic stages (dedup + heuristic
|
|
572
|
-
* triage) cost $0; only AMBIGUOUS survivors escalate to the LLM relevance
|
|
573
|
-
* verifier. `stats()` exposes where every decision was spent so the A/B can read
|
|
574
|
-
* the LLM $/calls the adaptive driver saved against the cleanliness it kept.
|
|
575
|
-
*
|
|
576
|
-
* Dedup state is kept on the driver instance: it tracks the canonical URLs and
|
|
577
|
-
* content hashes it has ACCEPTED, plus those it sees in the verification
|
|
578
|
-
* context's `acceptedThisRound` and the KB index. Use one driver per loop run.
|
|
579
|
-
*/
|
|
580
|
-
function createAdaptiveResearchDriver(options = {}) {
|
|
581
|
-
const authoritativeHosts = options.authoritativeHosts ?? defaultAuthoritativeHosts;
|
|
582
|
-
const spamPatterns = options.spamPatterns ?? defaultSpamPatterns;
|
|
583
|
-
const minBodyChars = Math.max(1, options.minBodyChars ?? 400);
|
|
584
|
-
const substantialBodyChars = Math.max(minBodyChars, options.substantialBodyChars ?? 600);
|
|
585
|
-
const relevance = createVerifyingResearchDriver({
|
|
586
|
-
router: options.router,
|
|
587
|
-
router_options: options.router_options,
|
|
588
|
-
acceptOnParseFailure: options.verifying?.acceptOnParseFailure
|
|
589
|
-
});
|
|
590
|
-
const acceptedUrlKeys = /* @__PURE__ */ new Set();
|
|
591
|
-
const acceptedContentKeys = /* @__PURE__ */ new Set();
|
|
592
|
-
const stats = {
|
|
593
|
-
total: 0,
|
|
594
|
-
dedupRejected: 0,
|
|
595
|
-
heuristicKept: 0,
|
|
596
|
-
heuristicDropped: 0,
|
|
597
|
-
llmCalls: 0,
|
|
598
|
-
llmAccepted: 0,
|
|
599
|
-
decisions: []
|
|
600
|
-
};
|
|
601
|
-
function record(decision) {
|
|
602
|
-
stats.decisions.push(decision);
|
|
603
|
-
options.onDecision?.(decision);
|
|
604
|
-
}
|
|
605
|
-
return {
|
|
606
|
-
async verifySource(source, ctx) {
|
|
607
|
-
stats.total += 1;
|
|
608
|
-
const urlKey = canonicalizeUrl(source.uri);
|
|
609
|
-
const cKey = contentKey(source.text);
|
|
610
|
-
const roundUrlKeys = new Set(ctx.acceptedThisRound.map((accepted) => canonicalizeUrl(accepted.uri)));
|
|
611
|
-
const roundContentKeys = new Set(ctx.acceptedThisRound.map((accepted) => contentKey(accepted.text)));
|
|
612
|
-
const indexUrlKeys = new Set(ctx.index.sources.flatMap((indexed) => typeof indexed.metadata?.originalUri === "string" ? [canonicalizeUrl(indexed.metadata.originalUri)] : []));
|
|
613
|
-
const dupUrl = acceptedUrlKeys.has(urlKey) || roundUrlKeys.has(urlKey) || indexUrlKeys.has(urlKey);
|
|
614
|
-
const dupContent = acceptedContentKeys.has(cKey) || roundContentKeys.has(cKey);
|
|
615
|
-
if (dupUrl || dupContent) {
|
|
616
|
-
stats.dedupRejected += 1;
|
|
617
|
-
const reason = dupUrl ? "duplicate-url" : "duplicate-content";
|
|
618
|
-
record({
|
|
619
|
-
uri: source.uri,
|
|
620
|
-
stage: "dedup",
|
|
621
|
-
accepted: false,
|
|
622
|
-
reason
|
|
623
|
-
});
|
|
624
|
-
return {
|
|
625
|
-
accept: false,
|
|
626
|
-
reason: `dedup: ${reason}`
|
|
627
|
-
};
|
|
628
|
-
}
|
|
629
|
-
const { triage, reason } = triageSource(source, {
|
|
630
|
-
authoritativeHosts,
|
|
631
|
-
spamPatterns,
|
|
632
|
-
minBodyChars,
|
|
633
|
-
substantialBodyChars
|
|
634
|
-
});
|
|
635
|
-
if (triage === "keep") {
|
|
636
|
-
stats.heuristicKept += 1;
|
|
637
|
-
acceptedUrlKeys.add(urlKey);
|
|
638
|
-
acceptedContentKeys.add(cKey);
|
|
639
|
-
record({
|
|
640
|
-
uri: source.uri,
|
|
641
|
-
stage: "heuristic",
|
|
642
|
-
accepted: true,
|
|
643
|
-
triage,
|
|
644
|
-
reason
|
|
645
|
-
});
|
|
646
|
-
return { accept: true };
|
|
647
|
-
}
|
|
648
|
-
if (triage === "drop") {
|
|
649
|
-
stats.heuristicDropped += 1;
|
|
650
|
-
record({
|
|
651
|
-
uri: source.uri,
|
|
652
|
-
stage: "heuristic",
|
|
653
|
-
accepted: false,
|
|
654
|
-
triage,
|
|
655
|
-
reason
|
|
656
|
-
});
|
|
657
|
-
return {
|
|
658
|
-
accept: false,
|
|
659
|
-
reason: `heuristic drop: ${reason}`
|
|
660
|
-
};
|
|
661
|
-
}
|
|
662
|
-
stats.llmCalls += 1;
|
|
663
|
-
const verdict = await relevance.verifySource(source, ctx);
|
|
664
|
-
if (verdict.accept) {
|
|
665
|
-
stats.llmAccepted += 1;
|
|
666
|
-
acceptedUrlKeys.add(urlKey);
|
|
667
|
-
acceptedContentKeys.add(cKey);
|
|
668
|
-
}
|
|
669
|
-
record({
|
|
670
|
-
uri: source.uri,
|
|
671
|
-
stage: "llm",
|
|
672
|
-
accepted: verdict.accept,
|
|
673
|
-
triage,
|
|
674
|
-
reason: verdict.accept ? "llm accepted" : verdict.reason
|
|
675
|
-
});
|
|
676
|
-
return verdict;
|
|
677
|
-
},
|
|
678
|
-
stats() {
|
|
679
|
-
return {
|
|
680
|
-
...stats,
|
|
681
|
-
decisions: [...stats.decisions]
|
|
682
|
-
};
|
|
683
|
-
}
|
|
684
|
-
};
|
|
685
|
-
}
|
|
686
|
-
//#endregion
|
|
687
16
|
//#region src/knowledge-state-scope.ts
|
|
688
17
|
const knowledgeStateScopeSchema = z.object({
|
|
689
18
|
pagesDirectory: z.string().transform((value) => normalizePagesDirectory(value)).optional(),
|
|
@@ -906,17 +235,6 @@ const EVALUATION_PHASES = [
|
|
|
906
235
|
"answer-quality",
|
|
907
236
|
"promotion"
|
|
908
237
|
];
|
|
909
|
-
/**
|
|
910
|
-
* The pages directory every knowledge-improvement path reads and writes.
|
|
911
|
-
*
|
|
912
|
-
* This subsystem has no option for a caller-chosen pages directory: the
|
|
913
|
-
* workspace it snapshots, the plan it hashes, and the transaction it applies
|
|
914
|
-
* all address the default tree. Naming it once means the write allowlist, the
|
|
915
|
-
* plan hash, and the generated-page set cannot disagree about which tree they
|
|
916
|
-
* are talking about, and threading a real choice through later changes one
|
|
917
|
-
* constant instead of hunting call sites.
|
|
918
|
-
*/
|
|
919
|
-
const KB_IMPROVEMENT_PAGES_DIRECTORY = DEFAULT_PAGES_DIRECTORY;
|
|
920
238
|
//#endregion
|
|
921
239
|
//#region src/kb-improvement/state.ts
|
|
922
240
|
function knowledgeImprovementRunId(root, goal) {
|
|
@@ -1163,6 +481,13 @@ function immutableJsonValue(value) {
|
|
|
1163
481
|
return Object.freeze(value);
|
|
1164
482
|
}
|
|
1165
483
|
//#endregion
|
|
484
|
+
//#region src/immutable-ref.ts
|
|
485
|
+
const SHA256_REF = /^sha256:[a-f0-9]{64}$/;
|
|
486
|
+
const GIT_REF = /^git:[a-f0-9]{40}$/;
|
|
487
|
+
function assertImmutableRef(value, label) {
|
|
488
|
+
if (typeof value !== "string" || !SHA256_REF.test(value) && !GIT_REF.test(value)) throw new Error(`${label} must be lowercase sha256:<64 hex> or git:<40 hex>`);
|
|
489
|
+
}
|
|
490
|
+
//#endregion
|
|
1166
491
|
//#region src/rag-answer-evidence.ts
|
|
1167
492
|
function assertRagAnswerEvidence(result) {
|
|
1168
493
|
if (!result || typeof result !== "object") throw new Error("answer-quality evidence must be an object");
|
|
@@ -1190,34 +515,108 @@ function exceedsCostCeiling$1(totalCostUsd, costCeiling) {
|
|
|
1190
515
|
return totalCostUsd - costCeiling > tolerance;
|
|
1191
516
|
}
|
|
1192
517
|
//#endregion
|
|
1193
|
-
//#region src/
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
const
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
518
|
+
//#region src/rag-eval/knowledge-base.ts
|
|
519
|
+
function scoreKnowledgeBaseIndex(index, options = {}) {
|
|
520
|
+
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
521
|
+
const lintFindings = lintKnowledgeIndex(index);
|
|
522
|
+
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
523
|
+
const staleSourceCount = index.sources.filter((source) => {
|
|
524
|
+
return source.validUntil ? Date.parse(source.validUntil) < now.getTime() : false;
|
|
525
|
+
}).length;
|
|
526
|
+
const sourceHashCounts = /* @__PURE__ */ new Map();
|
|
527
|
+
for (const source of index.sources) sourceHashCounts.set(source.contentHash, (sourceHashCounts.get(source.contentHash) ?? 0) + 1);
|
|
528
|
+
const duplicateSourceHashCount = [...sourceHashCounts.values()].filter((count) => count > 1).length;
|
|
529
|
+
const pagesWithInlineCitations = index.pages.filter((page) => sourceRefs(page.text).length > 0).length;
|
|
530
|
+
const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length;
|
|
531
|
+
const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates);
|
|
532
|
+
const maxNearDuplicatePageRate = unitInterval(options.maxNearDuplicatePageRate ?? 1, "maxNearDuplicatePageRate");
|
|
533
|
+
const metrics = {
|
|
534
|
+
page_count: index.pages.length,
|
|
535
|
+
source_count: index.sources.length,
|
|
536
|
+
citation_rate: index.pages.length === 0 ? 1 : pagesWithInlineCitations / index.pages.length,
|
|
537
|
+
source_backed_page_rate: index.pages.length === 0 ? 1 : pagesWithSources / index.pages.length,
|
|
538
|
+
stale_source_rate: index.sources.length === 0 ? 0 : staleSourceCount / index.sources.length,
|
|
539
|
+
duplicate_source_hash_rate: index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
|
|
540
|
+
lint_error_count: validation.findings.filter((finding) => finding.severity === "error").length,
|
|
541
|
+
lint_warning_count: lintFindings.filter((finding) => finding.severity === "warning").length,
|
|
542
|
+
near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
|
|
543
|
+
near_duplicate_page_count: nearDuplicates.duplicatePageCount,
|
|
544
|
+
near_duplicate_pair_count: nearDuplicates.duplicatePairCount
|
|
545
|
+
};
|
|
546
|
+
const findings = [];
|
|
547
|
+
if (metrics.lint_error_count > 0) findings.push({
|
|
548
|
+
id: "kb:lint-errors",
|
|
549
|
+
kind: "unknown",
|
|
550
|
+
severity: "critical",
|
|
551
|
+
message: `${metrics.lint_error_count} validation error(s) in the knowledge index.`,
|
|
552
|
+
evidence: { lint_error_count: metrics.lint_error_count }
|
|
553
|
+
});
|
|
554
|
+
if (metrics.citation_rate < (options.minCitationRate ?? 0)) findings.push({
|
|
555
|
+
id: "kb:citation-rate",
|
|
556
|
+
kind: "missing-source",
|
|
557
|
+
severity: "error",
|
|
558
|
+
message: "Inline citation coverage is below the configured minimum.",
|
|
559
|
+
evidence: { citation_rate: metrics.citation_rate }
|
|
560
|
+
});
|
|
561
|
+
if (metrics.stale_source_rate > (options.maxStaleSourceRate ?? 1)) findings.push({
|
|
562
|
+
id: "kb:stale-source-rate",
|
|
563
|
+
kind: "stale-source",
|
|
564
|
+
severity: "error",
|
|
565
|
+
message: "Stale source rate exceeds the configured maximum.",
|
|
566
|
+
evidence: { stale_source_rate: metrics.stale_source_rate }
|
|
567
|
+
});
|
|
568
|
+
if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) findings.push({
|
|
569
|
+
id: "kb:near-duplicate-analysis-truncated",
|
|
570
|
+
kind: "unknown",
|
|
571
|
+
severity: "error",
|
|
572
|
+
message: "Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.",
|
|
573
|
+
evidence: {
|
|
574
|
+
candidate_pair_count: nearDuplicates.candidatePairCount,
|
|
575
|
+
compared_pair_count: nearDuplicates.comparedPairCount,
|
|
576
|
+
duplicate_pair_count: nearDuplicates.duplicatePairCount
|
|
577
|
+
}
|
|
578
|
+
});
|
|
579
|
+
else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) findings.push({
|
|
580
|
+
id: "kb:near-duplicate-page-rate",
|
|
581
|
+
kind: "unknown",
|
|
582
|
+
severity: "error",
|
|
583
|
+
message: "Near-duplicate page rate exceeds the configured maximum.",
|
|
584
|
+
evidence: {
|
|
585
|
+
near_duplicate_page_rate: metrics.near_duplicate_page_rate,
|
|
586
|
+
near_duplicate_page_count: metrics.near_duplicate_page_count,
|
|
587
|
+
near_duplicate_pair_count: metrics.near_duplicate_pair_count,
|
|
588
|
+
threshold: nearDuplicates.threshold
|
|
1212
589
|
}
|
|
590
|
+
});
|
|
591
|
+
return {
|
|
592
|
+
ok: findings.length === 0,
|
|
593
|
+
metrics,
|
|
594
|
+
nearDuplicates,
|
|
595
|
+
findings
|
|
1213
596
|
};
|
|
1214
597
|
}
|
|
598
|
+
function sourceRefs(text) {
|
|
599
|
+
const refs = [];
|
|
600
|
+
const regex = /\[\^([A-Za-z0-9_-]+)(?:#([A-Za-z0-9_.:-]+))?\]/g;
|
|
601
|
+
let match = regex.exec(text);
|
|
602
|
+
while (match !== null) {
|
|
603
|
+
refs.push(match[0]);
|
|
604
|
+
match = regex.exec(text);
|
|
605
|
+
}
|
|
606
|
+
return refs;
|
|
607
|
+
}
|
|
608
|
+
function unitInterval(value, name) {
|
|
609
|
+
if (!Number.isFinite(value) || value < 0 || value > 1) throw new Error(`${name} must be a finite number in [0, 1]`);
|
|
610
|
+
return value;
|
|
611
|
+
}
|
|
612
|
+
//#endregion
|
|
613
|
+
//#region src/claim-grounding.ts
|
|
1215
614
|
/**
|
|
1216
615
|
* Stopwords stripped before overlap scoring so the bar measures the claim's
|
|
1217
616
|
* SUBSTANTIVE words (the numbers, nouns, methods it asserts), not filler a
|
|
1218
617
|
* misattributed page would trivially share with the real one.
|
|
1219
618
|
*/
|
|
1220
|
-
const stopwords$
|
|
619
|
+
const stopwords$1 = /* @__PURE__ */ new Set([
|
|
1221
620
|
"the",
|
|
1222
621
|
"a",
|
|
1223
622
|
"an",
|
|
@@ -1274,12 +673,11 @@ function normalize(text) {
|
|
|
1274
673
|
}
|
|
1275
674
|
/** The claim's substantive content words (deduped, stopwords + short words removed). */
|
|
1276
675
|
function contentWords$1(claim, minWordLength) {
|
|
1277
|
-
const words = normalize(claim).split(" ").filter((word) => word.length >= minWordLength && !stopwords$
|
|
676
|
+
const words = normalize(claim).split(" ").filter((word) => word.length >= minWordLength && !stopwords$1.has(word));
|
|
1278
677
|
return [...new Set(words)];
|
|
1279
678
|
}
|
|
1280
679
|
/**
|
|
1281
|
-
*
|
|
1282
|
-
* page the worker fetched)? Deterministic, no model call:
|
|
680
|
+
* Is `claim` grounded in `pageText`? Deterministic, no model call:
|
|
1283
681
|
*
|
|
1284
682
|
* 1. verbatim — the claim string appears as-is (case-insensitive).
|
|
1285
683
|
* 2. normalized — the claim appears after collapsing punctuation/whitespace
|
|
@@ -1288,9 +686,8 @@ function contentWords$1(claim, minWordLength) {
|
|
|
1288
686
|
* substantive content words appear in the page text. A misattributed page
|
|
1289
687
|
* fails here because the SPECIFIC words the claim asserts are absent.
|
|
1290
688
|
*
|
|
1291
|
-
* Returns the match mode, the measured overlap, and the missing content words
|
|
1292
|
-
*
|
|
1293
|
-
* audit WHY a claim grounded or didn't.
|
|
689
|
+
* Returns the match mode, the measured overlap, and the missing content words,
|
|
690
|
+
* so a caller can state why a claim grounded or did not.
|
|
1294
691
|
*/
|
|
1295
692
|
function groundClaimInText(claim, pageText, options = {}) {
|
|
1296
693
|
const minOverlap = options.minOverlap ?? .7;
|
|
@@ -1340,76 +737,27 @@ function groundClaimInText(claim, pageText, options = {}) {
|
|
|
1340
737
|
missingWords
|
|
1341
738
|
};
|
|
1342
739
|
}
|
|
740
|
+
//#endregion
|
|
741
|
+
//#region src/statistics.ts
|
|
1343
742
|
/**
|
|
1344
|
-
*
|
|
1345
|
-
* CLAIM GROUNDING: it rejects a source whose cited claim is not present in its
|
|
1346
|
-
* fetched page text — a misattributed / fabricated citation — and (optionally)
|
|
1347
|
-
* composes a relevance verifier after grounding passes.
|
|
743
|
+
* The mean of a set of measurements.
|
|
1348
744
|
*
|
|
1349
|
-
*
|
|
1350
|
-
*
|
|
745
|
+
* A measurement that is not a finite number is not a measurement: it is a
|
|
746
|
+
* scorer that divided by zero or an adapter that answered with nothing usable.
|
|
747
|
+
* Averaging it in would spread one broken probe across the whole aggregate,
|
|
748
|
+
* and carrying it through would make the aggregate itself unusable — a
|
|
749
|
+
* non-finite `scoreMean` fails every numeric comparison in the ranking
|
|
750
|
+
* comparator, so the candidate carrying it falls through to the identifier
|
|
751
|
+
* tie-break and places by name rather than by what it scored. Such values are
|
|
752
|
+
* left out, and the mean reports what was actually measured.
|
|
753
|
+
*
|
|
754
|
+
* An empty set answers `0`. That is a reported score, not an absence, and it
|
|
755
|
+
* is the behavior every caller in this package already depends on.
|
|
1351
756
|
*/
|
|
1352
|
-
function
|
|
1353
|
-
const
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
if (!claim) {
|
|
1357
|
-
if (onMissingClaim === "reject") return {
|
|
1358
|
-
accept: false,
|
|
1359
|
-
reason: "no cited claim: claim-grounding mode requires every source to declare its claim"
|
|
1360
|
-
};
|
|
1361
|
-
return options.relevanceVerifier ? options.relevanceVerifier(source, ctx) : { accept: true };
|
|
1362
|
-
}
|
|
1363
|
-
const grounding = groundClaimInText(claim, source.text, options);
|
|
1364
|
-
if (!grounding.grounded) return {
|
|
1365
|
-
accept: false,
|
|
1366
|
-
reason: `misattributed citation: ${grounding.mode === "empty-text" ? "fetched page has no text" : `claim not found in the fetched page (overlap ${(grounding.overlap * 100).toFixed(0)}%${grounding.missingWords.length ? `, missing: ${grounding.missingWords.slice(0, 6).join(", ")}` : ""})`}`
|
|
1367
|
-
};
|
|
1368
|
-
if (options.relevanceVerifier) return options.relevanceVerifier(source, ctx);
|
|
1369
|
-
return { accept: true };
|
|
1370
|
-
};
|
|
1371
|
-
}
|
|
1372
|
-
/**
|
|
1373
|
-
* Ask an LLM to state, for one source, the single specific factual claim a
|
|
1374
|
-
* researcher would cite THIS page for toward the goal. Used to DECORATE the
|
|
1375
|
-
* sources a relevance-only worker produced with the claim a citation would make,
|
|
1376
|
-
* so the claim-grounding verifier has something executable to check. The model
|
|
1377
|
-
* is told to ground the claim in the provided excerpt; the verifier then checks
|
|
1378
|
-
* it against the FULL page text independently — the model does not get to mark
|
|
1379
|
-
* its own homework.
|
|
1380
|
-
*
|
|
1381
|
-
* Returns the proposal annotated via `withCitedClaim`, or the original proposal
|
|
1382
|
-
* unchanged if the model returns nothing parseable (the verifier's
|
|
1383
|
-
* `onMissingClaim` policy then decides).
|
|
1384
|
-
*/
|
|
1385
|
-
function createClaimDecorator(options = {}) {
|
|
1386
|
-
const maxTokens = options.maxTokens ?? 1200;
|
|
1387
|
-
return async function decorate(source, goal) {
|
|
1388
|
-
const router = options.router ?? createTangleRouterClient(options.router_options);
|
|
1389
|
-
const excerpt = source.text.slice(0, 1500);
|
|
1390
|
-
const system = "You extract the single most important factual claim a researcher would cite a page for. State ONE concrete, checkable sentence using the page's own key terms and numbers. Do NOT invent facts not in the excerpt. Respond with ONLY the claim sentence, no prose.";
|
|
1391
|
-
const user = [
|
|
1392
|
-
`Research goal: ${goal}`,
|
|
1393
|
-
`Page title: ${source.title ?? "(none)"}`,
|
|
1394
|
-
`Page excerpt:\n${excerpt}`,
|
|
1395
|
-
"The single claim this page should be cited for:"
|
|
1396
|
-
].join("\n\n");
|
|
1397
|
-
let raw = "";
|
|
1398
|
-
try {
|
|
1399
|
-
raw = await router.chat([{
|
|
1400
|
-
role: "system",
|
|
1401
|
-
content: system
|
|
1402
|
-
}, {
|
|
1403
|
-
role: "user",
|
|
1404
|
-
content: user
|
|
1405
|
-
}], maxTokens);
|
|
1406
|
-
} catch {
|
|
1407
|
-
return source;
|
|
1408
|
-
}
|
|
1409
|
-
const claim = raw.trim().split("\n")[0]?.trim();
|
|
1410
|
-
if (!claim) return source;
|
|
1411
|
-
return withCitedClaim(source, claim);
|
|
1412
|
-
};
|
|
757
|
+
function mean(values) {
|
|
758
|
+
const finite = values.filter(Number.isFinite);
|
|
759
|
+
if (finite.length === 0) return 0;
|
|
760
|
+
return finite.reduce((sum, value) => sum + value, 0) / finite.length;
|
|
1413
761
|
}
|
|
1414
762
|
//#endregion
|
|
1415
763
|
//#region src/rag-eval/answer.ts
|
|
@@ -1532,7 +880,7 @@ function textSupportScore(needle, haystack) {
|
|
|
1532
880
|
return needleWords.filter((word) => haystackWords.has(word)).length / needleWords.length;
|
|
1533
881
|
}
|
|
1534
882
|
function contentWords(text) {
|
|
1535
|
-
const normalized = text.toLowerCase().replace(/[^\p{L}\p{N}\s]+/gu, " ").split(/\s+/).map((word) => stem(word.trim())).filter((word) => (word.length >= 3 || /^\d+$/.test(word)) && !stopwords
|
|
883
|
+
const normalized = text.toLowerCase().replace(/[^\p{L}\p{N}\s]+/gu, " ").split(/\s+/).map((word) => stem(word.trim())).filter((word) => (word.length >= 3 || /^\d+$/.test(word)) && !stopwords.has(word));
|
|
1536
884
|
return [...new Set(normalized)];
|
|
1537
885
|
}
|
|
1538
886
|
function stem(word) {
|
|
@@ -1549,7 +897,7 @@ function clamp01(value) {
|
|
|
1549
897
|
if (!Number.isFinite(value)) return 0;
|
|
1550
898
|
return Math.max(0, Math.min(1, value));
|
|
1551
899
|
}
|
|
1552
|
-
const stopwords
|
|
900
|
+
const stopwords = /* @__PURE__ */ new Set([
|
|
1553
901
|
"the",
|
|
1554
902
|
"a",
|
|
1555
903
|
"an",
|
|
@@ -1863,14 +1211,6 @@ function mergeMetrics(deterministic, providerScores, policy = "prefer-external")
|
|
|
1863
1211
|
}
|
|
1864
1212
|
return merged;
|
|
1865
1213
|
}
|
|
1866
|
-
function aggregateRagAnswerMetrics(summaries) {
|
|
1867
|
-
const keys = /* @__PURE__ */ new Set();
|
|
1868
|
-
for (const summary of summaries) for (const key of Object.keys(summary.metrics)) keys.add(key);
|
|
1869
|
-
const out = {};
|
|
1870
|
-
for (const key of [...keys].sort()) out[key] = mean(summaries.map((summary) => summary.metrics[key]));
|
|
1871
|
-
out.composite = mean(summaries.map((summary) => summary.composite));
|
|
1872
|
-
return out;
|
|
1873
|
-
}
|
|
1874
1214
|
function weightedComposite(metrics, weights) {
|
|
1875
1215
|
let weighted = 0;
|
|
1876
1216
|
let total = 0;
|
|
@@ -1888,162 +1228,169 @@ function below(metric, threshold) {
|
|
|
1888
1228
|
return threshold !== void 0 && metric < threshold;
|
|
1889
1229
|
}
|
|
1890
1230
|
//#endregion
|
|
1891
|
-
//#region src/
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1231
|
+
//#region src/optimization.ts
|
|
1232
|
+
/**
|
|
1233
|
+
* Runs one complete agent-eval method over a serialized knowledge candidate.
|
|
1234
|
+
* The method receives train and selection data; agent-eval owns final scoring.
|
|
1235
|
+
*/
|
|
1236
|
+
async function runSerializedKnowledgeOptimization(options) {
|
|
1237
|
+
assertCompleteOptimizationMethod(options.method);
|
|
1238
|
+
assertImmutableRef(options.executionRef, "knowledge optimization executionRef");
|
|
1239
|
+
const codec = options.codec ?? jsonCandidateCodec();
|
|
1240
|
+
const baseline = normalizeCandidate(options.baseline, codec, "baseline");
|
|
1241
|
+
const dispatchRef = [
|
|
1242
|
+
"knowledge-optimization",
|
|
1243
|
+
options.executionRef,
|
|
1244
|
+
baseline.surfaceHash,
|
|
1245
|
+
surfaceHash(options.method.name)
|
|
1246
|
+
].join(":");
|
|
1247
|
+
assertPartitionContent(options.trainScenarios, options.selectionScenarios, options.finalScenarios, options.scenarioFingerprint ?? scenarioContentFingerprint);
|
|
1248
|
+
const method = canonicalCandidateMethod(options.method, codec);
|
|
1249
|
+
const { baseline: _baseline, method: _method, trainScenarios, selectionScenarios, finalScenarios, dispatchCandidate, executionRef: _executionRef, codec: _codec, scenarioFingerprint: _scenarioFingerprint, ...comparisonOptions } = options;
|
|
1250
|
+
const optimizationRunOptions = {
|
|
1251
|
+
...comparisonOptions.reps !== void 0 ? { reps: comparisonOptions.reps } : {},
|
|
1252
|
+
...comparisonOptions.resumable !== void 0 ? { resumable: comparisonOptions.resumable } : {},
|
|
1253
|
+
...comparisonOptions.labeledStore !== void 0 ? { labeledStore: comparisonOptions.labeledStore } : {},
|
|
1254
|
+
...comparisonOptions.captureSource !== void 0 ? { captureSource: comparisonOptions.captureSource } : {},
|
|
1255
|
+
...comparisonOptions.captureSourceVersionHash !== void 0 ? { captureSourceVersionHash: comparisonOptions.captureSourceVersionHash } : {},
|
|
1256
|
+
...comparisonOptions.maxConcurrency !== void 0 ? { maxConcurrency: comparisonOptions.maxConcurrency } : {},
|
|
1257
|
+
...comparisonOptions.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: comparisonOptions.dispatchTimeoutMs } : {},
|
|
1258
|
+
...comparisonOptions.repo !== void 0 ? { repo: comparisonOptions.repo } : {},
|
|
1259
|
+
...comparisonOptions.tracing !== void 0 ? { tracing: comparisonOptions.tracing } : {},
|
|
1260
|
+
...comparisonOptions.expectUsage !== void 0 ? { expectUsage: comparisonOptions.expectUsage } : {},
|
|
1261
|
+
...comparisonOptions.now !== void 0 ? { now: comparisonOptions.now } : {},
|
|
1262
|
+
...comparisonOptions.buildTraceWriter !== void 0 ? { buildTraceWriter: comparisonOptions.buildTraceWriter } : {},
|
|
1263
|
+
...comparisonOptions.storage !== void 0 ? { storage: comparisonOptions.storage } : {},
|
|
1264
|
+
...comparisonOptions.cellPlacement !== void 0 ? { cellPlacement: comparisonOptions.cellPlacement } : {},
|
|
1265
|
+
...comparisonOptions.optimizationRunOptions ?? {},
|
|
1266
|
+
dispatchRef
|
|
1267
|
+
};
|
|
1268
|
+
const comparison = await compareOptimizationMethods({
|
|
1269
|
+
...comparisonOptions,
|
|
1270
|
+
dispatchRef,
|
|
1271
|
+
optimizationRunOptions,
|
|
1272
|
+
methods: [method],
|
|
1273
|
+
baselineSurface: baseline.surface,
|
|
1274
|
+
trainScenarios: trainScenarios.map((scenario) => structuredClone(scenario)),
|
|
1275
|
+
selectionScenarios: selectionScenarios.map((scenario) => structuredClone(scenario)),
|
|
1276
|
+
testScenarios: finalScenarios.map((scenario) => structuredClone(scenario)),
|
|
1277
|
+
dispatchWithSurface: async (surface, scenario, context) => {
|
|
1278
|
+
const candidate = normalizeSurface(surface, codec, "candidate");
|
|
1279
|
+
return dispatchCandidate({
|
|
1280
|
+
candidate: structuredClone(candidate.value),
|
|
1281
|
+
candidateSurface: candidate.surface,
|
|
1282
|
+
candidateSurfaceHash: candidate.surfaceHash,
|
|
1903
1283
|
scenario,
|
|
1904
|
-
|
|
1905
|
-
});
|
|
1906
|
-
const summary = scoreRagAnswerArtifact(external ? {
|
|
1907
|
-
...initialArtifact,
|
|
1908
|
-
externalScores: [...initialArtifact.externalScores ?? [], ...Array.isArray(external) ? external : [external]]
|
|
1909
|
-
} : initialArtifact, scenario, {
|
|
1910
|
-
thresholds: options.thresholds,
|
|
1911
|
-
weights: options.weights
|
|
1284
|
+
context
|
|
1912
1285
|
});
|
|
1913
|
-
summaries.push(summary);
|
|
1914
|
-
findings.push(...summary.findings);
|
|
1915
1286
|
}
|
|
1916
|
-
const metrics = aggregateRagAnswerMetrics(summaries);
|
|
1917
|
-
const cost = typeof options.cost === "function" ? await options.cost() : structuredClone(options.cost);
|
|
1918
|
-
return {
|
|
1919
|
-
passed: findings.length === 0,
|
|
1920
|
-
metrics,
|
|
1921
|
-
finalScenarioIds,
|
|
1922
|
-
datasetRef,
|
|
1923
|
-
evaluatorRef: options.evaluatorRef,
|
|
1924
|
-
cost,
|
|
1925
|
-
findings,
|
|
1926
|
-
metadata: { scenarioCount: options.scenarios.length }
|
|
1927
|
-
};
|
|
1928
|
-
};
|
|
1929
|
-
}
|
|
1930
|
-
async function calibrateRagAnswerJudge(options) {
|
|
1931
|
-
const judge = options.judge ?? ragAnswerQualityJudge();
|
|
1932
|
-
const strong = await judge.score({
|
|
1933
|
-
artifact: options.strong,
|
|
1934
|
-
scenario: options.scenario,
|
|
1935
|
-
signal: options.signal ?? new AbortController().signal
|
|
1936
1287
|
});
|
|
1937
|
-
const
|
|
1938
|
-
artifact: options.weak,
|
|
1939
|
-
scenario: options.scenario,
|
|
1940
|
-
signal: options.signal ?? new AbortController().signal
|
|
1941
|
-
});
|
|
1942
|
-
const strongScore = strong.composite;
|
|
1943
|
-
const weakScore = weak.composite;
|
|
1944
|
-
const minStrongScore = options.minStrongScore ?? .7;
|
|
1945
|
-
const maxWeakScore = options.maxWeakScore ?? .3;
|
|
1288
|
+
const winner = normalizeSurface(comparison.best.winnerSurface, codec, "winner");
|
|
1946
1289
|
return {
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1290
|
+
methodName: options.method.name,
|
|
1291
|
+
baseline,
|
|
1292
|
+
winner,
|
|
1293
|
+
comparison
|
|
1951
1294
|
};
|
|
1952
1295
|
}
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
function
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
const maxNearDuplicatePageRate = unitInterval(options.maxNearDuplicatePageRate ?? 1, "maxNearDuplicatePageRate");
|
|
1969
|
-
const metrics = {
|
|
1970
|
-
page_count: index.pages.length,
|
|
1971
|
-
source_count: index.sources.length,
|
|
1972
|
-
citation_rate: index.pages.length === 0 ? 1 : pagesWithInlineCitations / index.pages.length,
|
|
1973
|
-
source_backed_page_rate: index.pages.length === 0 ? 1 : pagesWithSources / index.pages.length,
|
|
1974
|
-
stale_source_rate: index.sources.length === 0 ? 0 : staleSourceCount / index.sources.length,
|
|
1975
|
-
duplicate_source_hash_rate: index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
|
|
1976
|
-
lint_error_count: validation.findings.filter((finding) => finding.severity === "error").length,
|
|
1977
|
-
lint_warning_count: lintFindings.filter((finding) => finding.severity === "warning").length,
|
|
1978
|
-
near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
|
|
1979
|
-
near_duplicate_page_count: nearDuplicates.duplicatePageCount,
|
|
1980
|
-
near_duplicate_pair_count: nearDuplicates.duplicatePairCount
|
|
1981
|
-
};
|
|
1982
|
-
const findings = [];
|
|
1983
|
-
if (metrics.lint_error_count > 0) findings.push({
|
|
1984
|
-
id: "kb:lint-errors",
|
|
1985
|
-
kind: "unknown",
|
|
1986
|
-
severity: "critical",
|
|
1987
|
-
message: `${metrics.lint_error_count} validation error(s) in the knowledge index.`,
|
|
1988
|
-
evidence: { lint_error_count: metrics.lint_error_count }
|
|
1989
|
-
});
|
|
1990
|
-
if (metrics.citation_rate < (options.minCitationRate ?? 0)) findings.push({
|
|
1991
|
-
id: "kb:citation-rate",
|
|
1992
|
-
kind: "missing-source",
|
|
1993
|
-
severity: "error",
|
|
1994
|
-
message: "Inline citation coverage is below the configured minimum.",
|
|
1995
|
-
evidence: { citation_rate: metrics.citation_rate }
|
|
1996
|
-
});
|
|
1997
|
-
if (metrics.stale_source_rate > (options.maxStaleSourceRate ?? 1)) findings.push({
|
|
1998
|
-
id: "kb:stale-source-rate",
|
|
1999
|
-
kind: "stale-source",
|
|
2000
|
-
severity: "error",
|
|
2001
|
-
message: "Stale source rate exceeds the configured maximum.",
|
|
2002
|
-
evidence: { stale_source_rate: metrics.stale_source_rate }
|
|
2003
|
-
});
|
|
2004
|
-
if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) findings.push({
|
|
2005
|
-
id: "kb:near-duplicate-analysis-truncated",
|
|
2006
|
-
kind: "unknown",
|
|
2007
|
-
severity: "error",
|
|
2008
|
-
message: "Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.",
|
|
2009
|
-
evidence: {
|
|
2010
|
-
candidate_pair_count: nearDuplicates.candidatePairCount,
|
|
2011
|
-
compared_pair_count: nearDuplicates.comparedPairCount,
|
|
2012
|
-
duplicate_pair_count: nearDuplicates.duplicatePairCount
|
|
1296
|
+
function assertCompleteOptimizationMethod(method) {
|
|
1297
|
+
const candidate = method;
|
|
1298
|
+
if (!candidate || typeof candidate.name !== "string" || candidate.name.trim().length === 0 || typeof candidate.optimize !== "function") throw new Error("knowledge optimization requires a complete OptimizationMethod with a non-empty name and optimize()");
|
|
1299
|
+
}
|
|
1300
|
+
function jsonCandidateCodec() {
|
|
1301
|
+
return {
|
|
1302
|
+
serialize: (candidate) => canonicalJson(candidate),
|
|
1303
|
+
parse(surface) {
|
|
1304
|
+
let parsed;
|
|
1305
|
+
try {
|
|
1306
|
+
parsed = JSON.parse(surface);
|
|
1307
|
+
} catch (error) {
|
|
1308
|
+
throw new Error("serialized knowledge candidate is not valid JSON", { cause: error });
|
|
1309
|
+
}
|
|
1310
|
+
return parsed;
|
|
2013
1311
|
}
|
|
2014
|
-
}
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
threshold: nearDuplicates.threshold
|
|
1312
|
+
};
|
|
1313
|
+
}
|
|
1314
|
+
function jsonObjectCandidateCodec() {
|
|
1315
|
+
const codec = jsonCandidateCodec();
|
|
1316
|
+
return {
|
|
1317
|
+
serialize: codec.serialize,
|
|
1318
|
+
parse(surface) {
|
|
1319
|
+
const candidate = codec.parse(surface);
|
|
1320
|
+
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) throw new Error("serialized knowledge candidate must be a JSON object");
|
|
1321
|
+
return candidate;
|
|
2025
1322
|
}
|
|
2026
|
-
}
|
|
1323
|
+
};
|
|
1324
|
+
}
|
|
1325
|
+
function scenarioContentFingerprint(scenario) {
|
|
1326
|
+
const { id: _id, tags: _tags, split: _split, splitTag: _splitTag, ...content } = scenario;
|
|
1327
|
+
return surfaceHash(canonicalJson(content));
|
|
1328
|
+
}
|
|
1329
|
+
function canonicalCandidateMethod(method, codec) {
|
|
2027
1330
|
return {
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
1331
|
+
name: method.name,
|
|
1332
|
+
async optimize(input) {
|
|
1333
|
+
const result = await method.optimize(input);
|
|
1334
|
+
const winner = normalizeSurface(result.winnerSurface, codec, `${method.name} winner`);
|
|
1335
|
+
return {
|
|
1336
|
+
...result,
|
|
1337
|
+
winnerSurface: winner.surface
|
|
1338
|
+
};
|
|
1339
|
+
}
|
|
2032
1340
|
};
|
|
2033
1341
|
}
|
|
2034
|
-
function
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
match = regex.exec(text);
|
|
1342
|
+
function normalizeCandidate(candidate, codec, label) {
|
|
1343
|
+
let surface;
|
|
1344
|
+
try {
|
|
1345
|
+
surface = codec.serialize(candidate);
|
|
1346
|
+
} catch (error) {
|
|
1347
|
+
throw new Error(`${label} serializer failed`, { cause: error });
|
|
2041
1348
|
}
|
|
2042
|
-
return
|
|
1349
|
+
return normalizeSurface(surface, codec, label);
|
|
2043
1350
|
}
|
|
2044
|
-
function
|
|
2045
|
-
if (
|
|
2046
|
-
|
|
1351
|
+
function normalizeSurface(surface, codec, label) {
|
|
1352
|
+
if (typeof surface !== "string" || surface.trim().length === 0) throw new Error(`${label} must be a non-empty serialized string`);
|
|
1353
|
+
let value;
|
|
1354
|
+
try {
|
|
1355
|
+
value = codec.parse(surface);
|
|
1356
|
+
} catch (error) {
|
|
1357
|
+
const detail = error instanceof Error ? `: ${error.message}` : "";
|
|
1358
|
+
throw new Error(`${label} parser failed${detail}`, { cause: error });
|
|
1359
|
+
}
|
|
1360
|
+
let canonicalSurface;
|
|
1361
|
+
try {
|
|
1362
|
+
canonicalSurface = codec.serialize(value);
|
|
1363
|
+
} catch (error) {
|
|
1364
|
+
throw new Error(`${label} serializer failed after parsing`, { cause: error });
|
|
1365
|
+
}
|
|
1366
|
+
if (typeof canonicalSurface !== "string" || canonicalSurface.trim().length === 0) throw new Error(`${label} serializer must return a non-empty string`);
|
|
1367
|
+
if (codec.serialize(codec.parse(canonicalSurface)) !== canonicalSurface) throw new Error(`${label} codec must round-trip to one canonical surface`);
|
|
1368
|
+
return {
|
|
1369
|
+
value: structuredClone(value),
|
|
1370
|
+
surface: canonicalSurface,
|
|
1371
|
+
surfaceHash: surfaceHash(canonicalSurface)
|
|
1372
|
+
};
|
|
1373
|
+
}
|
|
1374
|
+
function assertPartitionContent(train, selection, final, fingerprint) {
|
|
1375
|
+
if (!fingerprint) return;
|
|
1376
|
+
const owner = /* @__PURE__ */ new Map();
|
|
1377
|
+
for (const [name, scenarios] of [
|
|
1378
|
+
["train", train],
|
|
1379
|
+
["selection", selection],
|
|
1380
|
+
["final", final]
|
|
1381
|
+
]) for (const scenario of scenarios) {
|
|
1382
|
+
const identity = fingerprint(scenario);
|
|
1383
|
+
if (typeof identity !== "string" || identity.trim().length === 0) throw new Error(`scenarioFingerprint returned no identity for '${scenario.id}'`);
|
|
1384
|
+
const prior = owner.get(identity);
|
|
1385
|
+
if (prior) {
|
|
1386
|
+
const scope = prior.partition === name ? `${name} partition duplicates` : `${prior.partition}/${name} partitions duplicate`;
|
|
1387
|
+
throw new Error(`serialized knowledge optimization ${scope} scenario content at '${prior.scenarioId}'/'${scenario.id}'`);
|
|
1388
|
+
}
|
|
1389
|
+
owner.set(identity, {
|
|
1390
|
+
partition: name,
|
|
1391
|
+
scenarioId: scenario.id
|
|
1392
|
+
});
|
|
1393
|
+
}
|
|
2047
1394
|
}
|
|
2048
1395
|
//#endregion
|
|
2049
1396
|
//#region src/rag-optimization.ts
|
|
@@ -2411,6 +1758,231 @@ async function applyKnowledgeResearchDecision(options, decision, iteration = 1)
|
|
|
2411
1758
|
};
|
|
2412
1759
|
}
|
|
2413
1760
|
//#endregion
|
|
1761
|
+
//#region src/retrieval-eval.ts
|
|
1762
|
+
function retrievalConfigSurface(config) {
|
|
1763
|
+
return canonicalJson(config);
|
|
1764
|
+
}
|
|
1765
|
+
function retrievalConfigFromSurface(surface) {
|
|
1766
|
+
if (typeof surface !== "string") throw new Error(`retrievalConfigFromSurface expected a JSON string surface, got ${typeof surface}`);
|
|
1767
|
+
let parsed;
|
|
1768
|
+
try {
|
|
1769
|
+
parsed = JSON.parse(surface);
|
|
1770
|
+
} catch (error) {
|
|
1771
|
+
throw new Error(`retrievalConfigFromSurface could not parse JSON: ${error.message}`);
|
|
1772
|
+
}
|
|
1773
|
+
if (!isJsonObject(parsed)) throw new Error("retrievalConfigFromSurface expected a JSON object");
|
|
1774
|
+
return parsed;
|
|
1775
|
+
}
|
|
1776
|
+
function buildRetrievalEvalDispatch(options) {
|
|
1777
|
+
if (!options.retrieve && !options.index) throw new Error("buildRetrievalEvalDispatch requires either index or retrieve");
|
|
1778
|
+
const defaultK = options.defaultK ?? 5;
|
|
1779
|
+
const retrieve = options.retrieve ?? defaultSearchRetriever;
|
|
1780
|
+
return async (surface, scenario, context) => {
|
|
1781
|
+
const config = retrievalConfigFromSurface(surface);
|
|
1782
|
+
const k = retrievalK(config, scenario, defaultK);
|
|
1783
|
+
const startedAt = Date.now();
|
|
1784
|
+
const normalized = normalizeRetrieverResult(await retrieve({
|
|
1785
|
+
index: options.index,
|
|
1786
|
+
config,
|
|
1787
|
+
scenario,
|
|
1788
|
+
k,
|
|
1789
|
+
signal: context.signal,
|
|
1790
|
+
context
|
|
1791
|
+
}));
|
|
1792
|
+
return {
|
|
1793
|
+
config,
|
|
1794
|
+
query: scenario.query,
|
|
1795
|
+
requestedK: k,
|
|
1796
|
+
hits: normalized.hits,
|
|
1797
|
+
durationMs: Date.now() - startedAt,
|
|
1798
|
+
costUsd: normalized.costUsd,
|
|
1799
|
+
metadata: normalized.metadata
|
|
1800
|
+
};
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
function retrievalRecallJudge(options = {}) {
|
|
1804
|
+
const weights = normalizeWeights(options.weights ?? { recall: 1 });
|
|
1805
|
+
return {
|
|
1806
|
+
name: options.name ?? "retrieval-recall",
|
|
1807
|
+
dimensions: [
|
|
1808
|
+
{
|
|
1809
|
+
key: "recall",
|
|
1810
|
+
description: "fraction of expected knowledge targets retrieved"
|
|
1811
|
+
},
|
|
1812
|
+
{
|
|
1813
|
+
key: "mrr",
|
|
1814
|
+
description: "reciprocal rank of the first matching hit"
|
|
1815
|
+
},
|
|
1816
|
+
{
|
|
1817
|
+
key: "ndcg",
|
|
1818
|
+
description: "rank-aware gain for newly matched targets"
|
|
1819
|
+
},
|
|
1820
|
+
{
|
|
1821
|
+
key: "precision_at_k",
|
|
1822
|
+
description: "share of returned hits that match at least one target"
|
|
1823
|
+
}
|
|
1824
|
+
],
|
|
1825
|
+
appliesTo: (scenario) => scenario.kind === "retrieval-eval",
|
|
1826
|
+
async score({ artifact, scenario }) {
|
|
1827
|
+
const metrics = scoreRetrievalArtifact(artifact, scenario);
|
|
1828
|
+
const composite = (metrics.recall * weights.recall + metrics.mrr * weights.mrr + metrics.ndcg * weights.ndcg + metrics.precisionAtK * weights.precisionAtK) / (weights.recall + weights.mrr + weights.ndcg + weights.precisionAtK);
|
|
1829
|
+
return {
|
|
1830
|
+
dimensions: {
|
|
1831
|
+
recall: metrics.recall,
|
|
1832
|
+
mrr: metrics.mrr,
|
|
1833
|
+
ndcg: metrics.ndcg,
|
|
1834
|
+
precision_at_k: metrics.precisionAtK
|
|
1835
|
+
},
|
|
1836
|
+
composite,
|
|
1837
|
+
notes: `matched ${metrics.matchedCount}/${metrics.expectedCount}; first_hit_rank=${metrics.firstHitRank ?? "none"}`
|
|
1838
|
+
};
|
|
1839
|
+
}
|
|
1840
|
+
};
|
|
1841
|
+
}
|
|
1842
|
+
function scoreRetrievalArtifact(artifact, scenario) {
|
|
1843
|
+
const targets = normalizeTargets(scenario.expected);
|
|
1844
|
+
if (targets.length === 0) throw new Error(`retrieval eval scenario ${scenario.id} has no expected targets`);
|
|
1845
|
+
const matchedTargetIds = /* @__PURE__ */ new Set();
|
|
1846
|
+
let firstHitRank = null;
|
|
1847
|
+
let relevantHitCount = 0;
|
|
1848
|
+
let dcg = 0;
|
|
1849
|
+
for (const hit of artifact.hits) {
|
|
1850
|
+
const newlyMatched = targets.filter((target) => {
|
|
1851
|
+
const targetId = retrievalTargetId(target);
|
|
1852
|
+
return !matchedTargetIds.has(targetId) && hitMatchesTarget(hit, target);
|
|
1853
|
+
});
|
|
1854
|
+
if (newlyMatched.length === 0) continue;
|
|
1855
|
+
relevantHitCount += 1;
|
|
1856
|
+
if (firstHitRank === null) firstHitRank = hit.rank;
|
|
1857
|
+
dcg += 1 / Math.log2(hit.rank + 1);
|
|
1858
|
+
for (const target of newlyMatched) matchedTargetIds.add(retrievalTargetId(target));
|
|
1859
|
+
}
|
|
1860
|
+
const expectedCount = targets.length;
|
|
1861
|
+
const matchedCount = matchedTargetIds.size;
|
|
1862
|
+
const idcg = idealDcg(Math.min(expectedCount, Math.max(1, artifact.requestedK)));
|
|
1863
|
+
return {
|
|
1864
|
+
recall: matchedCount / expectedCount,
|
|
1865
|
+
mrr: firstHitRank === null ? 0 : 1 / firstHitRank,
|
|
1866
|
+
ndcg: idcg === 0 ? 0 : dcg / idcg,
|
|
1867
|
+
precisionAtK: artifact.hits.length === 0 ? 0 : relevantHitCount / artifact.hits.length,
|
|
1868
|
+
expectedCount,
|
|
1869
|
+
matchedCount,
|
|
1870
|
+
relevantHitCount,
|
|
1871
|
+
firstHitRank,
|
|
1872
|
+
matchedTargetIds: [...matchedTargetIds].sort()
|
|
1873
|
+
};
|
|
1874
|
+
}
|
|
1875
|
+
function partitionRetrievalScenarios(scenarios, options = {}) {
|
|
1876
|
+
if (scenarios.length < 4) throw new Error("partitionRetrievalScenarios requires at least 4 scenarios for non-empty train and selection partitions plus 2 final scenarios");
|
|
1877
|
+
const selectionFraction = options.selectionFraction ?? .2;
|
|
1878
|
+
const finalFraction = options.finalFraction ?? .2;
|
|
1879
|
+
assertPartitionFraction(selectionFraction, "selectionFraction");
|
|
1880
|
+
assertPartitionFraction(finalFraction, "finalFraction");
|
|
1881
|
+
if (selectionFraction + finalFraction >= 1) throw new Error("selectionFraction + finalFraction must be less than 1");
|
|
1882
|
+
const shuffled = seededShuffle(scenarios, options.seed ?? 42);
|
|
1883
|
+
const finalCount = Math.max(2, Math.round(shuffled.length * finalFraction));
|
|
1884
|
+
const selectionCount = Math.max(1, Math.round(shuffled.length * selectionFraction));
|
|
1885
|
+
if (finalCount + selectionCount >= shuffled.length) throw new Error("retrieval scenario fractions leave no training scenarios");
|
|
1886
|
+
const trainEnd = shuffled.length - selectionCount - finalCount;
|
|
1887
|
+
return {
|
|
1888
|
+
trainScenarios: shuffled.slice(0, trainEnd),
|
|
1889
|
+
selectionScenarios: shuffled.slice(trainEnd, trainEnd + selectionCount),
|
|
1890
|
+
finalScenarios: shuffled.slice(trainEnd + selectionCount)
|
|
1891
|
+
};
|
|
1892
|
+
}
|
|
1893
|
+
function assertPartitionFraction(value, name) {
|
|
1894
|
+
if (!Number.isFinite(value) || value <= 0 || value >= 1) throw new Error(`${name} must be greater than 0 and less than 1`);
|
|
1895
|
+
}
|
|
1896
|
+
function retrievalK(config, scenario, defaultK) {
|
|
1897
|
+
const rawK = config.k ?? config.topK ?? scenario.k ?? defaultK;
|
|
1898
|
+
if (typeof rawK !== "number" || !Number.isFinite(rawK) || rawK <= 0) throw new Error(`retrieval k must be a positive number, got ${String(rawK)}`);
|
|
1899
|
+
return Math.floor(rawK);
|
|
1900
|
+
}
|
|
1901
|
+
async function defaultSearchRetriever(input) {
|
|
1902
|
+
if (!input.index) throw new Error("default retrieval eval search requires an index");
|
|
1903
|
+
return { hits: searchKnowledge(input.index, input.scenario.query, input.k).map(hitFromSearchResult$1) };
|
|
1904
|
+
}
|
|
1905
|
+
function hitFromSearchResult$1(result) {
|
|
1906
|
+
return {
|
|
1907
|
+
pageId: result.page.id,
|
|
1908
|
+
path: result.page.path,
|
|
1909
|
+
title: result.page.title,
|
|
1910
|
+
rank: result.rank,
|
|
1911
|
+
score: result.score,
|
|
1912
|
+
normalizedScore: result.normalizedScore,
|
|
1913
|
+
sourceIds: result.page.sourceIds,
|
|
1914
|
+
snippet: result.snippet
|
|
1915
|
+
};
|
|
1916
|
+
}
|
|
1917
|
+
function normalizeRetrieverResult(result) {
|
|
1918
|
+
if (isHitArray(result)) return { hits: result };
|
|
1919
|
+
return result;
|
|
1920
|
+
}
|
|
1921
|
+
function normalizeTargets(expected) {
|
|
1922
|
+
return isTargetArray(expected) ? [...expected] : [expected];
|
|
1923
|
+
}
|
|
1924
|
+
function isHitArray(result) {
|
|
1925
|
+
return Array.isArray(result);
|
|
1926
|
+
}
|
|
1927
|
+
function isTargetArray(expected) {
|
|
1928
|
+
return Array.isArray(expected);
|
|
1929
|
+
}
|
|
1930
|
+
function hitMatchesTarget(hit, target) {
|
|
1931
|
+
switch (target.kind) {
|
|
1932
|
+
case "page": return hit.pageId === target.pageId;
|
|
1933
|
+
case "page-path": return hit.path === target.path;
|
|
1934
|
+
case "source": return Boolean(hit.sourceIds?.includes(target.sourceId));
|
|
1935
|
+
case "source-anchor": return Boolean(hit.sourceSpans?.some((span) => span.sourceId === target.sourceId && span.anchorId === target.anchorId));
|
|
1936
|
+
case "source-span": return Boolean(hit.sourceSpans?.some((span) => span.sourceId === target.sourceId && spanOverlaps(span, target)));
|
|
1937
|
+
}
|
|
1938
|
+
}
|
|
1939
|
+
function spanOverlaps(hit, target) {
|
|
1940
|
+
if (typeof hit.charStart !== "number" || typeof hit.charEnd !== "number") return false;
|
|
1941
|
+
return hit.charStart < target.charEnd && target.charStart < hit.charEnd;
|
|
1942
|
+
}
|
|
1943
|
+
function retrievalTargetId(target) {
|
|
1944
|
+
switch (target.kind) {
|
|
1945
|
+
case "page": return `page:${target.pageId}`;
|
|
1946
|
+
case "page-path": return `page-path:${target.path}`;
|
|
1947
|
+
case "source": return `source:${target.sourceId}`;
|
|
1948
|
+
case "source-anchor": return `source-anchor:${target.sourceId}:${target.anchorId}`;
|
|
1949
|
+
case "source-span": return `source-span:${target.sourceId}:${target.charStart}:${target.charEnd}`;
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
function idealDcg(count) {
|
|
1953
|
+
let score = 0;
|
|
1954
|
+
for (let i = 1; i <= count; i += 1) score += 1 / Math.log2(i + 1);
|
|
1955
|
+
return score;
|
|
1956
|
+
}
|
|
1957
|
+
function normalizeWeights(weights) {
|
|
1958
|
+
const normalized = {
|
|
1959
|
+
recall: normalizeWeight(weights.recall),
|
|
1960
|
+
mrr: normalizeWeight(weights.mrr),
|
|
1961
|
+
ndcg: normalizeWeight(weights.ndcg),
|
|
1962
|
+
precisionAtK: normalizeWeight(weights.precisionAtK)
|
|
1963
|
+
};
|
|
1964
|
+
if (normalized.recall + normalized.mrr + normalized.ndcg + normalized.precisionAtK <= 0) throw new Error("retrievalRecallJudge requires at least one positive metric weight");
|
|
1965
|
+
return normalized;
|
|
1966
|
+
}
|
|
1967
|
+
function normalizeWeight(value) {
|
|
1968
|
+
if (value === void 0) return 0;
|
|
1969
|
+
if (!Number.isFinite(value) || value < 0) throw new Error(`retrievalRecallJudge metric weights must be non-negative finite numbers, got ${String(value)}`);
|
|
1970
|
+
return value;
|
|
1971
|
+
}
|
|
1972
|
+
function isJsonObject(value) {
|
|
1973
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
1974
|
+
}
|
|
1975
|
+
function seededShuffle(items, seed) {
|
|
1976
|
+
const out = [...items];
|
|
1977
|
+
let state = seed >>> 0;
|
|
1978
|
+
for (let i = out.length - 1; i > 0; i -= 1) {
|
|
1979
|
+
state = state * 1664525 + 1013904223 >>> 0;
|
|
1980
|
+
const j = state % (i + 1);
|
|
1981
|
+
[out[i], out[j]] = [out[j], out[i]];
|
|
1982
|
+
}
|
|
1983
|
+
return out;
|
|
1984
|
+
}
|
|
1985
|
+
//#endregion
|
|
2414
1986
|
//#region src/retrieval-optimization.ts
|
|
2415
1987
|
async function runRetrievalImprovementLoop(options) {
|
|
2416
1988
|
const dispatch = buildRetrievalEvalDispatch({
|
|
@@ -3814,446 +3386,35 @@ async function improveKnowledgeBaseInRun(options, runId, runDir, now) {
|
|
|
3814
3386
|
}
|
|
3815
3387
|
}
|
|
3816
3388
|
//#endregion
|
|
3817
|
-
//#region src/
|
|
3818
|
-
/**
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
*/
|
|
3822
|
-
async function optimizeKnowledgeBasePolicy(options) {
|
|
3823
|
-
const { root, goal, baselinePolicy, method, trainScenarios, selectionScenarios, finalScenarios, policyApplicationRef, candidateRunLabel, candidate, applyPolicy, ...optimizationOptions } = options;
|
|
3824
|
-
if (typeof root !== "string" || !root.trim()) throw new Error("optimizeKnowledgeBasePolicy root must be non-empty");
|
|
3825
|
-
if (typeof goal !== "string" || !goal.trim()) throw new Error("optimizeKnowledgeBasePolicy goal must be non-empty");
|
|
3826
|
-
assertImmutableRef(policyApplicationRef, "optimizeKnowledgeBasePolicy policyApplicationRef");
|
|
3827
|
-
if (candidateRunLabel !== void 0 && (typeof candidateRunLabel !== "string" || !candidateRunLabel.trim())) throw new Error("optimizeKnowledgeBasePolicy candidateRunLabel must be non-empty");
|
|
3828
|
-
const baseHash = await hashKnowledgeBase(root, candidate?.stateScope);
|
|
3829
|
-
const optimization = await runSerializedKnowledgeOptimization({
|
|
3830
|
-
...optimizationOptions,
|
|
3831
|
-
executionRef: policyApplicationRef,
|
|
3832
|
-
baseline: baselinePolicy,
|
|
3833
|
-
method,
|
|
3834
|
-
trainScenarios,
|
|
3835
|
-
selectionScenarios,
|
|
3836
|
-
finalScenarios
|
|
3837
|
-
});
|
|
3838
|
-
const winner = optimization.winner;
|
|
3839
|
-
const currentBaseHash = await hashKnowledgeBase(root, candidate?.stateScope);
|
|
3840
|
-
if (currentBaseHash !== baseHash) throw new Error(`knowledge base changed during policy optimization: expected ${baseHash}, got ${currentBaseHash}`);
|
|
3841
|
-
const runId = stableId("kbpolicy", `${candidateRunLabel ?? "default"}:${goal}:${optimization.methodName}:${winner.surfaceHash}:${policyApplicationRef}:${baseHash}`);
|
|
3389
|
+
//#region src/agent-candidate.ts
|
|
3390
|
+
/** Convert a measured knowledge candidate into the shared review and execution identity. */
|
|
3391
|
+
function toAgentCandidateKnowledgeRef(candidate) {
|
|
3392
|
+
const parsed = KnowledgeImprovementCandidateRefSchema.parse(candidate);
|
|
3842
3393
|
return {
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
runId,
|
|
3850
|
-
maxCandidates: 1,
|
|
3851
|
-
updateKnowledge: async (input) => {
|
|
3852
|
-
if (input.baseHash !== baseHash) throw new Error(`knowledge base changed before policy materialization: expected ${baseHash}, got ${input.baseHash}`);
|
|
3853
|
-
const result = await applyPolicy({
|
|
3854
|
-
...input,
|
|
3855
|
-
policy: structuredClone(winner.value),
|
|
3856
|
-
policySurface: winner.surface,
|
|
3857
|
-
policySurfaceHash: winner.surfaceHash,
|
|
3858
|
-
optimizationMethod: optimization.methodName
|
|
3859
|
-
});
|
|
3860
|
-
return {
|
|
3861
|
-
...result,
|
|
3862
|
-
metadata: {
|
|
3863
|
-
...result.metadata ?? {},
|
|
3864
|
-
optimization: {
|
|
3865
|
-
method: optimization.methodName,
|
|
3866
|
-
policySurfaceHash: winner.surfaceHash,
|
|
3867
|
-
policyApplicationRef
|
|
3868
|
-
}
|
|
3869
|
-
}
|
|
3870
|
-
};
|
|
3871
|
-
}
|
|
3872
|
-
})
|
|
3394
|
+
...parsed,
|
|
3395
|
+
goalHash: prefixedDigest(parsed.goalHash),
|
|
3396
|
+
baseHash: prefixedDigest(parsed.baseHash),
|
|
3397
|
+
candidateHash: prefixedDigest(parsed.candidateHash),
|
|
3398
|
+
evidenceHash: prefixedDigest(parsed.evidenceHash),
|
|
3399
|
+
promotionPlanHash: prefixedDigest(parsed.promotionPlanHash)
|
|
3873
3400
|
};
|
|
3874
|
-
}
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
}
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
});
|
|
3892
|
-
const selectionMetadataSchema = z.record(z.string(), z.json());
|
|
3893
|
-
const measuredSelectionLifecycleSchema = z.object({
|
|
3894
|
-
kind: z.literal("measured-knowledge-change-selection"),
|
|
3895
|
-
version: z.literal(1),
|
|
3896
|
-
selectionDigest: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3897
|
-
sourceCandidateId: z.string().min(1),
|
|
3898
|
-
sourceEvidenceHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3899
|
-
sourcePlanHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3900
|
-
selectedPaths: z.array(selectionPathSchema),
|
|
3901
|
-
selectedCandidateHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3902
|
-
selectedPlanHash: z.string().regex(/^[a-f0-9]{64}$/)
|
|
3903
|
-
}).strict();
|
|
3904
|
-
const measuredSelectionReceiptSchema = z.object({
|
|
3905
|
-
kind: z.literal("measured-knowledge-change-selection-receipt"),
|
|
3906
|
-
version: z.literal(1),
|
|
3907
|
-
receiptHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3908
|
-
selectionDigest: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3909
|
-
sourceCandidate: KnowledgeImprovementCandidateRefSchema,
|
|
3910
|
-
sourcePlanHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3911
|
-
selectedPaths: z.array(selectionPathSchema),
|
|
3912
|
-
derivedImplementationRef: immutableRefSchema,
|
|
3913
|
-
runId: z.string().min(1),
|
|
3914
|
-
derivedCandidateId: z.string().min(1),
|
|
3915
|
-
derivedCandidateStatus: z.string().min(1),
|
|
3916
|
-
selectedCandidateHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3917
|
-
selectedPlanHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3918
|
-
selectedEvidenceHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
3919
|
-
rationale: z.string().trim().min(1).optional(),
|
|
3920
|
-
metadata: selectionMetadataSchema.optional()
|
|
3921
|
-
}).strict();
|
|
3922
|
-
/**
|
|
3923
|
-
* Derive a subset from an already measured candidate, then measure the subset as
|
|
3924
|
-
* a new candidate before it can be promoted.
|
|
3925
|
-
*
|
|
3926
|
-
* Directly filtering an activation plan is unsafe: a whole candidate can pass
|
|
3927
|
-
* while one page subset breaks links, removes supporting sources, or changes a
|
|
3928
|
-
* readiness score. This helper instead applies the chosen changed paths to an
|
|
3929
|
-
* isolated baseline, recomputes the index, runs the ordinary improvement
|
|
3930
|
-
* evaluator, and returns the ordinary candidate reference. Promotion therefore
|
|
3931
|
-
* remains one path and can never admit an unmeasured hybrid.
|
|
3932
|
-
*/
|
|
3933
|
-
async function improveSelectedKnowledgeCandidate(options) {
|
|
3934
|
-
const sourceCandidate = Object.freeze(KnowledgeImprovementCandidateRefSchema.parse(options.sourceCandidate));
|
|
3935
|
-
const requestedImplementationRef = immutableRefSchema.parse(options.implementationRef);
|
|
3936
|
-
const enabledEvaluationPhases = normalizeEvaluationPhases(options.enabledEvaluationPhases ?? ["gap-diagnosis", ...EVALUATION_PHASES], "enabledEvaluationPhases");
|
|
3937
|
-
const requiredEvaluationPhases = normalizeEvaluationPhases(options.requiredEvaluationPhases ?? [], "requiredEvaluationPhases");
|
|
3938
|
-
const metadata = options.selectionMetadata ? selectionMetadataSchema.parse(structuredClone(options.selectionMetadata)) : void 0;
|
|
3939
|
-
return withKnowledgeImprovementComparison({
|
|
3940
|
-
root: options.root,
|
|
3941
|
-
candidate: sourceCandidate
|
|
3942
|
-
}, async (source) => {
|
|
3943
|
-
const scope = normalizeKnowledgeStateScope(source.stateScope);
|
|
3944
|
-
if (options.stateScope !== void 0 && canonicalJson(normalizeKnowledgeStateScope(options.stateScope)) !== canonicalJson(scope)) throw new Error("selected knowledge candidate stateScope differs from its source");
|
|
3945
|
-
const sourcePlan = await knowledgeFilePlanEntries(source.baseline.root, source.candidate.root, scope);
|
|
3946
|
-
const sourcePlanHash = knowledgeFileTransactionPlanHash(sourcePlan, scope.pagesDirectory, scope.researchState);
|
|
3947
|
-
if (sourcePlanHash !== sourceCandidate.promotionPlanHash) throw new Error("source knowledge candidate plan no longer matches its measured identity");
|
|
3948
|
-
const changedSourcePlan = sourcePlan.filter(planEntryChanged).filter((entry) => notDerivedPath(entry, scope));
|
|
3949
|
-
const selectedPaths = normalizeSelectedPaths(options.selectedPaths, changedSourcePlan, scope);
|
|
3950
|
-
const selectionMaterial = immutableJson({
|
|
3951
|
-
kind: "measured-knowledge-change-selection",
|
|
3952
|
-
version: 1,
|
|
3953
|
-
sourceCandidate,
|
|
3954
|
-
sourcePlanHash,
|
|
3955
|
-
selectedPaths,
|
|
3956
|
-
...options.rationale?.trim() ? { rationale: options.rationale.trim() } : {},
|
|
3957
|
-
...metadata ? { metadata } : {}
|
|
3958
|
-
});
|
|
3959
|
-
const selectionDigest = contentHash(selectionMaterial);
|
|
3960
|
-
const derivedImplementationRef = immutableRefSchema.parse(`sha256:${contentHash({
|
|
3961
|
-
engine: "agent-knowledge/measured-selection-v1",
|
|
3962
|
-
implementationRef: requestedImplementationRef,
|
|
3963
|
-
selectionDigest
|
|
3964
|
-
})}`);
|
|
3965
|
-
const runId = options.runId ?? stableId("kimpsel", canonicalJson({
|
|
3966
|
-
root: options.root,
|
|
3967
|
-
goal: options.goal,
|
|
3968
|
-
derivedImplementationRef,
|
|
3969
|
-
selectionDigest
|
|
3970
|
-
}));
|
|
3971
|
-
const selectedEntries = selectedPaths.map((path) => {
|
|
3972
|
-
const entry = changedSourcePlan.find((candidate) => candidate.path === path);
|
|
3973
|
-
if (!entry) throw new Error(`selected knowledge path disappeared: ${path}`);
|
|
3974
|
-
return entry;
|
|
3975
|
-
});
|
|
3976
|
-
const selectionMutationPlanHash = knowledgeFileTransactionPlanHash(selectedEntries, scope.pagesDirectory, scope.researchState);
|
|
3977
|
-
let lifecycleSelection;
|
|
3978
|
-
const result = await improveKnowledgeBase({
|
|
3979
|
-
...improvementOptions(options),
|
|
3980
|
-
root: options.root,
|
|
3981
|
-
stateScope: scope,
|
|
3982
|
-
goal: options.goal,
|
|
3983
|
-
implementationRef: derivedImplementationRef,
|
|
3984
|
-
runId,
|
|
3985
|
-
enabledPhases: ["knowledge-update", ...enabledEvaluationPhases],
|
|
3986
|
-
requiredPhases: ["knowledge-update", ...requiredEvaluationPhases],
|
|
3987
|
-
async updateKnowledge(input) {
|
|
3988
|
-
const purpose = `knowledge-selected-candidate:${selectionDigest}`;
|
|
3989
|
-
const recoveryOwner = `knowledge-selected-candidate:${sourceCandidate.candidateId}`;
|
|
3990
|
-
await withKnowledgeMutation(input.candidateRoot, async (lock) => {
|
|
3991
|
-
lock.assertOwned();
|
|
3992
|
-
if (!lock.recovery && selectedEntries.length > 0) {
|
|
3993
|
-
const transaction = await prepareKnowledgeFileTransaction({
|
|
3994
|
-
root: input.candidateRoot,
|
|
3995
|
-
transactionRoot: lock.transactionRoot,
|
|
3996
|
-
purpose,
|
|
3997
|
-
...scope,
|
|
3998
|
-
recoveryOwner,
|
|
3999
|
-
mutations: await selectionMutations(source.candidate.root, selectedEntries),
|
|
4000
|
-
now: options.now
|
|
4001
|
-
});
|
|
4002
|
-
if (transaction) {
|
|
4003
|
-
assertSelectionTransaction(transaction, selectionMutationPlanHash);
|
|
4004
|
-
await applyKnowledgeFileTransaction({
|
|
4005
|
-
root: input.candidateRoot,
|
|
4006
|
-
transactionRoot: lock.transactionRoot,
|
|
4007
|
-
transaction,
|
|
4008
|
-
beforeCommit: lock.assertOwned
|
|
4009
|
-
});
|
|
4010
|
-
await finishKnowledgeFileTransaction({
|
|
4011
|
-
root: input.candidateRoot,
|
|
4012
|
-
transactionRoot: lock.transactionRoot,
|
|
4013
|
-
transaction,
|
|
4014
|
-
assertOwned: lock.assertOwned
|
|
4015
|
-
});
|
|
4016
|
-
}
|
|
4017
|
-
}
|
|
4018
|
-
await writeKnowledgeIndex(input.candidateRoot, scope);
|
|
4019
|
-
lock.assertOwned();
|
|
4020
|
-
}, { resumeTransaction: {
|
|
4021
|
-
purpose,
|
|
4022
|
-
recoveryOwner,
|
|
4023
|
-
validate: (transaction) => assertSelectionTransaction(transaction, selectionMutationPlanHash)
|
|
4024
|
-
} });
|
|
4025
|
-
const selectedPlan = await knowledgeFilePlanEntries(source.baseline.root, input.candidateRoot, scope);
|
|
4026
|
-
assertExactSelectedChanges(selectedPlan, selectedPaths, scope);
|
|
4027
|
-
const selectedPlanHash = knowledgeFileTransactionPlanHash(selectedPlan, scope.pagesDirectory, scope.researchState);
|
|
4028
|
-
const selectedCandidateHash = await hashKnowledgeBase(input.candidateRoot, scope);
|
|
4029
|
-
lifecycleSelection = measuredSelectionLifecycleSchema.parse({
|
|
4030
|
-
kind: "measured-knowledge-change-selection",
|
|
4031
|
-
version: 1,
|
|
4032
|
-
selectionDigest,
|
|
4033
|
-
sourceCandidateId: sourceCandidate.candidateId,
|
|
4034
|
-
sourceEvidenceHash: sourceCandidate.evidenceHash,
|
|
4035
|
-
sourcePlanHash,
|
|
4036
|
-
selectedPaths,
|
|
4037
|
-
selectedCandidateHash,
|
|
4038
|
-
selectedPlanHash
|
|
4039
|
-
});
|
|
4040
|
-
return {
|
|
4041
|
-
applied: selectedPaths.length > 0,
|
|
4042
|
-
summary: selectedPaths.length > 0 ? `Applied ${selectedPaths.length} selected measured change(s).` : "Selected no changes; measuring the exact baseline as a null candidate.",
|
|
4043
|
-
metadata: { selection: lifecycleSelection }
|
|
4044
|
-
};
|
|
4045
|
-
}
|
|
4046
|
-
});
|
|
4047
|
-
const candidate = result.candidate;
|
|
4048
|
-
if (!candidate?.candidateHash || !candidate.promotionPlanHash || !candidate.evidenceHash) throw new Error("selected knowledge candidate did not produce measured candidate evidence");
|
|
4049
|
-
const recordedSelection = measuredSelectionLifecycleSchema.parse(result.lifecycle?.knowledgeUpdate?.metadata?.selection ?? lifecycleSelection);
|
|
4050
|
-
if (recordedSelection.selectionDigest !== selectionDigest || recordedSelection.selectedCandidateHash !== candidate.candidateHash || recordedSelection.selectedPlanHash !== candidate.promotionPlanHash || canonicalJson(recordedSelection.selectedPaths) !== canonicalJson(selectedPaths)) throw new Error("selected knowledge candidate evidence does not bind the measured subset");
|
|
4051
|
-
const receiptWithoutHash = immutableJson({
|
|
4052
|
-
kind: "measured-knowledge-change-selection-receipt",
|
|
4053
|
-
version: 1,
|
|
4054
|
-
selectionDigest,
|
|
4055
|
-
sourceCandidate,
|
|
4056
|
-
sourcePlanHash,
|
|
4057
|
-
selectedPaths,
|
|
4058
|
-
derivedImplementationRef,
|
|
4059
|
-
runId: result.runId,
|
|
4060
|
-
derivedCandidateId: candidate.candidateId,
|
|
4061
|
-
derivedCandidateStatus: candidate.status,
|
|
4062
|
-
selectedCandidateHash: candidate.candidateHash,
|
|
4063
|
-
selectedPlanHash: candidate.promotionPlanHash,
|
|
4064
|
-
selectedEvidenceHash: candidate.evidenceHash,
|
|
4065
|
-
...options.rationale?.trim() ? { rationale: options.rationale.trim() } : {},
|
|
4066
|
-
...metadata ? { metadata } : {}
|
|
4067
|
-
});
|
|
4068
|
-
const receipt = measuredSelectionReceiptSchema.parse({
|
|
4069
|
-
...receiptWithoutHash,
|
|
4070
|
-
receiptHash: contentHash(receiptWithoutHash)
|
|
4071
|
-
});
|
|
4072
|
-
await persistSelectionReceipt(options.root, receipt);
|
|
4073
|
-
return {
|
|
4074
|
-
...result,
|
|
4075
|
-
selection: receipt
|
|
4076
|
-
};
|
|
4077
|
-
});
|
|
4078
|
-
}
|
|
4079
|
-
function improvementOptions(options) {
|
|
4080
|
-
const { root: _root, goal: _goal, implementationRef: _implementationRef, sourceCandidate: _sourceCandidate, selectedPaths: _selectedPaths, runId: _runId, rationale: _rationale, selectionMetadata: _selectionMetadata, enabledEvaluationPhases: _enabledEvaluationPhases, requiredEvaluationPhases: _requiredEvaluationPhases, ...rest } = options;
|
|
4081
|
-
return rest;
|
|
4082
|
-
}
|
|
4083
|
-
function normalizeEvaluationPhases(phases, field) {
|
|
4084
|
-
const unique = [...new Set(phases)];
|
|
4085
|
-
for (const phase of unique) if (phase === "knowledge-acquisition" || phase === "knowledge-update") throw new Error(`${field} cannot contain the selection-owned phase '${phase}'`);
|
|
4086
|
-
return unique;
|
|
4087
|
-
}
|
|
4088
|
-
function normalizeSelectedPaths(paths, changedPlan, scope) {
|
|
4089
|
-
const available = new Set(changedPlan.map((entry) => entry.path));
|
|
4090
|
-
const selected = [];
|
|
4091
|
-
const seen = /* @__PURE__ */ new Set();
|
|
4092
|
-
for (const input of paths) {
|
|
4093
|
-
const path = assertKnowledgeMutationPath(selectionPathSchema.parse(input), normalizeKnowledgeStateScope(scope).pagesDirectory, scope.researchState);
|
|
4094
|
-
if (!notDerivedPath({ path }, scope)) throw new Error(`derived knowledge path cannot be selected directly: ${path}`);
|
|
4095
|
-
if (seen.has(path)) throw new Error(`selected knowledge path is repeated: ${path}`);
|
|
4096
|
-
if (!available.has(path)) throw new Error(`selected knowledge path is not a changed source-candidate file: ${path}`);
|
|
4097
|
-
seen.add(path);
|
|
4098
|
-
selected.push(path);
|
|
4099
|
-
}
|
|
4100
|
-
return selected.sort((left, right) => left.localeCompare(right));
|
|
4101
|
-
}
|
|
4102
|
-
function planEntryChanged(entry) {
|
|
4103
|
-
return entry.beforeHash !== entry.afterHash || entry.beforeHash !== null && entry.afterHash !== null && entry.beforeMode !== entry.afterMode;
|
|
4104
|
-
}
|
|
4105
|
-
function notDerivedPath(entry, scope) {
|
|
4106
|
-
return entry.path !== `${normalizeKnowledgeStateScope(scope).pagesDirectory}/index.md`;
|
|
4107
|
-
}
|
|
4108
|
-
async function selectionMutations(sourceCandidateRoot, entries) {
|
|
4109
|
-
return Promise.all(entries.map(async (entry) => {
|
|
4110
|
-
if (entry.afterHash === null) return {
|
|
4111
|
-
path: entry.path,
|
|
4112
|
-
content: null
|
|
4113
|
-
};
|
|
4114
|
-
const file = await readRegularFileWithinRoot(sourceCandidateRoot, entry.path);
|
|
4115
|
-
if (createHash("sha256").update(file.bytes).digest("hex") !== entry.afterHash || file.mode !== entry.afterMode) throw new Error(`source candidate changed before subset materialization: ${entry.path}`);
|
|
4116
|
-
return {
|
|
4117
|
-
path: entry.path,
|
|
4118
|
-
content: file.bytes,
|
|
4119
|
-
mode: file.mode
|
|
4120
|
-
};
|
|
4121
|
-
}));
|
|
4122
|
-
}
|
|
4123
|
-
function assertSelectionTransaction(transaction, expectedPlanHash) {
|
|
4124
|
-
if (knowledgeFileTransactionPlanHash(transaction.entries, normalizePagesDirectory(transaction.pagesDirectory), transaction.researchState) !== expectedPlanHash) throw new Error("selected knowledge transaction does not match its approved path set");
|
|
4125
|
-
}
|
|
4126
|
-
function assertExactSelectedChanges(plan, selectedPaths, scope) {
|
|
4127
|
-
const actual = plan.filter(planEntryChanged).filter((entry) => notDerivedPath(entry, scope)).map((entry) => entry.path).sort((left, right) => left.localeCompare(right));
|
|
4128
|
-
if (canonicalJson(actual) !== canonicalJson(selectedPaths)) throw new Error(`selected knowledge candidate changed the wrong files: expected ${selectedPaths.join(", ") || "(none)"}, got ${actual.join(", ") || "(none)"}`);
|
|
4129
|
-
}
|
|
4130
|
-
async function persistSelectionReceipt(root, receipt) {
|
|
4131
|
-
const runDir = knowledgeImprovementRunDir(root, receipt.runId);
|
|
4132
|
-
const relativePath = join("candidates", safePathSegmentSchema.parse(receipt.derivedCandidateId), "selection.json").replace(/\\/g, "/");
|
|
4133
|
-
try {
|
|
4134
|
-
const existing = measuredSelectionReceiptSchema.parse(JSON.parse((await readRegularFileWithinRoot(runDir, relativePath)).bytes.toString("utf8")));
|
|
4135
|
-
if (canonicalJson(existing) !== canonicalJson(receipt)) throw new Error("measured knowledge selection receipt conflicts with durable content");
|
|
4136
|
-
return;
|
|
4137
|
-
} catch (error) {
|
|
4138
|
-
if (!isMissingFile(error)) throw error;
|
|
4139
|
-
}
|
|
4140
|
-
await writeJsonDurableWithinRoot(runDir, relativePath, receipt);
|
|
4141
|
-
}
|
|
4142
|
-
function immutableJson(value) {
|
|
4143
|
-
if (value === null || typeof value !== "object") return value;
|
|
4144
|
-
const children = Array.isArray(value) ? value : Object.values(value);
|
|
4145
|
-
for (const child of children) immutableJson(child);
|
|
4146
|
-
return Object.freeze(value);
|
|
4147
|
-
}
|
|
4148
|
-
//#endregion
|
|
4149
|
-
//#region src/agent-candidate.ts
|
|
4150
|
-
/** Convert a measured knowledge candidate into the shared review and execution identity. */
|
|
4151
|
-
function toAgentCandidateKnowledgeRef(candidate) {
|
|
4152
|
-
const parsed = KnowledgeImprovementCandidateRefSchema.parse(candidate);
|
|
4153
|
-
return {
|
|
4154
|
-
...parsed,
|
|
4155
|
-
goalHash: prefixedDigest(parsed.goalHash),
|
|
4156
|
-
baseHash: prefixedDigest(parsed.baseHash),
|
|
4157
|
-
candidateHash: prefixedDigest(parsed.candidateHash),
|
|
4158
|
-
evidenceHash: prefixedDigest(parsed.evidenceHash),
|
|
4159
|
-
promotionPlanHash: prefixedDigest(parsed.promotionPlanHash)
|
|
4160
|
-
};
|
|
4161
|
-
}
|
|
4162
|
-
/** Recover agent-knowledge's candidate identity from the shared contract. */
|
|
4163
|
-
function fromAgentCandidateKnowledgeRef(candidate) {
|
|
4164
|
-
return KnowledgeImprovementCandidateRefSchema.parse({
|
|
4165
|
-
...candidate,
|
|
4166
|
-
goalHash: rawDigest(candidate.goalHash),
|
|
4167
|
-
baseHash: rawDigest(candidate.baseHash),
|
|
4168
|
-
candidateHash: rawDigest(candidate.candidateHash),
|
|
4169
|
-
evidenceHash: rawDigest(candidate.evidenceHash),
|
|
4170
|
-
promotionPlanHash: rawDigest(candidate.promotionPlanHash)
|
|
4171
|
-
});
|
|
4172
|
-
}
|
|
4173
|
-
function prefixedDigest(value) {
|
|
4174
|
-
return sha256DigestSchema.parse(`sha256:${value}`);
|
|
4175
|
-
}
|
|
4176
|
-
function rawDigest(value) {
|
|
4177
|
-
return sha256DigestSchema.parse(value).slice(7);
|
|
4178
|
-
}
|
|
4179
|
-
//#endregion
|
|
4180
|
-
//#region src/changes.ts
|
|
4181
|
-
function detectChanges(prev, next, options = {}) {
|
|
4182
|
-
const skipUnverifiable = options.skipUnverifiable ?? true;
|
|
4183
|
-
const warnings = [];
|
|
4184
|
-
const { map: prevMap, warnings: prevWarn } = indexFragments(prev, skipUnverifiable, "prev");
|
|
4185
|
-
const { map: nextMap, warnings: nextWarn } = indexFragments(next, skipUnverifiable, "next");
|
|
4186
|
-
warnings.push(...prevWarn, ...nextWarn);
|
|
4187
|
-
const changes = [];
|
|
4188
|
-
const seen = /* @__PURE__ */ new Set();
|
|
4189
|
-
for (const [id, nextFragment] of nextMap) {
|
|
4190
|
-
seen.add(id);
|
|
4191
|
-
const prevFragment = prevMap.get(id);
|
|
4192
|
-
if (!prevFragment) {
|
|
4193
|
-
changes.push({
|
|
4194
|
-
fragmentId: id,
|
|
4195
|
-
kind: "added",
|
|
4196
|
-
diff: { after: nextFragment.body },
|
|
4197
|
-
affectedDimensions: dedup(nextFragment.dimensionHints),
|
|
4198
|
-
url: nextFragment.provenance.url,
|
|
4199
|
-
detectedAt: nextFragment.provenance.sourceUpdatedAt
|
|
4200
|
-
});
|
|
4201
|
-
continue;
|
|
4202
|
-
}
|
|
4203
|
-
if (prevFragment.bodyHash !== nextFragment.bodyHash) changes.push({
|
|
4204
|
-
fragmentId: id,
|
|
4205
|
-
kind: "modified",
|
|
4206
|
-
diff: {
|
|
4207
|
-
before: prevFragment.body,
|
|
4208
|
-
after: nextFragment.body
|
|
4209
|
-
},
|
|
4210
|
-
affectedDimensions: dedup([...prevFragment.dimensionHints, ...nextFragment.dimensionHints]),
|
|
4211
|
-
url: nextFragment.provenance.url,
|
|
4212
|
-
detectedAt: nextFragment.provenance.sourceUpdatedAt
|
|
4213
|
-
});
|
|
4214
|
-
}
|
|
4215
|
-
for (const [id, prevFragment] of prevMap) {
|
|
4216
|
-
if (seen.has(id)) continue;
|
|
4217
|
-
changes.push({
|
|
4218
|
-
fragmentId: id,
|
|
4219
|
-
kind: "removed",
|
|
4220
|
-
diff: { before: prevFragment.body },
|
|
4221
|
-
affectedDimensions: dedup(prevFragment.dimensionHints),
|
|
4222
|
-
url: prevFragment.provenance.url,
|
|
4223
|
-
detectedAt: prevFragment.provenance.sourceUpdatedAt
|
|
4224
|
-
});
|
|
4225
|
-
}
|
|
4226
|
-
const filtered = options.filterDimensions ? changes.filter((c) => c.affectedDimensions.some((d) => options.filterDimensions?.includes(d))) : changes;
|
|
4227
|
-
return {
|
|
4228
|
-
changes: filtered,
|
|
4229
|
-
summary: {
|
|
4230
|
-
added: filtered.filter((c) => c.kind === "added").length,
|
|
4231
|
-
removed: filtered.filter((c) => c.kind === "removed").length,
|
|
4232
|
-
modified: filtered.filter((c) => c.kind === "modified").length
|
|
4233
|
-
},
|
|
4234
|
-
warnings
|
|
4235
|
-
};
|
|
4236
|
-
}
|
|
4237
|
-
function indexFragments(fragments, skipUnverifiable, side) {
|
|
4238
|
-
const map = /* @__PURE__ */ new Map();
|
|
4239
|
-
const warnings = [];
|
|
4240
|
-
let dropped = 0;
|
|
4241
|
-
for (const fragment of fragments) {
|
|
4242
|
-
if (skipUnverifiable && !fragment.provenance.verifiable) {
|
|
4243
|
-
dropped += 1;
|
|
4244
|
-
continue;
|
|
4245
|
-
}
|
|
4246
|
-
if (map.has(fragment.id)) warnings.push(`${side}: duplicate fragment id ${fragment.id} — keeping last`);
|
|
4247
|
-
map.set(fragment.id, fragment);
|
|
4248
|
-
}
|
|
4249
|
-
if (dropped > 0) warnings.push(`${side}: dropped ${dropped} unverifiable fragment(s) before diff`);
|
|
4250
|
-
return {
|
|
4251
|
-
map,
|
|
4252
|
-
warnings
|
|
4253
|
-
};
|
|
4254
|
-
}
|
|
4255
|
-
function dedup(items) {
|
|
4256
|
-
return [...new Set(items)];
|
|
3401
|
+
}
|
|
3402
|
+
/** Recover agent-knowledge's candidate identity from the shared contract. */
|
|
3403
|
+
function fromAgentCandidateKnowledgeRef(candidate) {
|
|
3404
|
+
return KnowledgeImprovementCandidateRefSchema.parse({
|
|
3405
|
+
...candidate,
|
|
3406
|
+
goalHash: rawDigest(candidate.goalHash),
|
|
3407
|
+
baseHash: rawDigest(candidate.baseHash),
|
|
3408
|
+
candidateHash: rawDigest(candidate.candidateHash),
|
|
3409
|
+
evidenceHash: rawDigest(candidate.evidenceHash),
|
|
3410
|
+
promotionPlanHash: rawDigest(candidate.promotionPlanHash)
|
|
3411
|
+
});
|
|
3412
|
+
}
|
|
3413
|
+
function prefixedDigest(value) {
|
|
3414
|
+
return sha256DigestSchema.parse(`sha256:${value}`);
|
|
3415
|
+
}
|
|
3416
|
+
function rawDigest(value) {
|
|
3417
|
+
return sha256DigestSchema.parse(value).slice(7);
|
|
4257
3418
|
}
|
|
4258
3419
|
//#endregion
|
|
4259
3420
|
//#region src/chunking.ts
|
|
@@ -4408,57 +3569,6 @@ function mergeTinyChunks(chunks, opts) {
|
|
|
4408
3569
|
return out;
|
|
4409
3570
|
}
|
|
4410
3571
|
//#endregion
|
|
4411
|
-
//#region src/citation-lint.ts
|
|
4412
|
-
/** Convert a chain-aware citation audit into the package lint vocabulary. */
|
|
4413
|
-
function knowledgeCitationAuditFindings(report) {
|
|
4414
|
-
return report.issues.map(issueToFinding);
|
|
4415
|
-
}
|
|
4416
|
-
/** Lint only current-run pages against their complete declared visibility chain. */
|
|
4417
|
-
async function lintCurrentRunCitations(stores, runId) {
|
|
4418
|
-
return knowledgeCitationAuditFindings(await auditCurrentRunCitations(stores, runId));
|
|
4419
|
-
}
|
|
4420
|
-
function issueToFinding(issue) {
|
|
4421
|
-
const candidates = issue.candidates.map((candidate) => ({
|
|
4422
|
-
pageId: candidate.pageId,
|
|
4423
|
-
path: candidate.page.path,
|
|
4424
|
-
origin: candidate.origin
|
|
4425
|
-
}));
|
|
4426
|
-
if (issue.kind === "ambiguous") return {
|
|
4427
|
-
type: "ambiguous-citation",
|
|
4428
|
-
severity: "error",
|
|
4429
|
-
page: issue.sourcePath,
|
|
4430
|
-
message: `Citation "${issue.persistedCitation}" resolves to ${issue.candidates.length} visible pages; qualify it as here::<pageId>, inherited:<runId>::<pageId>, or shared::<pageId>.`,
|
|
4431
|
-
metadata: {
|
|
4432
|
-
sourcePageId: issue.sourcePageId,
|
|
4433
|
-
sourceOrigin: issue.sourceOrigin,
|
|
4434
|
-
candidates
|
|
4435
|
-
}
|
|
4436
|
-
};
|
|
4437
|
-
return {
|
|
4438
|
-
type: "broken-citation",
|
|
4439
|
-
severity: "error",
|
|
4440
|
-
page: issue.sourcePath,
|
|
4441
|
-
message: issue.kind === "self" ? `Page "${issue.sourcePageId}" cites itself through "${issue.persistedCitation}".` : `Citation "${issue.persistedCitation}" resolves to no visible page.`,
|
|
4442
|
-
metadata: {
|
|
4443
|
-
sourcePageId: issue.sourcePageId,
|
|
4444
|
-
sourceOrigin: issue.sourceOrigin,
|
|
4445
|
-
candidates
|
|
4446
|
-
}
|
|
4447
|
-
};
|
|
4448
|
-
}
|
|
4449
|
-
//#endregion
|
|
4450
|
-
//#region src/collection-research-driver.ts
|
|
4451
|
-
/**
|
|
4452
|
-
* Build the single-agent collection driver. Accepts every source; never gates,
|
|
4453
|
-
* never researches, never steers beyond the loop's default open-gap list. The
|
|
4454
|
-
* worker is the only agent that thinks.
|
|
4455
|
-
*/
|
|
4456
|
-
function createCollectionResearchDriver() {
|
|
4457
|
-
return { verifySource(_source) {
|
|
4458
|
-
return { accept: true };
|
|
4459
|
-
} };
|
|
4460
|
-
}
|
|
4461
|
-
//#endregion
|
|
4462
3572
|
//#region src/discovery.ts
|
|
4463
3573
|
/**
|
|
4464
3574
|
* Dispatch discovery tasks and recursively pursue worker-proposed follow-ups.
|
|
@@ -4611,1340 +3721,33 @@ var FileSystemSearchProvider = class {
|
|
|
4611
3721
|
lexicalIndex: this.lexicalIndex
|
|
4612
3722
|
});
|
|
4613
3723
|
}
|
|
4614
|
-
async retrieve(query, options = {}) {
|
|
4615
|
-
return (await this.search(query, options)).map(hitFromSearchResult);
|
|
4616
|
-
}
|
|
4617
|
-
asRetrievalEvalRetriever() {
|
|
4618
|
-
return async ({ config, scenario, k }) => ({ hits: await this.retrieve(scenario.query, {
|
|
4619
|
-
limit: k,
|
|
4620
|
-
refresh: config.refresh === true
|
|
4621
|
-
}) });
|
|
4622
|
-
}
|
|
4623
|
-
invalidate() {
|
|
4624
|
-
this.index = void 0;
|
|
4625
|
-
this.lexicalIndex = void 0;
|
|
4626
|
-
}
|
|
4627
|
-
};
|
|
4628
|
-
function createFileSystemSearchProvider(options) {
|
|
4629
|
-
return new FileSystemSearchProvider(options);
|
|
4630
|
-
}
|
|
4631
|
-
function hitFromSearchResult(result) {
|
|
4632
|
-
return {
|
|
4633
|
-
pageId: result.page.id,
|
|
4634
|
-
path: result.page.path,
|
|
4635
|
-
title: result.page.title,
|
|
4636
|
-
rank: result.rank,
|
|
4637
|
-
score: result.score,
|
|
4638
|
-
normalizedScore: result.normalizedScore,
|
|
4639
|
-
sourceIds: result.page.sourceIds,
|
|
4640
|
-
snippet: result.snippet
|
|
4641
|
-
};
|
|
4642
|
-
}
|
|
4643
|
-
//#endregion
|
|
4644
|
-
//#region src/freshness.ts
|
|
4645
|
-
const freshnessRecordSchema = z.object({
|
|
4646
|
-
workspaceId: z.string().min(1),
|
|
4647
|
-
sourceId: z.string().min(1),
|
|
4648
|
-
lastRefreshedAt: z.iso.datetime(),
|
|
4649
|
-
contentHash: z.string().min(1).optional()
|
|
4650
|
-
}).strict();
|
|
4651
|
-
const freshnessFileSchema = z.object({ records: z.record(z.string(), freshnessRecordSchema) }).strict();
|
|
4652
|
-
/**
|
|
4653
|
-
* Filesystem-backed implementation. Single JSON file per knowledge root,
|
|
4654
|
-
* indexed by `${workspaceId}::${sourceId}`. Reads parse on every call;
|
|
4655
|
-
* cron tick rate is well below the cost of one JSON parse.
|
|
4656
|
-
*
|
|
4657
|
-
* Writes share the package-wide filesystem lock, so multiple workers cannot
|
|
4658
|
-
* overwrite one another or run through an interrupted knowledge promotion.
|
|
4659
|
-
*/
|
|
4660
|
-
function createFileSystemFreshnessStore(options) {
|
|
4661
|
-
const path = join(options.root, ".agent-knowledge", "freshness.json");
|
|
4662
|
-
const read = async () => {
|
|
4663
|
-
return withKnowledgeRead(options.root, async () => {
|
|
4664
|
-
try {
|
|
4665
|
-
return freshnessFileSchema.parse(JSON.parse(await readFile(path, "utf8"))).records;
|
|
4666
|
-
} catch (error) {
|
|
4667
|
-
if (isMissingFile(error)) return {};
|
|
4668
|
-
throw error;
|
|
4669
|
-
}
|
|
4670
|
-
});
|
|
4671
|
-
};
|
|
4672
|
-
const write = async (records) => {
|
|
4673
|
-
await writeJsonDurableWithinRoot(options.root, ".agent-knowledge/freshness.json", freshnessFileSchema.parse({ records }));
|
|
4674
|
-
};
|
|
4675
|
-
return {
|
|
4676
|
-
async last(key) {
|
|
4677
|
-
const record = (await read())[buildKey(key)];
|
|
4678
|
-
return record ? new Date(record.lastRefreshedAt) : null;
|
|
4679
|
-
},
|
|
4680
|
-
async mark(input) {
|
|
4681
|
-
await withKnowledgeMutation(options.root, async () => {
|
|
4682
|
-
const records = await read();
|
|
4683
|
-
records[buildKey(input)] = {
|
|
4684
|
-
workspaceId: input.workspaceId,
|
|
4685
|
-
sourceId: input.sourceId,
|
|
4686
|
-
lastRefreshedAt: input.when.toISOString(),
|
|
4687
|
-
...input.contentHash === void 0 ? {} : { contentHash: input.contentHash }
|
|
4688
|
-
};
|
|
4689
|
-
await write(records);
|
|
4690
|
-
});
|
|
4691
|
-
},
|
|
4692
|
-
async stale(input) {
|
|
4693
|
-
const last = await this.last(input);
|
|
4694
|
-
if (!last) return true;
|
|
4695
|
-
return (input.now ?? /* @__PURE__ */ new Date()).getTime() - last.getTime() > input.ttlMs;
|
|
4696
|
-
},
|
|
4697
|
-
async list(workspaceId) {
|
|
4698
|
-
const records = await read();
|
|
4699
|
-
return Object.values(records).filter((r) => r.workspaceId === workspaceId);
|
|
4700
|
-
}
|
|
4701
|
-
};
|
|
4702
|
-
}
|
|
4703
|
-
function createD1FreshnessStoreStub(adapter) {
|
|
4704
|
-
return {
|
|
4705
|
-
async last(key) {
|
|
4706
|
-
const record = await adapter.get(key.workspaceId, key.sourceId);
|
|
4707
|
-
return record ? new Date(record.lastRefreshedAt) : null;
|
|
4708
|
-
},
|
|
4709
|
-
async mark(input) {
|
|
4710
|
-
await adapter.upsert({
|
|
4711
|
-
workspaceId: input.workspaceId,
|
|
4712
|
-
sourceId: input.sourceId,
|
|
4713
|
-
lastRefreshedAt: input.when.toISOString(),
|
|
4714
|
-
contentHash: input.contentHash
|
|
4715
|
-
});
|
|
4716
|
-
},
|
|
4717
|
-
async stale(input) {
|
|
4718
|
-
const last = await this.last(input);
|
|
4719
|
-
if (!last) return true;
|
|
4720
|
-
return (input.now ?? /* @__PURE__ */ new Date()).getTime() - last.getTime() > input.ttlMs;
|
|
4721
|
-
},
|
|
4722
|
-
async list(workspaceId) {
|
|
4723
|
-
return adapter.listByWorkspace(workspaceId);
|
|
4724
|
-
}
|
|
4725
|
-
};
|
|
4726
|
-
}
|
|
4727
|
-
function buildKey(key) {
|
|
4728
|
-
return `${key.workspaceId}::${key.sourceId}`;
|
|
4729
|
-
}
|
|
4730
|
-
//#endregion
|
|
4731
|
-
//#region src/invalidation-propagation.ts
|
|
4732
|
-
/**
|
|
4733
|
-
* Invalidation propagation.
|
|
4734
|
-
*
|
|
4735
|
-
* A page whose own evidence refuted it carries an `invalidation`. That verdict
|
|
4736
|
-
* is invisible to a reader who arrives through a citation, so a page that cites
|
|
4737
|
-
* a refuted page records which of its citations are refuted. The pass is
|
|
4738
|
-
* planned as a diff, so a store already stamped produces no mutation and the
|
|
4739
|
-
* pass can run after every grading round.
|
|
4740
|
-
*/
|
|
4741
|
-
/** Frontmatter field naming the cited pages whose evidence refuted them. */
|
|
4742
|
-
const CITES_INVALIDATED_FIELD = "citesInvalidated";
|
|
4743
|
-
/**
|
|
4744
|
-
* Plan the `citesInvalidated` stamp for every page authored in the target
|
|
4745
|
-
* store.
|
|
4746
|
-
*
|
|
4747
|
-
* Citations resolve over the whole chain, so a page here may be stamped for
|
|
4748
|
-
* citing a refuted inherited or shared page. Only `here` pages are stamped: a
|
|
4749
|
-
* run does not write the stores it inherits or shares.
|
|
4750
|
-
*/
|
|
4751
|
-
function planInvalidationPropagation(visiblePages) {
|
|
4752
|
-
if (!Array.isArray(visiblePages)) throw new TypeError("knowledge invalidation propagation requires the visible pages");
|
|
4753
|
-
const invalidatedPageIds = [...new Set(visiblePages.filter((entry) => entry.page.invalidation !== void 0).map((entry) => entry.page.id))].sort();
|
|
4754
|
-
const stamps = [];
|
|
4755
|
-
for (const entry of visiblePages) {
|
|
4756
|
-
if (entry.origin !== "here") continue;
|
|
4757
|
-
const page = entry.page;
|
|
4758
|
-
const refuted = /* @__PURE__ */ new Set();
|
|
4759
|
-
for (const persisted of page.cites ?? []) {
|
|
4760
|
-
const resolution = resolveKnowledgeCitation(visiblePages, parseKnowledgeCitationReference(persisted));
|
|
4761
|
-
if (resolution.resolved?.page.invalidation !== void 0) refuted.add(resolution.resolved.page.id);
|
|
4762
|
-
}
|
|
4763
|
-
const citesInvalidated = [...refuted].sort();
|
|
4764
|
-
const current = idList(page.frontmatter[CITES_INVALIDATED_FIELD]);
|
|
4765
|
-
if (sameOrder(current, citesInvalidated)) continue;
|
|
4766
|
-
stamps.push({
|
|
4767
|
-
page,
|
|
4768
|
-
citesInvalidated,
|
|
4769
|
-
current
|
|
4770
|
-
});
|
|
4771
|
-
}
|
|
4772
|
-
stamps.sort((left, right) => left.page.path.localeCompare(right.page.path));
|
|
4773
|
-
return {
|
|
4774
|
-
invalidatedPageIds,
|
|
4775
|
-
stamps
|
|
4776
|
-
};
|
|
4777
|
-
}
|
|
4778
|
-
/**
|
|
4779
|
-
* Render one plan as a write-block proposal for `applyKnowledgeWriteBlocks`.
|
|
4780
|
-
*
|
|
4781
|
-
* Only the stamped field changes. The page is rendered through
|
|
4782
|
-
* `formatFrontmatter`, so its frontmatter is written in that writer's
|
|
4783
|
-
* normalized form.
|
|
4784
|
-
*/
|
|
4785
|
-
function formatKnowledgeInvalidationProposal(plan) {
|
|
4786
|
-
return plan.stamps.map((stamp) => renderStampedBlock(stamp)).join("\n");
|
|
4787
|
-
}
|
|
4788
|
-
function renderStampedBlock(stamp) {
|
|
4789
|
-
const frontmatter = { ...stamp.page.frontmatter };
|
|
4790
|
-
if (stamp.citesInvalidated.length === 0) delete frontmatter[CITES_INVALIDATED_FIELD];
|
|
4791
|
-
else frontmatter[CITES_INVALIDATED_FIELD] = [...stamp.citesInvalidated];
|
|
4792
|
-
const content = formatFrontmatter(frontmatter, stamp.page.text);
|
|
4793
|
-
return `---FILE: ${stamp.page.path}---\n${content.replace(/\n+$/, "")}\n---END FILE---`;
|
|
4794
|
-
}
|
|
4795
|
-
function idList(value) {
|
|
4796
|
-
return (typeof value === "string" ? [value] : Array.isArray(value) ? value : []).filter((item) => typeof item === "string" && item.trim() !== "");
|
|
4797
|
-
}
|
|
4798
|
-
function sameOrder(left, right) {
|
|
4799
|
-
return left.length === right.length && left.every((item, index) => item === right[index]);
|
|
4800
|
-
}
|
|
4801
|
-
//#endregion
|
|
4802
|
-
//#region src/investment-thesis-set.ts
|
|
4803
|
-
/**
|
|
4804
|
-
* The eval set. 5 public companies, 5-8 held-out material facts each, every fact
|
|
4805
|
-
* grounded in a primary SEC EDGAR 10-K filed on or before the cutoff.
|
|
4806
|
-
*
|
|
4807
|
-
* CURATION-BIAS DISCLOSURE (full version in the doc): all five are companies
|
|
4808
|
-
* whose buried risks later materialized, because that is where the material-vs-
|
|
4809
|
-
* surface distinction is sharpest AND where the figures are easy to verify after
|
|
4810
|
-
* the fact. This biases the set toward downside risks (two of the eight lenses,
|
|
4811
|
-
* concentration + leverage, dominate) and toward distressed names. A production
|
|
4812
|
-
* eval would balance these with companies whose buried facts were POSITIVE
|
|
4813
|
-
* drivers and with survivors. This set is honest about that and reports the lens
|
|
4814
|
-
* distribution so the bias is measurable, not hidden.
|
|
4815
|
-
*/
|
|
4816
|
-
const investmentThesisSet = [
|
|
4817
|
-
{
|
|
4818
|
-
ticker: "SIVB",
|
|
4819
|
-
company: "SVB Financial Group",
|
|
4820
|
-
cik: "719739",
|
|
4821
|
-
cutoff: "2023-02-24",
|
|
4822
|
-
sector: "Banking",
|
|
4823
|
-
knownOutcome: "Failed in a deposit run and was placed in FDIC receivership on March 10, 2023; the holding company filed Chapter 11 on March 17, 2023.",
|
|
4824
|
-
facts: [
|
|
4825
|
-
{
|
|
4826
|
-
id: "SIVB/f1",
|
|
4827
|
-
lens: "off-balance-sheet",
|
|
4828
|
-
fact: "Held-to-maturity (HTM) securities carried at $91.3B amortized cost had a fair value of only $76.2B — a ~$15.1B unrealized loss that, because the portfolio is HTM, never touched earnings or equity and sat only in the footnotes.",
|
|
4829
|
-
expected: [{
|
|
4830
|
-
label: "HTM securities",
|
|
4831
|
-
anyOf: [
|
|
4832
|
-
"held-to-maturity",
|
|
4833
|
-
"held to maturity",
|
|
4834
|
-
"htm"
|
|
4835
|
-
]
|
|
4836
|
-
}, {
|
|
4837
|
-
label: "large unrealized loss (~$15B) / fair value gap",
|
|
4838
|
-
anyOf: [
|
|
4839
|
-
"15.1",
|
|
4840
|
-
"15.2",
|
|
4841
|
-
"$15 billion",
|
|
4842
|
-
"15 billion",
|
|
4843
|
-
"76,169",
|
|
4844
|
-
"76.2 billion",
|
|
4845
|
-
"91,321",
|
|
4846
|
-
"unrealized loss",
|
|
4847
|
-
"below amortized cost",
|
|
4848
|
-
"fair value"
|
|
4849
|
-
]
|
|
4850
|
-
}],
|
|
4851
|
-
minGroups: 2,
|
|
4852
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
4853
|
-
evidence: "Balance sheet (Dec 31, 2022): \"Held-to-maturity securities, at amortized cost ... 91,321\" with parenthetical \"(fair value of $ 76,169 ...)\". The gap = $91,321M - $76,169M = ~$15,152M unrealized loss, disclosed only in the notes."
|
|
4854
|
-
},
|
|
4855
|
-
{
|
|
4856
|
-
id: "SIVB/f2",
|
|
4857
|
-
lens: "off-balance-sheet",
|
|
4858
|
-
fact: "The HTM unrealized loss (~$15.1B) was roughly equal to the company's entire $16.0B total stockholders' equity — a mark-to-market wipeout hidden by HTM accounting.",
|
|
4859
|
-
expected: [{
|
|
4860
|
-
label: "loss near/exceeds total equity",
|
|
4861
|
-
anyOf: [
|
|
4862
|
-
"equity",
|
|
4863
|
-
"capital",
|
|
4864
|
-
"insolvent",
|
|
4865
|
-
"wipe out",
|
|
4866
|
-
"exceeds",
|
|
4867
|
-
"nearly all",
|
|
4868
|
-
"tangible book",
|
|
4869
|
-
"16,004",
|
|
4870
|
-
"16 billion"
|
|
4871
|
-
]
|
|
4872
|
-
}],
|
|
4873
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
4874
|
-
evidence: "\"Total SVBFG stockholders' equity 16,004\" (Dec 31, 2022). The ~$15.15B HTM unrealized loss is ~95% of the $16.0B reported equity."
|
|
4875
|
-
},
|
|
4876
|
-
{
|
|
4877
|
-
id: "SIVB/f3",
|
|
4878
|
-
lens: "concentration",
|
|
4879
|
-
fact: "Estimated uninsured deposits in U.S. offices were $151.5B at year-end 2022 — the run-prone funding base; a high share of total deposits exceeded the FDIC limit.",
|
|
4880
|
-
expected: [{
|
|
4881
|
-
label: "large uninsured deposit base",
|
|
4882
|
-
anyOf: [
|
|
4883
|
-
"uninsured deposit",
|
|
4884
|
-
"above the fdic",
|
|
4885
|
-
"exceed the fdic",
|
|
4886
|
-
"exceeds the fdic",
|
|
4887
|
-
"151.5",
|
|
4888
|
-
"$151 billion",
|
|
4889
|
-
"fdic insurance limit"
|
|
4890
|
-
]
|
|
4891
|
-
}],
|
|
4892
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
4893
|
-
evidence: "\"As of December 31, 2022 ... the amount of estimated uninsured deposits in U.S. offices that exceed the FDIC insurance limit were $151.5 billion\"."
|
|
4894
|
-
},
|
|
4895
|
-
{
|
|
4896
|
-
id: "SIVB/f4",
|
|
4897
|
-
lens: "margin-trend",
|
|
4898
|
-
fact: "Cheap noninterest-bearing demand deposits fell 20 percentage points in one year — to 47% of total deposits from 67% — meaning funding costs were set to rise sharply as clients moved to interest-bearing accounts.",
|
|
4899
|
-
expected: [{
|
|
4900
|
-
label: "deposit mix shift to costlier funding",
|
|
4901
|
-
anyOf: [
|
|
4902
|
-
"noninterest-bearing",
|
|
4903
|
-
"non-interest-bearing",
|
|
4904
|
-
"noninterest bearing",
|
|
4905
|
-
"deposit mix",
|
|
4906
|
-
"funding cost",
|
|
4907
|
-
"cost of deposits",
|
|
4908
|
-
"interest-bearing",
|
|
4909
|
-
"47 percent",
|
|
4910
|
-
"20 percentage"
|
|
4911
|
-
]
|
|
4912
|
-
}],
|
|
4913
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
4914
|
-
evidence: "\"Noninterest-bearing demand deposits to total deposits decreased by 20 percentage points to 47 percent as of December 31, 2022, compared to ... 2021.\""
|
|
4915
|
-
},
|
|
4916
|
-
{
|
|
4917
|
-
id: "SIVB/f5",
|
|
4918
|
-
lens: "concentration",
|
|
4919
|
-
fact: "The deposit and loan base was concentrated in a single client type — the \"innovation economy\" (venture-backed technology and life-science startups) — so a downturn in venture funding would hit deposits and credit simultaneously.",
|
|
4920
|
-
expected: [{
|
|
4921
|
-
label: "concentration in tech / startups / innovation economy",
|
|
4922
|
-
anyOf: [
|
|
4923
|
-
"innovation economy",
|
|
4924
|
-
"technology",
|
|
4925
|
-
"life science",
|
|
4926
|
-
"venture",
|
|
4927
|
-
"startup",
|
|
4928
|
-
"early-stage",
|
|
4929
|
-
"concentrat",
|
|
4930
|
-
"single industry",
|
|
4931
|
-
"sector concentration"
|
|
4932
|
-
]
|
|
4933
|
-
}],
|
|
4934
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
4935
|
-
evidence: "The 10-K repeatedly frames the franchise around clients in \"the innovation economy\" (technology, life science / healthcare, and the venture firms that back them) — a single-sector deposit + credit concentration."
|
|
4936
|
-
},
|
|
4937
|
-
{
|
|
4938
|
-
id: "SIVB/f6",
|
|
4939
|
-
lens: "off-balance-sheet",
|
|
4940
|
-
fact: "Available-for-sale (AFS) securities of $28.6B amortized cost were marked to a $26.1B fair value — a ~$2.5B loss that DID flow through equity (AOCI), the visible tip of a much larger unrealized-loss iceberg dominated by the footnote-only HTM book.",
|
|
4941
|
-
expected: [{
|
|
4942
|
-
label: "AFS unrealized loss / AOCI",
|
|
4943
|
-
anyOf: [
|
|
4944
|
-
"available-for-sale",
|
|
4945
|
-
"available for sale",
|
|
4946
|
-
"afs",
|
|
4947
|
-
"aoci",
|
|
4948
|
-
"accumulated other comprehensive",
|
|
4949
|
-
"28,602",
|
|
4950
|
-
"26,069",
|
|
4951
|
-
"2.5 billion"
|
|
4952
|
-
]
|
|
4953
|
-
}],
|
|
4954
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm",
|
|
4955
|
-
evidence: "\"Available-for-sale securities, at fair value (cost of $ 28,602 ...) 26,069\" — a ~$2.5B AFS unrealized loss carried in AOCI, separate from and much smaller than the HTM gap."
|
|
4956
|
-
}
|
|
4957
|
-
]
|
|
4958
|
-
},
|
|
4959
|
-
{
|
|
4960
|
-
ticker: "BBBY",
|
|
4961
|
-
company: "Bed Bath & Beyond Inc.",
|
|
4962
|
-
cik: "886158",
|
|
4963
|
-
cutoff: "2022-04-21",
|
|
4964
|
-
sector: "Specialty retail",
|
|
4965
|
-
knownOutcome: "Filed for Chapter 11 bankruptcy on April 23, 2023; shareholders were wiped out.",
|
|
4966
|
-
facts: [
|
|
4967
|
-
{
|
|
4968
|
-
id: "BBBY/f1",
|
|
4969
|
-
lens: "capital-return",
|
|
4970
|
-
fact: "The company had repurchased ~$11.685B of its own stock since 2004 — including $574.9M in fiscal 2021 alone, \"two years ahead of schedule\" — draining the balance sheet of a business that was losing money.",
|
|
4971
|
-
expected: [{
|
|
4972
|
-
label: "massive buyback program",
|
|
4973
|
-
anyOf: [
|
|
4974
|
-
"repurchas",
|
|
4975
|
-
"buyback",
|
|
4976
|
-
"buy back",
|
|
4977
|
-
"share repurchase",
|
|
4978
|
-
"11.685",
|
|
4979
|
-
"$11.7 billion",
|
|
4980
|
-
"574.9",
|
|
4981
|
-
"$575 million"
|
|
4982
|
-
]
|
|
4983
|
-
}],
|
|
4984
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
4985
|
-
evidence: "\"Since 2004 through the end of Fiscal 2021, we have repurchased approximately $11.685 billion of our common stock\"; FY2021 alone \"completed share repurchases of $574.9 million ... two years ahead of schedule.\""
|
|
4986
|
-
},
|
|
4987
|
-
{
|
|
4988
|
-
id: "BBBY/f2",
|
|
4989
|
-
lens: "liquidity",
|
|
4990
|
-
fact: "Operating cash flow collapsed to just $17.9M in FY2021, down from $268.1M and $590.9M in the two prior years — a near-total loss of internally generated cash while it kept buying back stock.",
|
|
4991
|
-
expected: [{
|
|
4992
|
-
label: "operating cash flow collapse",
|
|
4993
|
-
anyOf: [
|
|
4994
|
-
"operating cash flow",
|
|
4995
|
-
"cash from operations",
|
|
4996
|
-
"cash provided by operating",
|
|
4997
|
-
"cash flow from operations",
|
|
4998
|
-
"17.9",
|
|
4999
|
-
"17,854",
|
|
5000
|
-
"declining cash flow",
|
|
5001
|
-
"cash generation"
|
|
5002
|
-
]
|
|
5003
|
-
}],
|
|
5004
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
5005
|
-
evidence: "Statement of cash flows: \"Net cash provided by operating activities 17,854 268,108 590,941\" (FY2021 / FY2020 / FY2019, $ thousands)."
|
|
5006
|
-
},
|
|
5007
|
-
{
|
|
5008
|
-
id: "BBBY/f3",
|
|
5009
|
-
lens: "liquidity",
|
|
5010
|
-
fact: "Total shareholders' equity fell ~86% in one year — from $1.277B to $174.1M — as losses plus buybacks ate the equity cushion.",
|
|
5011
|
-
expected: [{
|
|
5012
|
-
label: "equity erosion",
|
|
5013
|
-
anyOf: [
|
|
5014
|
-
"shareholders’ equity",
|
|
5015
|
-
"shareholders' equity",
|
|
5016
|
-
"stockholders equity",
|
|
5017
|
-
"book value",
|
|
5018
|
-
"equity",
|
|
5019
|
-
"net worth",
|
|
5020
|
-
"174.1",
|
|
5021
|
-
"174,145",
|
|
5022
|
-
"eroded"
|
|
5023
|
-
]
|
|
5024
|
-
}],
|
|
5025
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
5026
|
-
evidence: "\"Total shareholders' equity 174,145 1,276,936\" (FY2021 vs FY2020, $ thousands) — an ~86% decline in one year."
|
|
5027
|
-
},
|
|
5028
|
-
{
|
|
5029
|
-
id: "BBBY/f4",
|
|
5030
|
-
lens: "liquidity",
|
|
5031
|
-
fact: "The company posted a $559.6M net loss in FY2021 — yet spent $574.9M on buybacks the same year, i.e. it returned more cash to shareholders than it had, let alone earned.",
|
|
5032
|
-
expected: [{
|
|
5033
|
-
label: "net loss FY2021",
|
|
5034
|
-
anyOf: [
|
|
5035
|
-
"net loss",
|
|
5036
|
-
"unprofitable",
|
|
5037
|
-
"lost money",
|
|
5038
|
-
"losing money",
|
|
5039
|
-
"559.6",
|
|
5040
|
-
"559,623",
|
|
5041
|
-
"$560 million"
|
|
5042
|
-
]
|
|
5043
|
-
}],
|
|
5044
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
5045
|
-
evidence: "\"Net loss $ ( 559,623 )\" for fiscal 2021 ($ thousands)."
|
|
5046
|
-
},
|
|
5047
|
-
{
|
|
5048
|
-
id: "BBBY/f5",
|
|
5049
|
-
lens: "margin-trend",
|
|
5050
|
-
fact: "Merchandise inventories rose to $1.725B even as sales fell — inventory building into a demand decline, a classic markdown-risk and cash-trap signal.",
|
|
5051
|
-
expected: [{
|
|
5052
|
-
label: "inventory building into falling demand",
|
|
5053
|
-
anyOf: [
|
|
5054
|
-
"inventor",
|
|
5055
|
-
"merchandise inventories",
|
|
5056
|
-
"overstock",
|
|
5057
|
-
"markdown",
|
|
5058
|
-
"1,725",
|
|
5059
|
-
"1.7 billion",
|
|
5060
|
-
"stockpile"
|
|
5061
|
-
]
|
|
5062
|
-
}],
|
|
5063
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm",
|
|
5064
|
-
evidence: "\"Merchandise inventories 1,725,410 1,671,909\" ($ thousands) — inventory grew year over year while comparable sales declined."
|
|
5065
|
-
}
|
|
5066
|
-
]
|
|
5067
|
-
},
|
|
5068
|
-
{
|
|
5069
|
-
ticker: "CVNA",
|
|
5070
|
-
company: "Carvana Co.",
|
|
5071
|
-
cik: "1690820",
|
|
5072
|
-
cutoff: "2023-02-23",
|
|
5073
|
-
sector: "Auto e-commerce",
|
|
5074
|
-
knownOutcome: "The stock fell ~98% from its 2021 peak; the company narrowly avoided bankruptcy via a 2023 debt-exchange that cut and extended its obligations.",
|
|
5075
|
-
facts: [
|
|
5076
|
-
{
|
|
5077
|
-
id: "CVNA/f1",
|
|
5078
|
-
lens: "leverage",
|
|
5079
|
-
fact: "Total debt had grown to $8.39B by year-end 2022 (from $5.45B) — a debt load far larger than the equity base, built up funding growth and the ADESA deal.",
|
|
5080
|
-
expected: [{
|
|
5081
|
-
label: "large/growing debt load",
|
|
5082
|
-
anyOf: [
|
|
5083
|
-
"total debt",
|
|
5084
|
-
"long-term debt",
|
|
5085
|
-
"leverage",
|
|
5086
|
-
"highly leveraged",
|
|
5087
|
-
"debt load",
|
|
5088
|
-
"8,391",
|
|
5089
|
-
"8.4 billion",
|
|
5090
|
-
"$8.4 billion"
|
|
5091
|
-
]
|
|
5092
|
-
}],
|
|
5093
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
5094
|
-
evidence: "\"Total debt 8,391 5,447\" (Dec 31, 2022 vs 2021, $ millions)."
|
|
5095
|
-
},
|
|
5096
|
-
{
|
|
5097
|
-
id: "CVNA/f2",
|
|
5098
|
-
lens: "leverage",
|
|
5099
|
-
fact: "Interest expense nearly tripled to $486M in 2022 (from $176M) — debt-service was consuming cash a still-unprofitable company did not have.",
|
|
5100
|
-
expected: [{
|
|
5101
|
-
label: "rising interest burden",
|
|
5102
|
-
anyOf: [
|
|
5103
|
-
"interest expense",
|
|
5104
|
-
"interest cost",
|
|
5105
|
-
"debt service",
|
|
5106
|
-
"cost of debt",
|
|
5107
|
-
"486",
|
|
5108
|
-
"interest burden"
|
|
5109
|
-
]
|
|
5110
|
-
}],
|
|
5111
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
5112
|
-
evidence: "\"Interest expense 486 176\" (FY2022 vs FY2021, $ millions)."
|
|
5113
|
-
},
|
|
5114
|
-
{
|
|
5115
|
-
id: "CVNA/f3",
|
|
5116
|
-
lens: "leverage",
|
|
5117
|
-
fact: "In May 2022 Carvana bought ADESA's U.S. physical auction business for ~$2.2B in cash — a debt-funded acquisition that stretched the balance sheet right as used-car demand turned.",
|
|
5118
|
-
expected: [{
|
|
5119
|
-
label: "ADESA acquisition ~$2.2B",
|
|
5120
|
-
anyOf: [
|
|
5121
|
-
"adesa",
|
|
5122
|
-
"2.2 billion",
|
|
5123
|
-
"$2.2 billion",
|
|
5124
|
-
"physical auction",
|
|
5125
|
-
"acquisition"
|
|
5126
|
-
]
|
|
5127
|
-
}],
|
|
5128
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
5129
|
-
evidence: "\"physical auction business of ADESA US Auction, LLC for approximately $2.2 billion in cash (the 'ADESA Acquisition')\", closed 2022-05-09."
|
|
5130
|
-
},
|
|
5131
|
-
{
|
|
5132
|
-
id: "CVNA/f4",
|
|
5133
|
-
lens: "governance",
|
|
5134
|
-
fact: "Carvana leases hubs and properties from DriveTime — a company controlled by founder/CEO Ernest Garcia III and his father Ernest Garcia II — a recurring related-party arrangement with the controlling family.",
|
|
5135
|
-
expected: [{
|
|
5136
|
-
label: "related-party with founder family / DriveTime",
|
|
5137
|
-
anyOf: [
|
|
5138
|
-
"related party",
|
|
5139
|
-
"related-party",
|
|
5140
|
-
"drivetime",
|
|
5141
|
-
"garcia",
|
|
5142
|
-
"controlled by",
|
|
5143
|
-
"affiliate of",
|
|
5144
|
-
"conflict of interest"
|
|
5145
|
-
]
|
|
5146
|
-
}],
|
|
5147
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
5148
|
-
evidence: "Related Party Transactions note: lease agreements with \"DriveTime Automotive Group\", a related party \"due to Ernest Garcia II, Ernest Garcia III, and entities controlled by one or both of them\"."
|
|
5149
|
-
},
|
|
5150
|
-
{
|
|
5151
|
-
id: "CVNA/f5",
|
|
5152
|
-
lens: "liquidity",
|
|
5153
|
-
fact: "The 2022 net loss was $2.894B — a loss far wider than prior years, showing the unit economics had not turned even at scale.",
|
|
5154
|
-
expected: [{
|
|
5155
|
-
label: "large net loss FY2022",
|
|
5156
|
-
anyOf: [
|
|
5157
|
-
"net loss",
|
|
5158
|
-
"unprofitable",
|
|
5159
|
-
"losing money",
|
|
5160
|
-
"cash burn",
|
|
5161
|
-
"2,894",
|
|
5162
|
-
"2.9 billion",
|
|
5163
|
-
"$2.9 billion"
|
|
5164
|
-
]
|
|
5165
|
-
}],
|
|
5166
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm",
|
|
5167
|
-
evidence: "\"Net loss $ (2,894 ...\" for fiscal 2022 ($ millions)."
|
|
5168
|
-
}
|
|
5169
|
-
]
|
|
5170
|
-
},
|
|
5171
|
-
{
|
|
5172
|
-
ticker: "PTON",
|
|
5173
|
-
company: "Peloton Interactive, Inc.",
|
|
5174
|
-
cik: "1639825",
|
|
5175
|
-
cutoff: "2022-09-07",
|
|
5176
|
-
sector: "Consumer fitness hardware",
|
|
5177
|
-
knownOutcome: "The stock fell ~95% from its 2021 peak; the founder-CEO departed, the company underwent mass layoffs and a multi-year turnaround through fiscal 2024.",
|
|
5178
|
-
facts: [
|
|
5179
|
-
{
|
|
5180
|
-
id: "PTON/f1",
|
|
5181
|
-
lens: "margin-trend",
|
|
5182
|
-
fact: "Connected Fitness (hardware) gross margin turned NEGATIVE — to (11)% in FY2022 — meaning Peloton lost money on every bike/tread it sold before any operating cost; revenue growth was masking a broken unit economics.",
|
|
5183
|
-
expected: [{
|
|
5184
|
-
label: "negative / collapsing hardware gross margin",
|
|
5185
|
-
anyOf: [
|
|
5186
|
-
"gross margin",
|
|
5187
|
-
"negative margin",
|
|
5188
|
-
"gross profit",
|
|
5189
|
-
"margin compression",
|
|
5190
|
-
"losing money on each",
|
|
5191
|
-
"below cost",
|
|
5192
|
-
"(11)",
|
|
5193
|
-
"-11",
|
|
5194
|
-
"negative gross"
|
|
5195
|
-
]
|
|
5196
|
-
}],
|
|
5197
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
5198
|
-
evidence: "MD&A \"Gross Profit, and Gross Margin\" table: Connected Fitness \"Gross Margin decreased to (11)\" percent in fiscal 2022 — a negative hardware gross margin."
|
|
5199
|
-
},
|
|
5200
|
-
{
|
|
5201
|
-
id: "PTON/f2",
|
|
5202
|
-
lens: "liquidity",
|
|
5203
|
-
fact: "Inventories climbed to $1.105B as pandemic demand normalized — a glut of unsold equipment that tied up cash and risked markdowns.",
|
|
5204
|
-
expected: [{
|
|
5205
|
-
label: "inventory glut",
|
|
5206
|
-
anyOf: [
|
|
5207
|
-
"inventor",
|
|
5208
|
-
"overstock",
|
|
5209
|
-
"excess inventory",
|
|
5210
|
-
"glut",
|
|
5211
|
-
"markdown",
|
|
5212
|
-
"unsold",
|
|
5213
|
-
"1,104",
|
|
5214
|
-
"1.1 billion"
|
|
5215
|
-
]
|
|
5216
|
-
}],
|
|
5217
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
5218
|
-
evidence: "\"Inventories, net 1,104.5 937\" (FY2022 vs FY2021, $ millions)."
|
|
5219
|
-
},
|
|
5220
|
-
{
|
|
5221
|
-
id: "PTON/f3",
|
|
5222
|
-
lens: "governance",
|
|
5223
|
-
fact: "A dual-class structure gives Class B holders 20 votes per share vs 1 for Class A — concentrating control with insiders/founders and limiting public shareholders' say.",
|
|
5224
|
-
expected: [{
|
|
5225
|
-
label: "dual-class super-voting control",
|
|
5226
|
-
anyOf: [
|
|
5227
|
-
"dual-class",
|
|
5228
|
-
"dual class",
|
|
5229
|
-
"class b",
|
|
5230
|
-
"20 votes",
|
|
5231
|
-
"super-voting",
|
|
5232
|
-
"supervoting",
|
|
5233
|
-
"voting control",
|
|
5234
|
-
"multiple votes per share"
|
|
5235
|
-
]
|
|
5236
|
-
}],
|
|
5237
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
5238
|
-
evidence: "\"Class B common stock has 20 votes per share and our Class A common stock has one vote per share.\""
|
|
5239
|
-
},
|
|
5240
|
-
{
|
|
5241
|
-
id: "PTON/f4",
|
|
5242
|
-
lens: "liquidity",
|
|
5243
|
-
fact: "Peloton reported a $2.827B net loss in FY2022 — an order-of-magnitude wider loss than the prior year, signaling the demand normalization had broken the model, not just dented it.",
|
|
5244
|
-
expected: [{
|
|
5245
|
-
label: "large net loss FY2022",
|
|
5246
|
-
anyOf: [
|
|
5247
|
-
"net loss",
|
|
5248
|
-
"unprofitable",
|
|
5249
|
-
"losing money",
|
|
5250
|
-
"cash burn",
|
|
5251
|
-
"2,827",
|
|
5252
|
-
"2.8 billion",
|
|
5253
|
-
"$2.8 billion"
|
|
5254
|
-
]
|
|
5255
|
-
}],
|
|
5256
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
5257
|
-
evidence: "\"Net loss $ (2,827 ...\" for fiscal 2022 ($ millions)."
|
|
5258
|
-
},
|
|
5259
|
-
{
|
|
5260
|
-
id: "PTON/f5",
|
|
5261
|
-
lens: "regulatory",
|
|
5262
|
-
fact: "Peloton was running a CPSC recall of its Tread+ treadmill (tied to injuries and a child death) — an open product-safety and legal exposure beyond the demand story.",
|
|
5263
|
-
expected: [{
|
|
5264
|
-
label: "Tread+ / CPSC recall exposure",
|
|
5265
|
-
anyOf: [
|
|
5266
|
-
"recall",
|
|
5267
|
-
"cpsc",
|
|
5268
|
-
"consumer product safety",
|
|
5269
|
-
"tread+",
|
|
5270
|
-
"tread plus",
|
|
5271
|
-
"product safety",
|
|
5272
|
-
"injuries",
|
|
5273
|
-
"safety"
|
|
5274
|
-
]
|
|
5275
|
-
}],
|
|
5276
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
5277
|
-
evidence: "\"recall on Tread+, which we are conducting in collaboration with the Consumer Product Safety Commission ('CPSC')\"; the Tread product recalls \"in the fourth quarter of fiscal 2021 continued to impact\" results."
|
|
5278
|
-
},
|
|
5279
|
-
{
|
|
5280
|
-
id: "PTON/f6",
|
|
5281
|
-
lens: "leverage",
|
|
5282
|
-
fact: "Peloton was locked into ~$334M of manufacturing purchase commitments even as demand fell — contractual inventory it had to take on regardless of whether it could sell it.",
|
|
5283
|
-
expected: [{
|
|
5284
|
-
label: "locked-in purchase commitments",
|
|
5285
|
-
anyOf: [
|
|
5286
|
-
"purchase commitment",
|
|
5287
|
-
"purchase obligation",
|
|
5288
|
-
"minimum purchase",
|
|
5289
|
-
"take-or-pay",
|
|
5290
|
-
"committed to purchase",
|
|
5291
|
-
"334",
|
|
5292
|
-
"manufacturing commitment"
|
|
5293
|
-
]
|
|
5294
|
-
}],
|
|
5295
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm",
|
|
5296
|
-
evidence: "\"purchase commitments related to the manufacture of Peloton products were estimated to be approximately $334\" million."
|
|
5297
|
-
}
|
|
5298
|
-
]
|
|
5299
|
-
},
|
|
5300
|
-
{
|
|
5301
|
-
ticker: "SI",
|
|
5302
|
-
company: "Silvergate Capital Corporation",
|
|
5303
|
-
cik: "1312109",
|
|
5304
|
-
cutoff: "2022-02-28",
|
|
5305
|
-
sector: "Banking (digital-asset)",
|
|
5306
|
-
knownOutcome: "After the FTX collapse triggered a deposit run, Silvergate announced a voluntary wind-down of Silvergate Bank and liquidation in March 2023.",
|
|
5307
|
-
facts: [
|
|
5308
|
-
{
|
|
5309
|
-
id: "SI/f1",
|
|
5310
|
-
lens: "concentration",
|
|
5311
|
-
fact: "About 99.5% of total deposits were noninterest-bearing — essentially all funding was non-term money that could leave on demand, an extreme run-risk masked by very low funding cost.",
|
|
5312
|
-
expected: [{
|
|
5313
|
-
label: "almost all deposits noninterest-bearing / on-demand",
|
|
5314
|
-
anyOf: [
|
|
5315
|
-
"noninterest bearing",
|
|
5316
|
-
"noninterest-bearing",
|
|
5317
|
-
"non-interest-bearing",
|
|
5318
|
-
"demand deposit",
|
|
5319
|
-
"no term",
|
|
5320
|
-
"on demand",
|
|
5321
|
-
"99.5",
|
|
5322
|
-
"99 percent",
|
|
5323
|
-
"leave at any time"
|
|
5324
|
-
]
|
|
5325
|
-
}],
|
|
5326
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
5327
|
-
evidence: "\"noninterest bearing deposits as a percentage of total deposits was 99.5% as of December 31, 2021.\""
|
|
5328
|
-
},
|
|
5329
|
-
{
|
|
5330
|
-
id: "SI/f2",
|
|
5331
|
-
lens: "concentration",
|
|
5332
|
-
fact: "Roughly 58% of deposits came from digital-currency EXCHANGES alone — a handful of correlated crypto counterparties whose own troubles would pull deposits out together.",
|
|
5333
|
-
expected: [{
|
|
5334
|
-
label: "deposits concentrated in crypto exchanges",
|
|
5335
|
-
anyOf: [
|
|
5336
|
-
"digital currency exchange",
|
|
5337
|
-
"crypto exchange",
|
|
5338
|
-
"exchanges represent",
|
|
5339
|
-
"counterpart",
|
|
5340
|
-
"concentrat",
|
|
5341
|
-
"58%",
|
|
5342
|
-
"58 percent",
|
|
5343
|
-
"approximately 58"
|
|
5344
|
-
]
|
|
5345
|
-
}],
|
|
5346
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
5347
|
-
evidence: "\"Deposits from digital currency exchanges represent approximately 58%\" of deposits."
|
|
5348
|
-
},
|
|
5349
|
-
{
|
|
5350
|
-
id: "SI/f3",
|
|
5351
|
-
lens: "concentration",
|
|
5352
|
-
fact: "The entire deposit franchise was tied to a single, volatile industry — digital-currency (crypto) customers — so a crypto downturn was a direct, undiversified funding shock.",
|
|
5353
|
-
expected: [{
|
|
5354
|
-
label: "single-industry deposit CONCENTRATION (not just \"it banks crypto\")",
|
|
5355
|
-
anyOf: [
|
|
5356
|
-
"digital currency customers",
|
|
5357
|
-
"single industry",
|
|
5358
|
-
"one industry",
|
|
5359
|
-
"single volatile industry",
|
|
5360
|
-
"sector concentration",
|
|
5361
|
-
"undiversified",
|
|
5362
|
-
"concentrat"
|
|
5363
|
-
]
|
|
5364
|
-
}],
|
|
5365
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
5366
|
-
evidence: "The 10-K's strategy and risk factors center the bank on \"digital currency customers\" and \"the concentration of our deposits\" in that single industry."
|
|
5367
|
-
},
|
|
5368
|
-
{
|
|
5369
|
-
id: "SI/f4",
|
|
5370
|
-
lens: "liquidity",
|
|
5371
|
-
fact: "Total deposits had ballooned to $14.3B (from $10.4B) — fast, hot-money growth from the crypto boom that could reverse just as fast.",
|
|
5372
|
-
expected: [{
|
|
5373
|
-
label: "specific hot-money deposit base ($14.3B / volatile)",
|
|
5374
|
-
anyOf: [
|
|
5375
|
-
"hot money",
|
|
5376
|
-
"hot-money",
|
|
5377
|
-
"volatile deposit",
|
|
5378
|
-
"could reverse",
|
|
5379
|
-
"14.3 billion",
|
|
5380
|
-
"14,290",
|
|
5381
|
-
"$14.3b",
|
|
5382
|
-
"$14 billion"
|
|
5383
|
-
]
|
|
5384
|
-
}],
|
|
5385
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
5386
|
-
evidence: "\"Total deposits $ 14,290,628\" ... prior year \"$ 10,411,278\" ($ thousands)."
|
|
5387
|
-
},
|
|
5388
|
-
{
|
|
5389
|
-
id: "SI/f5",
|
|
5390
|
-
lens: "concentration",
|
|
5391
|
-
fact: "The franchise hinged on a single proprietary product — the Silvergate Exchange Network (SEN), a payment network built exclusively for the digital-currency industry — so its competitive moat and its deposit base were the SAME crypto-dependent bet, not two diversified ones.",
|
|
5392
|
-
expected: [{
|
|
5393
|
-
label: "names the SEN single-product dependence (not generic \"payment network\")",
|
|
5394
|
-
anyOf: [
|
|
5395
|
-
"silvergate exchange network",
|
|
5396
|
-
"the sen",
|
|
5397
|
-
"sen)",
|
|
5398
|
-
"sen'",
|
|
5399
|
-
"single product",
|
|
5400
|
-
"single-product",
|
|
5401
|
-
"core product",
|
|
5402
|
-
"one product",
|
|
5403
|
-
"same bet"
|
|
5404
|
-
]
|
|
5405
|
-
}],
|
|
5406
|
-
sourceUrl: "https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm",
|
|
5407
|
-
evidence: "\"Silvergate Exchange Network ('SEN'), our proprietary, virtually instantaneous payment network for participants in the digital currency industry\" — the bank's differentiator and its deposit magnet are the same crypto-only product."
|
|
5408
|
-
}
|
|
5409
|
-
]
|
|
5410
|
-
}
|
|
5411
|
-
];
|
|
5412
|
-
/**
|
|
5413
|
-
* Grade ONE material fact against an investment thesis's full text. Returns
|
|
5414
|
-
* whether the thesis SURFACED it plus which expected groups were found. The
|
|
5415
|
-
* check is a deterministic case-insensitive substring scan — $0, model-free,
|
|
5416
|
-
* reproducible — so the eval never leaks into a model the loop could observe.
|
|
5417
|
-
*/
|
|
5418
|
-
function gradeFactAgainstText(fact, thesisText) {
|
|
5419
|
-
const haystack = thesisText.toLowerCase();
|
|
5420
|
-
const found = fact.expected.filter((group) => group.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase())));
|
|
5421
|
-
const minGroups = fact.minGroups ?? fact.expected.length;
|
|
5422
|
-
return {
|
|
5423
|
-
surfaced: found.length >= minGroups,
|
|
5424
|
-
groupsFound: found.length,
|
|
5425
|
-
groupsTotal: fact.expected.length,
|
|
5426
|
-
foundLabels: found.map((group) => group.label)
|
|
5427
|
-
};
|
|
5428
|
-
}
|
|
5429
|
-
/** Grade a whole company's thesis text: how many of its held-out facts it surfaces. */
|
|
5430
|
-
function gradeCompanyAgainstText(company, thesisText) {
|
|
5431
|
-
const perFact = company.facts.map((fact) => gradeFactAgainstText(fact, thesisText));
|
|
5432
|
-
return {
|
|
5433
|
-
surfaced: perFact.filter((result) => result.surfaced).length,
|
|
5434
|
-
total: company.facts.length,
|
|
5435
|
-
perFact
|
|
5436
|
-
};
|
|
5437
|
-
}
|
|
5438
|
-
/** Total held-out facts across the set (the denominator the doc reports). */
|
|
5439
|
-
function totalMaterialFacts(set = investmentThesisSet) {
|
|
5440
|
-
return set.reduce((sum, company) => sum + company.facts.length, 0);
|
|
5441
|
-
}
|
|
5442
|
-
/** Count facts per lens across the set — used to report (and bound) curation bias. */
|
|
5443
|
-
function lensDistribution(set = investmentThesisSet) {
|
|
5444
|
-
const dist = {};
|
|
5445
|
-
for (const company of set) for (const fact of company.facts) dist[fact.lens] = (dist[fact.lens] ?? 0) + 1;
|
|
5446
|
-
return dist;
|
|
5447
|
-
}
|
|
5448
|
-
//#endregion
|
|
5449
|
-
//#region src/material-facts-metric.ts
|
|
5450
|
-
/**
|
|
5451
|
-
* `materialFactsSurfaced` — the held-out investment-research METRIC.
|
|
5452
|
-
*
|
|
5453
|
-
* Given a knowledge base a research loop built for a company and the company's
|
|
5454
|
-
* HELD-OUT material-fact checklist (`tests/eval/investment-thesis-set.ts`, never
|
|
5455
|
-
* shown to the loop), this returns the FRACTION of checklist items the KB's
|
|
5456
|
-
* pages surface + ground. The check is the same `$0`, model-free, deterministic
|
|
5457
|
-
* substring grader the loop's checklist already ships (`gradeFactAgainstText` /
|
|
5458
|
-
* `gradeCompanyAgainstText`) — so the answer key never reaches a model the loop
|
|
5459
|
-
* could observe, exactly the firewall the ML deep-question exam uses.
|
|
5460
|
-
*
|
|
5461
|
-
* The ONLY thing this file adds over the raw grader is the KB→text join: it reads
|
|
5462
|
-
* the curated pages (and the raw source text) the loop wrote and hands their
|
|
5463
|
-
* concatenation to the grader. That join mirrors `kbText` in the research-quality
|
|
5464
|
-
* A/B (research-driving-ab.test.ts) so the thesis metric and the ML-exam metric
|
|
5465
|
-
* read a KB the same way.
|
|
5466
|
-
*
|
|
5467
|
-
* WHY pages AND source text: an honest thesis surfaces a buried fact in its
|
|
5468
|
-
* curated thesis PAGE (the judgment), but a loop whose page is thin while its
|
|
5469
|
-
* fetched filings are rich should still get credit for what it actually pulled.
|
|
5470
|
-
* Grading the union is the faithful, not the lenient, choice — it rewards the
|
|
5471
|
-
* loop that REACHED the filing even if its synthesis was terse, and it cannot
|
|
5472
|
-
* manufacture a hit the underlying evidence does not contain.
|
|
5473
|
-
*/
|
|
5474
|
-
/**
|
|
5475
|
-
* Join a KB index into the single text blob the grader scans: every curated PAGE
|
|
5476
|
-
* (title + body) followed by every raw SOURCE (title + fetched text). This is the
|
|
5477
|
-
* text read AFTER the loop finished — it is never handed to the loop. Identical
|
|
5478
|
-
* in spirit to `kbText` in the research-quality A/B so both metrics read a KB the
|
|
5479
|
-
* same way.
|
|
5480
|
-
*/
|
|
5481
|
-
function kbIndexToText(index) {
|
|
5482
|
-
return `${index.pages.map((page) => `${page.title}\n${page.text}`).join("\n\n")}\n\n${index.sources.map((source) => `${source.title ?? ""}\n${source.text ?? ""}`).join("\n\n")}`;
|
|
5483
|
-
}
|
|
5484
|
-
/**
|
|
5485
|
-
* Grade one company's KB against its held-out material-fact checklist, given the
|
|
5486
|
-
* KB's already-joined text. The pure core — no I/O — so calibration can score a
|
|
5487
|
-
* hand-written shallow/deep thesis string directly and the live path can score a
|
|
5488
|
-
* real KB. Returns the surfaced FRACTION plus the per-fact audit trail.
|
|
5489
|
-
*/
|
|
5490
|
-
function materialFactsSurfacedInText(company, kbText) {
|
|
5491
|
-
const grade = gradeCompanyAgainstText(company, kbText);
|
|
5492
|
-
const perFact = company.facts.map((fact) => {
|
|
5493
|
-
const r = gradeFactAgainstText(fact, kbText);
|
|
5494
|
-
return {
|
|
5495
|
-
id: fact.id,
|
|
5496
|
-
lens: fact.lens,
|
|
5497
|
-
surfaced: r.surfaced,
|
|
5498
|
-
groupsFound: r.groupsFound,
|
|
5499
|
-
groupsTotal: r.groupsTotal,
|
|
5500
|
-
foundLabels: r.foundLabels
|
|
5501
|
-
};
|
|
5502
|
-
});
|
|
5503
|
-
return {
|
|
5504
|
-
ticker: company.ticker,
|
|
5505
|
-
company: company.company,
|
|
5506
|
-
surfaced: grade.surfaced,
|
|
5507
|
-
total: grade.total,
|
|
5508
|
-
fraction: grade.total === 0 ? 0 : grade.surfaced / grade.total,
|
|
5509
|
-
perFact
|
|
5510
|
-
};
|
|
5511
|
-
}
|
|
5512
|
-
/**
|
|
5513
|
-
* `materialFactsSurfaced(kb, checklist)` — the metric in its KB-reading form.
|
|
5514
|
-
*
|
|
5515
|
-
* `kb` is EITHER a knowledge-base root directory (the loop wrote pages there) or
|
|
5516
|
-
* an already-built `KnowledgeIndex`. `checklist` is the company's held-out
|
|
5517
|
-
* `CompanyEvalCase`. Returns the surfaced fraction + per-fact trail.
|
|
5518
|
-
*
|
|
5519
|
-
* The checklist is HELD OUT by construction: it lives in the test eval set, is
|
|
5520
|
-
* never passed to the loop, and is read only here, after the loop finished.
|
|
5521
|
-
*/
|
|
5522
|
-
async function materialFactsSurfaced(kb, checklist) {
|
|
5523
|
-
return materialFactsSurfacedInText(checklist, kbIndexToText(typeof kb === "string" ? await buildKnowledgeIndex(kb) : kb));
|
|
5524
|
-
}
|
|
5525
|
-
//#endregion
|
|
5526
|
-
//#region src/verified-research-loop.ts
|
|
5527
|
-
/**
|
|
5528
|
-
* Two-agent (driver + worker) sibling of `runKnowledgeResearchLoop`.
|
|
5529
|
-
*
|
|
5530
|
-
* Both agents research to grow ONE knowledge base. The roles are differentiated:
|
|
5531
|
-
*
|
|
5532
|
-
* - **WORKER** = primary research. Each round it reads the open gaps, discovers
|
|
5533
|
-
* new sources, and proposes additions (`sources` + `proposalText`). It ADDS.
|
|
5534
|
-
* - **DRIVER** = the verifier / gap-filler / gate. It (1) VERIFIES the worker's
|
|
5535
|
-
* sources before they commit — dedup against the KB, then `verifySource`
|
|
5536
|
-
* rejects ones that aren't real/relevant; (2) GAP-FILLS the gaps the worker
|
|
5537
|
-
* missed with its own research pass (when `driverResearches`); (3) folds the
|
|
5538
|
-
* remaining gaps into the worker's next prompt; and (4) GATES on
|
|
5539
|
-
* `scoreKnowledgeReadiness` and optional `isComplete` — the loop stops when
|
|
5540
|
-
* readiness passes and the driver has no unfinished research.
|
|
5541
|
-
*
|
|
5542
|
-
* Set `driverResearches: false` (default) for the pure-coordinator mode: the
|
|
5543
|
-
* driver only verifies + gates and contributes no research itself.
|
|
5544
|
-
*
|
|
5545
|
-
* Composes existing atoms — `initKnowledgeBase`, `addSourceText`,
|
|
5546
|
-
* `applyKnowledgeWriteBlocks`, `buildEvalKnowledgeBundle` (the readiness gate),
|
|
5547
|
-
* and `searchKnowledge` — and reinvents none of them.
|
|
5548
|
-
*/
|
|
5549
|
-
async function runVerifiedResearchLoop(options) {
|
|
5550
|
-
const assertActive = () => options.signal?.throwIfAborted();
|
|
5551
|
-
assertActive();
|
|
5552
|
-
const maxRounds = Math.max(1, options.maxRounds ?? 3);
|
|
5553
|
-
await initKnowledgeBase(options.root);
|
|
5554
|
-
assertActive();
|
|
5555
|
-
const store = new FileSystemKbStore({ root: options.root });
|
|
5556
|
-
const steps = [];
|
|
5557
|
-
let index = await buildKnowledgeIndex(options.root);
|
|
5558
|
-
assertActive();
|
|
5559
|
-
await confirmRegisteredSources(options.driver, index.sources);
|
|
5560
|
-
assertActive();
|
|
5561
|
-
let readiness = readinessFor(options, index);
|
|
5562
|
-
let ready = isReady(readiness?.report) && (options.driver.isComplete?.() ?? true);
|
|
5563
|
-
let steer;
|
|
5564
|
-
for (let round = 1; round <= maxRounds && !ready; round++) {
|
|
5565
|
-
assertActive();
|
|
5566
|
-
const gaps = gapsFromReadiness(readiness);
|
|
5567
|
-
const workerContribution = await options.worker({
|
|
5568
|
-
root: options.root,
|
|
5569
|
-
goal: options.goal,
|
|
5570
|
-
round,
|
|
5571
|
-
index,
|
|
5572
|
-
gaps,
|
|
5573
|
-
steer,
|
|
5574
|
-
readiness: requireReadiness(readiness, options),
|
|
5575
|
-
signal: options.signal
|
|
5576
|
-
});
|
|
5577
|
-
assertActive();
|
|
5578
|
-
const accepted = [];
|
|
5579
|
-
const rejectedWorkerSources = [];
|
|
5580
|
-
const existingUris = new Set(index.sources.flatMap((source) => typeof source.metadata?.originalUri === "string" ? [source.metadata.originalUri] : []));
|
|
5581
|
-
for (const proposedSource of workerContribution.sources ?? []) {
|
|
5582
|
-
const source = snapshotSourceTextInput(proposedSource);
|
|
5583
|
-
if (isDuplicate(source, existingUris, accepted)) {
|
|
5584
|
-
rejectedWorkerSources.push({
|
|
5585
|
-
source,
|
|
5586
|
-
reason: "duplicate: already in the knowledge base"
|
|
5587
|
-
});
|
|
5588
|
-
continue;
|
|
5589
|
-
}
|
|
5590
|
-
const verdict = await options.driver.verifySource(source, {
|
|
5591
|
-
root: options.root,
|
|
5592
|
-
goal: options.goal,
|
|
5593
|
-
round,
|
|
5594
|
-
index,
|
|
5595
|
-
gaps,
|
|
5596
|
-
acceptedThisRound: accepted,
|
|
5597
|
-
signal: options.signal
|
|
5598
|
-
});
|
|
5599
|
-
assertActive();
|
|
5600
|
-
if (verdict.accept) accepted.push(source);
|
|
5601
|
-
else rejectedWorkerSources.push({
|
|
5602
|
-
source,
|
|
5603
|
-
reason: verdict.reason
|
|
5604
|
-
});
|
|
5605
|
-
}
|
|
5606
|
-
const acceptedWorkerSources = await registerSources(options, accepted);
|
|
5607
|
-
assertActive();
|
|
5608
|
-
await confirmRegisteredSources(options.driver, acceptedWorkerSources);
|
|
5609
|
-
assertActive();
|
|
5610
|
-
const writtenPages = [];
|
|
5611
|
-
writtenPages.push(...await applyPages(options.root, workerContribution, acceptedWorkerSources));
|
|
5612
|
-
assertActive();
|
|
5613
|
-
index = await buildKnowledgeIndex(options.root);
|
|
5614
|
-
assertActive();
|
|
5615
|
-
readiness = readinessFor(options, index);
|
|
5616
|
-
let driverSources = [];
|
|
5617
|
-
let driverNotes;
|
|
5618
|
-
if (options.driverResearches && options.driver.research) {
|
|
5619
|
-
const remainingGaps = gapsFromReadiness(readiness);
|
|
5620
|
-
const driverContribution = await options.driver.research({
|
|
5621
|
-
root: options.root,
|
|
5622
|
-
goal: options.goal,
|
|
5623
|
-
round,
|
|
5624
|
-
index,
|
|
5625
|
-
remainingGaps,
|
|
5626
|
-
readiness: requireReadiness(readiness, options),
|
|
5627
|
-
signal: options.signal
|
|
5628
|
-
});
|
|
5629
|
-
assertActive();
|
|
5630
|
-
driverNotes = driverContribution.notes;
|
|
5631
|
-
driverSources = await registerSources(options, driverContribution.sources ?? []);
|
|
5632
|
-
assertActive();
|
|
5633
|
-
await confirmRegisteredSources(options.driver, driverSources);
|
|
5634
|
-
assertActive();
|
|
5635
|
-
writtenPages.push(...await applyPages(options.root, driverContribution, driverSources));
|
|
5636
|
-
assertActive();
|
|
5637
|
-
index = await buildKnowledgeIndex(options.root);
|
|
5638
|
-
assertActive();
|
|
5639
|
-
readiness = readinessFor(options, index);
|
|
5640
|
-
}
|
|
5641
|
-
const driverComplete = options.driver.isComplete?.() ?? true;
|
|
5642
|
-
ready = isReady(readiness?.report) && driverComplete;
|
|
5643
|
-
const remainingGaps = gapsFromReadiness(readiness);
|
|
5644
|
-
if (ready || remainingGaps.length === 0 && driverComplete) steer = void 0;
|
|
5645
|
-
else {
|
|
5646
|
-
await options.driver.prepareFold?.();
|
|
5647
|
-
assertActive();
|
|
5648
|
-
steer = foldGaps(options.driver, remainingGaps);
|
|
5649
|
-
}
|
|
5650
|
-
const step = {
|
|
5651
|
-
round,
|
|
5652
|
-
gaps,
|
|
5653
|
-
acceptedWorkerSources,
|
|
5654
|
-
rejectedWorkerSources,
|
|
5655
|
-
driverSources,
|
|
5656
|
-
writtenPages,
|
|
5657
|
-
readiness,
|
|
5658
|
-
ready,
|
|
5659
|
-
event: createKnowledgeEvent({
|
|
5660
|
-
type: "research.iteration",
|
|
5661
|
-
actor: options.actor,
|
|
5662
|
-
target: options.root,
|
|
5663
|
-
metadata: {
|
|
5664
|
-
goal: options.goal,
|
|
5665
|
-
round,
|
|
5666
|
-
ready,
|
|
5667
|
-
acceptedWorkerSourceCount: acceptedWorkerSources.length,
|
|
5668
|
-
rejectedWorkerSourceCount: rejectedWorkerSources.length,
|
|
5669
|
-
driverSourceCount: driverSources.length,
|
|
5670
|
-
writtenPageCount: writtenPages.length,
|
|
5671
|
-
remainingGapCount: remainingGaps.length
|
|
5672
|
-
}
|
|
5673
|
-
}),
|
|
5674
|
-
notes: {
|
|
5675
|
-
worker: workerContribution.notes,
|
|
5676
|
-
driver: driverNotes
|
|
5677
|
-
}
|
|
5678
|
-
};
|
|
5679
|
-
await options.driver.checkpoint?.();
|
|
5680
|
-
assertActive();
|
|
5681
|
-
await store.putEvent(step.event);
|
|
5682
|
-
assertActive();
|
|
5683
|
-
steps.push(step);
|
|
5684
|
-
await options.onRound?.(step);
|
|
5685
|
-
assertActive();
|
|
5686
|
-
}
|
|
5687
|
-
assertActive();
|
|
5688
|
-
return {
|
|
5689
|
-
root: options.root,
|
|
5690
|
-
goal: options.goal,
|
|
5691
|
-
rounds: steps.length,
|
|
5692
|
-
ready,
|
|
5693
|
-
index,
|
|
5694
|
-
readiness,
|
|
5695
|
-
steps
|
|
5696
|
-
};
|
|
5697
|
-
}
|
|
5698
|
-
function isReady(report) {
|
|
5699
|
-
if (!report) return false;
|
|
5700
|
-
return report.blockingMissingRequirements.length === 0;
|
|
5701
|
-
}
|
|
5702
|
-
function gapsFromReadiness(readiness) {
|
|
5703
|
-
if (!readiness) return [];
|
|
5704
|
-
const blocking = readiness.report.blockingMissingRequirements.map((requirement) => gapFor(requirement, readiness, true));
|
|
5705
|
-
const nonBlocking = readiness.report.nonBlockingGaps.map((requirement) => gapFor(requirement, readiness, false));
|
|
5706
|
-
return [...blocking, ...nonBlocking];
|
|
5707
|
-
}
|
|
5708
|
-
function gapFor(requirement, readiness, blocking) {
|
|
5709
|
-
const spec = readiness.requirements.find((entry) => entry.id === requirement.id);
|
|
5710
|
-
const query = typeof spec?.metadata?.query === "string" ? spec.metadata.query : requirement.description;
|
|
5711
|
-
return {
|
|
5712
|
-
id: requirement.id,
|
|
5713
|
-
description: requirement.description,
|
|
5714
|
-
query,
|
|
5715
|
-
blocking
|
|
5716
|
-
};
|
|
5717
|
-
}
|
|
5718
|
-
function foldGaps(driver, gaps) {
|
|
5719
|
-
if (driver.foldGaps) return driver.foldGaps(gaps);
|
|
5720
|
-
if (gaps.length === 0) return void 0;
|
|
5721
|
-
return ["The knowledge base is still missing the following. Prioritise these next round:", ...gaps.map((gap) => `- (${gap.blocking ? "blocking" : "soft"}) ${gap.description} [${gap.id}]`)].join("\n");
|
|
5722
|
-
}
|
|
5723
|
-
function isDuplicate(source, existingUris, accepted) {
|
|
5724
|
-
return existingUris.has(source.uri) || accepted.some((candidate) => candidate.uri === source.uri);
|
|
5725
|
-
}
|
|
5726
|
-
async function registerSources(options, sources) {
|
|
5727
|
-
const records = [];
|
|
5728
|
-
for (const candidate of sources) {
|
|
5729
|
-
options.signal?.throwIfAborted();
|
|
5730
|
-
const source = snapshotSourceTextInput(candidate);
|
|
5731
|
-
records.push(await addSourceText(options.root, source, options.sourceOptions));
|
|
5732
|
-
options.signal?.throwIfAborted();
|
|
5733
|
-
}
|
|
5734
|
-
return records;
|
|
5735
|
-
}
|
|
5736
|
-
async function confirmRegisteredSources(driver, sources) {
|
|
5737
|
-
if (!driver.commitSources) return;
|
|
5738
|
-
const textSources = sources.filter((source) => typeof source.metadata?.originalUri === "string");
|
|
5739
|
-
if (textSources.length === 0) return;
|
|
5740
|
-
await driver.commitSources(textSources);
|
|
5741
|
-
}
|
|
5742
|
-
/**
|
|
5743
|
-
* Apply a contribution's curated pages. Static `proposalText` plus a
|
|
5744
|
-
* `buildPages(acceptedSources)` result are concatenated and run through the safe
|
|
5745
|
-
* write protocol — but ONLY when at least one source survived verification, so a
|
|
5746
|
-
* page never cites a rejected (or absent) source.
|
|
5747
|
-
*/
|
|
5748
|
-
async function applyPages(root, contribution, acceptedSources) {
|
|
5749
|
-
if (acceptedSources.length === 0) return [];
|
|
5750
|
-
const parts = [];
|
|
5751
|
-
if (contribution.proposalText) parts.push(contribution.proposalText);
|
|
5752
|
-
const built = contribution.buildPages?.(acceptedSources);
|
|
5753
|
-
if (built) parts.push(built);
|
|
5754
|
-
if (parts.length === 0) return [];
|
|
5755
|
-
return (await applyKnowledgeWriteBlocks(root, parts.join("\n"))).written;
|
|
5756
|
-
}
|
|
5757
|
-
function requireReadiness(readiness, options) {
|
|
5758
|
-
if (readiness) return readiness;
|
|
5759
|
-
return buildEvalKnowledgeBundle({
|
|
5760
|
-
...options.readiness ?? {},
|
|
5761
|
-
taskId: options.readinessTaskId ?? options.goal,
|
|
5762
|
-
index: emptyIndex(options.root),
|
|
5763
|
-
specs: []
|
|
5764
|
-
});
|
|
5765
|
-
}
|
|
5766
|
-
function emptyIndex(root) {
|
|
5767
|
-
return {
|
|
5768
|
-
root,
|
|
5769
|
-
generatedAt: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
5770
|
-
sources: [],
|
|
5771
|
-
pages: [],
|
|
5772
|
-
graph: {
|
|
5773
|
-
nodes: [],
|
|
5774
|
-
edges: []
|
|
5775
|
-
}
|
|
5776
|
-
};
|
|
5777
|
-
}
|
|
5778
|
-
/**
|
|
5779
|
-
* Helper for verifiers: does the candidate source's text/title overlap any page
|
|
5780
|
-
* the readiness search returns for a gap query? A cheap relevance heuristic the
|
|
5781
|
-
* driver can compose into `verifySource` (real verifiers can do more).
|
|
5782
|
-
*/
|
|
5783
|
-
function sourceMatchesGaps(source, index, gaps) {
|
|
5784
|
-
const haystack = `${source.title ?? ""}\n${source.text}`.toLowerCase();
|
|
5785
|
-
const hits = [];
|
|
5786
|
-
for (const gap of gaps) for (const token of gap.query.toLowerCase().split(/\s+/).filter(Boolean)) if (haystack.includes(token)) {
|
|
5787
|
-
hits.push(...searchKnowledge(index, gap.query, 1));
|
|
5788
|
-
break;
|
|
5789
|
-
}
|
|
5790
|
-
return hits;
|
|
5791
|
-
}
|
|
5792
|
-
//#endregion
|
|
5793
|
-
//#region src/investment-thesis-task.ts
|
|
5794
|
-
/**
|
|
5795
|
-
* The INVESTMENT-THESIS research task.
|
|
5796
|
-
*
|
|
5797
|
-
* Given `{ company, ticker, cik, cutoff }`, drive the SAME two-agent research
|
|
5798
|
-
* loop the ML deep-question A/B uses (`runVerifiedResearchLoop` + the real web
|
|
5799
|
-
* worker) to research the company AS OF the cutoff — web + SEC EDGAR, both public
|
|
5800
|
-
* — and produce an investment-thesis PAGE in the knowledge base: a judgment, the
|
|
5801
|
-
* drivers, and the risks, grounded in what it fetched.
|
|
5802
|
-
*
|
|
5803
|
-
* This file builds NOTHING new for the loop: it composes the existing worker +
|
|
5804
|
-
* driver + loop, supplies the readiness specs that steer the worker toward the
|
|
5805
|
-
* filing-level evidence (the analyst lenses), then writes a synthesis thesis page
|
|
5806
|
-
* the metric (`materialFactsSurfaced`) grades against the HELD-OUT checklist.
|
|
5807
|
-
*
|
|
5808
|
-
* THE FIREWALL: the task is told ONLY company + ticker + cutoff (+ the generic
|
|
5809
|
-
* analyst-lens readiness specs every company gets). It is NEVER shown the
|
|
5810
|
-
* checklist. The checklist is read only afterward, by the metric. So a high score
|
|
5811
|
-
* is research depth, not teaching-to-the-test.
|
|
5812
|
-
*/
|
|
5813
|
-
/**
|
|
5814
|
-
* The generic analyst-lens readiness specs every company gets. They are the ONLY
|
|
5815
|
-
* thing the loop is told about WHAT to look for, and they name the LENSES a
|
|
5816
|
-
* thorough analyst checks (balance-sheet risk, concentration, leverage, margins,
|
|
5817
|
-
* liquidity, governance, regulatory) and where they live (the latest SEC 10-K) —
|
|
5818
|
-
* NOT the answers. They steer the worker's web/EDGAR search toward the filing,
|
|
5819
|
-
* not toward the held-out facts (which the loop never sees).
|
|
5820
|
-
*
|
|
5821
|
-
* `minSources` is set above 1 so the readiness gate stays UNMET after a single
|
|
5822
|
-
* fetch and the loop runs multiple rounds — the depth-driving driver needs >1
|
|
5823
|
-
* round to steer, exactly as the ML-exam multi-round probe established.
|
|
5824
|
-
*/
|
|
5825
|
-
function thesisReadinessSpecs(input) {
|
|
5826
|
-
const c = input.company;
|
|
5827
|
-
const t = input.ticker;
|
|
5828
|
-
const filing = `${c} ${t} SEC 10-K annual report SEC.gov EDGAR filing`;
|
|
5829
|
-
return [
|
|
5830
|
-
defineReadinessSpec({
|
|
5831
|
-
id: "thesis/filing",
|
|
5832
|
-
description: `the most recent SEC 10-K annual report for ${c} (${t}) filed on or before ${input.cutoff}, from SEC EDGAR`,
|
|
5833
|
-
query: filing,
|
|
5834
|
-
requiredFor: ["ResearchAgent"],
|
|
5835
|
-
importance: "blocking",
|
|
5836
|
-
minSources: 2,
|
|
5837
|
-
minHits: 1
|
|
5838
|
-
}),
|
|
5839
|
-
defineReadinessSpec({
|
|
5840
|
-
id: "thesis/balance-sheet",
|
|
5841
|
-
description: `${c} balance-sheet risks: securities marked below cost, unrealized losses, leverage / total debt, debt maturities, interest expense`,
|
|
5842
|
-
query: `${c} ${t} 10-K balance sheet total debt unrealized losses interest expense leverage`,
|
|
5843
|
-
requiredFor: ["ResearchAgent"],
|
|
5844
|
-
importance: "blocking",
|
|
5845
|
-
minSources: 2,
|
|
5846
|
-
minHits: 1
|
|
5847
|
-
}),
|
|
5848
|
-
defineReadinessSpec({
|
|
5849
|
-
id: "thesis/concentration-liquidity",
|
|
5850
|
-
description: `${c} concentration + liquidity: customer / deposit / revenue concentration, uninsured deposits, operating cash flow, net loss, inventory, equity erosion`,
|
|
5851
|
-
query: `${c} ${t} 10-K customer deposit concentration operating cash flow net loss inventory`,
|
|
5852
|
-
requiredFor: ["ResearchAgent"],
|
|
5853
|
-
importance: "blocking",
|
|
5854
|
-
minSources: 2,
|
|
5855
|
-
minHits: 1
|
|
5856
|
-
}),
|
|
5857
|
-
defineReadinessSpec({
|
|
5858
|
-
id: "thesis/governance-regulatory",
|
|
5859
|
-
description: `${c} governance + regulatory: related-party transactions, dual-class / super-voting control, buybacks / dividends, recalls, regulatory or legal exposure, margin trends`,
|
|
5860
|
-
query: `${c} ${t} 10-K related party dual class share repurchase recall regulatory gross margin`,
|
|
5861
|
-
requiredFor: ["ResearchAgent"],
|
|
5862
|
-
importance: "blocking",
|
|
5863
|
-
minSources: 2,
|
|
5864
|
-
minHits: 1
|
|
5865
|
-
})
|
|
5866
|
-
];
|
|
5867
|
-
}
|
|
5868
|
-
/**
|
|
5869
|
-
* The thesis-writer prompt. After the loop has fetched + curated the filings, we
|
|
5870
|
-
* ask the model to SYNTHESIZE a thesis page from the curated KB text — a
|
|
5871
|
-
* judgment, the key drivers, and the material risks, grounded ONLY in the fetched
|
|
5872
|
-
* evidence. The held-out checklist is NOT in this prompt; the model writes from
|
|
5873
|
-
* what the loop actually pulled, so a fact only appears if the research surfaced
|
|
5874
|
-
* the underlying evidence.
|
|
5875
|
-
*/
|
|
5876
|
-
function thesisSynthesisMessages(input, kbText) {
|
|
5877
|
-
const system = "You are a buy-side investment analyst writing a thesis memo. You are given the raw research your team gathered from public filings (SEC 10-K) and the web. Write a thesis that a thorough analyst would write: lead with your JUDGMENT, then the KEY DRIVERS, then the MATERIAL RISKS. Be specific and quantitative — name the actual figures, balance-sheet items, concentrations, leverage, margin trends, governance items, and regulatory exposures that appear in the research. Surface the buried, non-obvious drivers a one-line ticker summary misses. Ground every claim in the research provided; do NOT invent figures. If the research does not contain a figure, do not state it.";
|
|
5878
|
-
const user = [
|
|
5879
|
-
`Company: ${input.company} (${input.ticker})`,
|
|
5880
|
-
`As-of date (reason as if it is this date): ${input.cutoff}`,
|
|
5881
|
-
input.sector ? `Sector: ${input.sector}` : "",
|
|
5882
|
-
"",
|
|
5883
|
-
"Research gathered (filings + web excerpts):",
|
|
5884
|
-
"\"\"\"",
|
|
5885
|
-
kbText.slice(0, 24e3),
|
|
5886
|
-
"\"\"\"",
|
|
5887
|
-
"",
|
|
5888
|
-
"Write the investment thesis now. Structure:",
|
|
5889
|
-
"## Judgment",
|
|
5890
|
-
"## Key drivers",
|
|
5891
|
-
"## Material risks"
|
|
5892
|
-
].filter(Boolean).join("\n");
|
|
5893
|
-
return [{
|
|
5894
|
-
role: "system",
|
|
5895
|
-
content: system
|
|
5896
|
-
}, {
|
|
5897
|
-
role: "user",
|
|
5898
|
-
content: user
|
|
5899
|
-
}];
|
|
5900
|
-
}
|
|
5901
|
-
/** Write the synthesis thesis page into the KB so the index + metric pick it up. */
|
|
5902
|
-
async function writeThesisPage(root, input, thesis) {
|
|
5903
|
-
return withKnowledgeMutation(root, async () => {
|
|
5904
|
-
const relativePath = `knowledge/thesis-${slugify(input.ticker)}.md`;
|
|
5905
|
-
const body = [
|
|
5906
|
-
"---",
|
|
5907
|
-
`title: Investment thesis — ${input.company} (${input.ticker})`,
|
|
5908
|
-
`ticker: ${input.ticker}`,
|
|
5909
|
-
`cutoff: ${input.cutoff}`,
|
|
5910
|
-
"kind: investment-thesis",
|
|
5911
|
-
"---",
|
|
5912
|
-
`# Investment thesis — ${input.company} (${input.ticker}), as of ${input.cutoff}`,
|
|
5913
|
-
"",
|
|
5914
|
-
thesis.trim(),
|
|
5915
|
-
""
|
|
5916
|
-
].join("\n");
|
|
5917
|
-
await writeFileDurableWithinRoot(root, relativePath, body, { encoding: "utf8" });
|
|
5918
|
-
return join(root, relativePath);
|
|
5919
|
-
});
|
|
3724
|
+
async retrieve(query, options = {}) {
|
|
3725
|
+
return (await this.search(query, options)).map(hitFromSearchResult);
|
|
3726
|
+
}
|
|
3727
|
+
asRetrievalEvalRetriever() {
|
|
3728
|
+
return async ({ config, scenario, k }) => ({ hits: await this.retrieve(scenario.query, {
|
|
3729
|
+
limit: k,
|
|
3730
|
+
refresh: config.refresh === true
|
|
3731
|
+
}) });
|
|
3732
|
+
}
|
|
3733
|
+
invalidate() {
|
|
3734
|
+
this.index = void 0;
|
|
3735
|
+
this.lexicalIndex = void 0;
|
|
3736
|
+
}
|
|
3737
|
+
};
|
|
3738
|
+
function createFileSystemSearchProvider(options) {
|
|
3739
|
+
return new FileSystemSearchProvider(options);
|
|
5920
3740
|
}
|
|
5921
|
-
|
|
5922
|
-
* Run the full thesis task: drive the two-agent loop to research the company AS
|
|
5923
|
-
* OF the cutoff, then synthesize + write the thesis page. Returns the loop result
|
|
5924
|
-
* + the thesis text + the page path. The caller grades the KB with
|
|
5925
|
-
* `materialFactsSurfaced(root, checklist)` — the checklist is never passed here.
|
|
5926
|
-
*/
|
|
5927
|
-
async function runInvestmentThesisTask(input, options) {
|
|
5928
|
-
const worker = createWebResearchWorker({
|
|
5929
|
-
...options.workerOptions,
|
|
5930
|
-
router: options.router
|
|
5931
|
-
});
|
|
5932
|
-
const goal = `${input.company} (${input.ticker}) investment thesis as of ${input.cutoff}`;
|
|
5933
|
-
const loop = await runVerifiedResearchLoop({
|
|
5934
|
-
root: options.root,
|
|
5935
|
-
goal,
|
|
5936
|
-
worker,
|
|
5937
|
-
driver: options.driver,
|
|
5938
|
-
readinessSpecs: thesisReadinessSpecs(input),
|
|
5939
|
-
maxRounds: options.maxRounds ?? 3,
|
|
5940
|
-
signal: options.signal
|
|
5941
|
-
});
|
|
5942
|
-
const messages = thesisSynthesisMessages(input, kbIndexToText(await buildKnowledgeIndex(options.root)));
|
|
5943
|
-
const thesis = await options.router.chat(messages, options.synthesisMaxTokens ?? 1600);
|
|
3741
|
+
function hitFromSearchResult(result) {
|
|
5944
3742
|
return {
|
|
5945
|
-
|
|
5946
|
-
|
|
5947
|
-
|
|
3743
|
+
pageId: result.page.id,
|
|
3744
|
+
path: result.page.path,
|
|
3745
|
+
title: result.page.title,
|
|
3746
|
+
rank: result.rank,
|
|
3747
|
+
score: result.score,
|
|
3748
|
+
normalizedScore: result.normalizedScore,
|
|
3749
|
+
sourceIds: result.page.sourceIds,
|
|
3750
|
+
snippet: result.snippet
|
|
5948
3751
|
};
|
|
5949
3752
|
}
|
|
5950
3753
|
//#endregion
|
|
@@ -7398,227 +5201,6 @@ async function evaluateKnowledgeBaseReadiness(options) {
|
|
|
7398
5201
|
};
|
|
7399
5202
|
}
|
|
7400
5203
|
//#endregion
|
|
7401
|
-
//#region src/relation-graph.ts
|
|
7402
|
-
/** A relation graph input or query that cannot be honored exactly. */
|
|
7403
|
-
var KnowledgeRelationGraphError = class extends Error {
|
|
7404
|
-
code;
|
|
7405
|
-
constructor(code, message) {
|
|
7406
|
-
super(message);
|
|
7407
|
-
this.name = "KnowledgeRelationGraphError";
|
|
7408
|
-
this.code = code;
|
|
7409
|
-
}
|
|
7410
|
-
};
|
|
7411
|
-
/**
|
|
7412
|
-
* Build a labeled multi-edge graph from caller relations.
|
|
7413
|
-
*
|
|
7414
|
-
* The graph keeps one edge per `(sourceId, targetId, predicate)`, in first-seen
|
|
7415
|
-
* order. A repeated triple is accepted only when its weight and metadata are
|
|
7416
|
-
* identical in canonical JSON; any other repeat is a conflict and is refused.
|
|
7417
|
-
* Node and relation metadata must be finite, acyclic JSON so the graph
|
|
7418
|
-
* round-trips through `KnowledgeRelationGraphSchema` unchanged.
|
|
7419
|
-
*/
|
|
7420
|
-
function buildKnowledgeRelationGraph(input) {
|
|
7421
|
-
const declared = input.nodes === void 0 ? void 0 : /* @__PURE__ */ new Map();
|
|
7422
|
-
const nodes = [];
|
|
7423
|
-
if (declared !== void 0) for (const [position, node] of input.nodes.entries()) {
|
|
7424
|
-
const normalized = normalizeNode(node, position);
|
|
7425
|
-
if (declared.has(normalized.id)) throw new KnowledgeRelationGraphError("duplicate-node", `node "${normalized.id}" is declared more than once`);
|
|
7426
|
-
declared.set(normalized.id, normalized);
|
|
7427
|
-
nodes.push(normalized);
|
|
7428
|
-
}
|
|
7429
|
-
const edges = [];
|
|
7430
|
-
const canonicalByTriple = /* @__PURE__ */ new Map();
|
|
7431
|
-
for (const [position, relation] of input.relations.entries()) {
|
|
7432
|
-
const normalized = normalizeRelation(relation, position);
|
|
7433
|
-
if (declared !== void 0) {
|
|
7434
|
-
for (const endpoint of [normalized.sourceId, normalized.targetId]) if (!declared.has(endpoint)) throw new KnowledgeRelationGraphError("undeclared-endpoint", `relation ${describeTriple(normalized)} names undeclared node "${endpoint}"`);
|
|
7435
|
-
}
|
|
7436
|
-
const canonical = canonicalRelationJson(normalized);
|
|
7437
|
-
const key = tripleKey(normalized);
|
|
7438
|
-
const prior = canonicalByTriple.get(key);
|
|
7439
|
-
if (prior !== void 0) {
|
|
7440
|
-
if (prior === canonical) continue;
|
|
7441
|
-
throw new KnowledgeRelationGraphError("duplicate-relation", `relation ${describeTriple(normalized)} is repeated with different weight or metadata`);
|
|
7442
|
-
}
|
|
7443
|
-
canonicalByTriple.set(key, canonical);
|
|
7444
|
-
edges.push(normalized);
|
|
7445
|
-
}
|
|
7446
|
-
return {
|
|
7447
|
-
nodes,
|
|
7448
|
-
edges
|
|
7449
|
-
};
|
|
7450
|
-
}
|
|
7451
|
-
/** Relations touching `id`, each with the node at the other end, in graph order. */
|
|
7452
|
-
function neighbors(graph, id, query) {
|
|
7453
|
-
const index = indexOf(graph);
|
|
7454
|
-
validateQuery(query);
|
|
7455
|
-
assertKnownNode(index, id);
|
|
7456
|
-
return neighborsOf(index, id, query);
|
|
7457
|
-
}
|
|
7458
|
-
/**
|
|
7459
|
-
* Breadth-first traversal from `id` over matching relations. Each reachable
|
|
7460
|
-
* node is reported once, at the depth where the search first reached it; the
|
|
7461
|
-
* start node is never reported, so a cycle back to it adds no step.
|
|
7462
|
-
*/
|
|
7463
|
-
function walk(graph, id, options) {
|
|
7464
|
-
const index = indexOf(graph);
|
|
7465
|
-
validateQuery(options);
|
|
7466
|
-
validateMaxDepth(options.maxDepth);
|
|
7467
|
-
assertKnownNode(index, id);
|
|
7468
|
-
const steps = [];
|
|
7469
|
-
for (const step of traverse(index, id, options, options.maxDepth)) steps.push(step);
|
|
7470
|
-
return steps;
|
|
7471
|
-
}
|
|
7472
|
-
/**
|
|
7473
|
-
* Whether `to` is `from` or lies on a path of matching relations out of `from`.
|
|
7474
|
-
*/
|
|
7475
|
-
function isReachable(graph, from, to, query) {
|
|
7476
|
-
const index = indexOf(graph);
|
|
7477
|
-
validateQuery(query);
|
|
7478
|
-
assertKnownNode(index, from);
|
|
7479
|
-
assertKnownNode(index, to);
|
|
7480
|
-
if (from === to) return true;
|
|
7481
|
-
for (const step of traverse(index, from, query, void 0)) if (step.nodeId === to) return true;
|
|
7482
|
-
return false;
|
|
7483
|
-
}
|
|
7484
|
-
const indexes = /* @__PURE__ */ new WeakMap();
|
|
7485
|
-
function indexOf(graph) {
|
|
7486
|
-
const cached = indexes.get(graph);
|
|
7487
|
-
if (cached !== void 0) return cached;
|
|
7488
|
-
const outgoing = /* @__PURE__ */ new Map();
|
|
7489
|
-
const incoming = /* @__PURE__ */ new Map();
|
|
7490
|
-
const known = /* @__PURE__ */ new Set();
|
|
7491
|
-
for (const node of graph.nodes) known.add(node.id);
|
|
7492
|
-
for (const relation of graph.edges) {
|
|
7493
|
-
known.add(relation.sourceId);
|
|
7494
|
-
known.add(relation.targetId);
|
|
7495
|
-
push(outgoing, relation.sourceId, relation);
|
|
7496
|
-
push(incoming, relation.targetId, relation);
|
|
7497
|
-
}
|
|
7498
|
-
const index = {
|
|
7499
|
-
outgoing,
|
|
7500
|
-
incoming,
|
|
7501
|
-
known
|
|
7502
|
-
};
|
|
7503
|
-
indexes.set(graph, index);
|
|
7504
|
-
return index;
|
|
7505
|
-
}
|
|
7506
|
-
function push(map, key, relation) {
|
|
7507
|
-
const list = map.get(key);
|
|
7508
|
-
if (list === void 0) map.set(key, [relation]);
|
|
7509
|
-
else list.push(relation);
|
|
7510
|
-
}
|
|
7511
|
-
function neighborsOf(index, id, query) {
|
|
7512
|
-
const matches = (relation) => query.predicate === void 0 || relation.predicate === query.predicate;
|
|
7513
|
-
const result = [];
|
|
7514
|
-
if (query.direction !== "in") {
|
|
7515
|
-
for (const relation of index.outgoing.get(id) ?? []) if (matches(relation)) result.push({
|
|
7516
|
-
nodeId: relation.targetId,
|
|
7517
|
-
relation
|
|
7518
|
-
});
|
|
7519
|
-
}
|
|
7520
|
-
if (query.direction !== "out") for (const relation of index.incoming.get(id) ?? []) {
|
|
7521
|
-
if (query.direction === "both" && relation.sourceId === id) continue;
|
|
7522
|
-
if (matches(relation)) result.push({
|
|
7523
|
-
nodeId: relation.sourceId,
|
|
7524
|
-
relation
|
|
7525
|
-
});
|
|
7526
|
-
}
|
|
7527
|
-
return result;
|
|
7528
|
-
}
|
|
7529
|
-
function* traverse(index, start, query, maxDepth) {
|
|
7530
|
-
const visited = /* @__PURE__ */ new Set([start]);
|
|
7531
|
-
const queue = [{
|
|
7532
|
-
nodeId: start,
|
|
7533
|
-
depth: 0
|
|
7534
|
-
}];
|
|
7535
|
-
for (let head = 0; head < queue.length; head++) {
|
|
7536
|
-
const current = queue[head];
|
|
7537
|
-
if (maxDepth !== void 0 && current.depth >= maxDepth) continue;
|
|
7538
|
-
for (const neighbor of neighborsOf(index, current.nodeId, query)) {
|
|
7539
|
-
if (visited.has(neighbor.nodeId)) continue;
|
|
7540
|
-
visited.add(neighbor.nodeId);
|
|
7541
|
-
const depth = current.depth + 1;
|
|
7542
|
-
yield {
|
|
7543
|
-
nodeId: neighbor.nodeId,
|
|
7544
|
-
depth,
|
|
7545
|
-
from: current.nodeId,
|
|
7546
|
-
relation: neighbor.relation
|
|
7547
|
-
};
|
|
7548
|
-
queue.push({
|
|
7549
|
-
nodeId: neighbor.nodeId,
|
|
7550
|
-
depth
|
|
7551
|
-
});
|
|
7552
|
-
}
|
|
7553
|
-
}
|
|
7554
|
-
}
|
|
7555
|
-
function assertKnownNode(index, id) {
|
|
7556
|
-
if (!index.known.has(id)) throw new KnowledgeRelationGraphError("unknown-node", `node "${id}" is not in the graph`);
|
|
7557
|
-
}
|
|
7558
|
-
function validateQuery(query) {
|
|
7559
|
-
if (query.direction !== "out" && query.direction !== "in" && query.direction !== "both") throw new KnowledgeRelationGraphError("invalid-query", `direction must be "out", "in", or "both", got ${JSON.stringify(query.direction)}`);
|
|
7560
|
-
if (query.predicate !== void 0 && !isNonEmptyString(query.predicate)) throw new KnowledgeRelationGraphError("invalid-query", "predicate must be a non-empty string");
|
|
7561
|
-
}
|
|
7562
|
-
function validateMaxDepth(maxDepth) {
|
|
7563
|
-
if (maxDepth === void 0) return;
|
|
7564
|
-
if (!Number.isInteger(maxDepth) || maxDepth < 0) throw new KnowledgeRelationGraphError("invalid-query", `maxDepth must be a non-negative integer, got ${JSON.stringify(maxDepth)}`);
|
|
7565
|
-
}
|
|
7566
|
-
function normalizeNode(node, position) {
|
|
7567
|
-
if (!isNonEmptyString(node.id)) throw new KnowledgeRelationGraphError("invalid-node", `node at ${position} has no id`);
|
|
7568
|
-
if (!isNonEmptyString(node.kind)) throw new KnowledgeRelationGraphError("invalid-node", `node "${node.id}" has no kind`);
|
|
7569
|
-
if (node.label !== void 0 && typeof node.label !== "string") throw new KnowledgeRelationGraphError("invalid-node", `node "${node.id}" label is not a string`);
|
|
7570
|
-
if (node.metadata !== void 0) assertCanonicalMetadata(node.metadata, `node "${node.id}"`, "invalid-node");
|
|
7571
|
-
return {
|
|
7572
|
-
id: node.id,
|
|
7573
|
-
kind: node.kind,
|
|
7574
|
-
...node.label !== void 0 ? { label: node.label } : {},
|
|
7575
|
-
...node.metadata !== void 0 ? { metadata: node.metadata } : {}
|
|
7576
|
-
};
|
|
7577
|
-
}
|
|
7578
|
-
function normalizeRelation(relation, position) {
|
|
7579
|
-
for (const field of [
|
|
7580
|
-
"sourceId",
|
|
7581
|
-
"targetId",
|
|
7582
|
-
"predicate"
|
|
7583
|
-
]) if (!isNonEmptyString(relation[field])) throw new KnowledgeRelationGraphError("invalid-relation", `relation at ${position} has no ${field}`);
|
|
7584
|
-
if (relation.weight !== void 0 && !Number.isFinite(relation.weight)) throw new KnowledgeRelationGraphError("invalid-relation", `relation ${describeTriple(relation)} weight must be a finite number`);
|
|
7585
|
-
return {
|
|
7586
|
-
sourceId: relation.sourceId,
|
|
7587
|
-
targetId: relation.targetId,
|
|
7588
|
-
predicate: relation.predicate,
|
|
7589
|
-
...relation.weight !== void 0 ? { weight: relation.weight } : {},
|
|
7590
|
-
...relation.metadata !== void 0 ? { metadata: relation.metadata } : {}
|
|
7591
|
-
};
|
|
7592
|
-
}
|
|
7593
|
-
function canonicalRelationJson(relation) {
|
|
7594
|
-
if (relation.metadata !== void 0) assertCanonicalMetadata(relation.metadata, `relation ${describeTriple(relation)}`, "invalid-relation");
|
|
7595
|
-
return canonicalCandidateJson({
|
|
7596
|
-
...relation.weight !== void 0 ? { weight: relation.weight } : {},
|
|
7597
|
-
...relation.metadata !== void 0 ? { metadata: relation.metadata } : {}
|
|
7598
|
-
});
|
|
7599
|
-
}
|
|
7600
|
-
function assertCanonicalMetadata(metadata, subject, code) {
|
|
7601
|
-
if (metadata === null || typeof metadata !== "object" || Array.isArray(metadata)) throw new KnowledgeRelationGraphError(code, `${subject} metadata must be an object`);
|
|
7602
|
-
try {
|
|
7603
|
-
canonicalCandidateJson(metadata);
|
|
7604
|
-
} catch (error) {
|
|
7605
|
-
throw new KnowledgeRelationGraphError(code, `${subject} metadata must be finite, acyclic JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
7606
|
-
}
|
|
7607
|
-
}
|
|
7608
|
-
function tripleKey(relation) {
|
|
7609
|
-
return JSON.stringify([
|
|
7610
|
-
relation.sourceId,
|
|
7611
|
-
relation.targetId,
|
|
7612
|
-
relation.predicate
|
|
7613
|
-
]);
|
|
7614
|
-
}
|
|
7615
|
-
function describeTriple(relation) {
|
|
7616
|
-
return `${relation.sourceId} -[${relation.predicate}]-> ${relation.targetId}`;
|
|
7617
|
-
}
|
|
7618
|
-
function isNonEmptyString(value) {
|
|
7619
|
-
return typeof value === "string" && value.length > 0;
|
|
7620
|
-
}
|
|
7621
|
-
//#endregion
|
|
7622
5204
|
//#region src/release.ts
|
|
7623
5205
|
function knowledgeReleaseReport(input) {
|
|
7624
5206
|
const baselineRuns = input.baselineRuns ?? [];
|
|
@@ -7655,680 +5237,6 @@ function knowledgeReleaseReport(input) {
|
|
|
7655
5237
|
};
|
|
7656
5238
|
}
|
|
7657
5239
|
//#endregion
|
|
7658
|
-
|
|
7659
|
-
/**
|
|
7660
|
-
* Research-DRIVING driver for `runVerifiedResearchLoop`.
|
|
7661
|
-
*
|
|
7662
|
-
* The shipped drivers all FILTER the worker's sources:
|
|
7663
|
-
* - `createVerifyingResearchDriver` judges on-topic relevance,
|
|
7664
|
-
* - `createAdaptiveResearchDriver` dedups then triages then escalates,
|
|
7665
|
-
* - `createClaimGroundingVerifier` rejects misattributed citations.
|
|
7666
|
-
*
|
|
7667
|
-
* This driver does the OPPOSITE job: instead of narrowing the worker's output,
|
|
7668
|
-
* it DRIVES the research DEEPER each round. Its value is not "fewer sources" —
|
|
7669
|
-
* it is "more answered, better-corroborated sub-questions". Concretely, each
|
|
7670
|
-
* round it:
|
|
7671
|
-
*
|
|
7672
|
-
* 1. EXTRACTS the key claims from the worker's new sources (one LLM pass per
|
|
7673
|
-
* source, in `verifySource`; falls back to a deterministic sentence-pull
|
|
7674
|
-
* when the model is unavailable so a round never silently extracts nothing).
|
|
7675
|
-
* 2. TRACKS each claim's support — the set of INDEPENDENT sources (by canonical
|
|
7676
|
-
* host) that assert it — and detects CONTRADICTIONS between a new claim and
|
|
7677
|
-
* one already on the ledger.
|
|
7678
|
-
* 3. GENERATES the next round's DEEP sub-questions from the accumulated claims,
|
|
7679
|
-
* in four kinds — comparative ("how does X's tradeoff differ from Y's?"),
|
|
7680
|
-
* mechanism ("under what precise condition does X fail?"), gap ("what
|
|
7681
|
-
* specific result is missing?"), and contradiction ("does any source
|
|
7682
|
-
* challenge claim Z?").
|
|
7683
|
-
* 4. FLAGS weakly-supported claims (only ONE independent source) and
|
|
7684
|
-
* contradicted claims as INVALIDATION targets and demands the worker find
|
|
7685
|
-
* corroborating / refuting evidence for them.
|
|
7686
|
-
* 5. FOLDS the deep sub-questions + invalidation challenges into the worker's
|
|
7687
|
-
* next prompt via the loop's `foldGaps` → `steer` channel — that is the
|
|
7688
|
-
* mechanism that drives DEPTH and VALIDATION rather than breadth.
|
|
7689
|
-
*
|
|
7690
|
-
* COMPLETION (`isComplete` / the `done` judgment the caller gates on) does NOT
|
|
7691
|
-
* look at source COUNT. It is done only when every deep sub-question it raised
|
|
7692
|
-
* has been addressed AND every key claim is either supported by >= 2 independent
|
|
7693
|
-
* sources OR explicitly marked CONTESTED (a contradiction the loop surfaced and
|
|
7694
|
-
* could not resolve). A KB with twenty sources all asserting one unchallenged
|
|
7695
|
-
* claim is NOT done; a KB whose handful of claims are each corroborated or
|
|
7696
|
-
* contested IS.
|
|
7697
|
-
*
|
|
7698
|
-
* It reuses `runVerifiedResearchLoop` (it is a plain `ResearchDriver`), the web
|
|
7699
|
-
* worker, `claim-ledger.ts` (claim identity, independent-source identity, and
|
|
7700
|
-
* the merge rule), and the `RouterClient` chat surface; it reinvents none of
|
|
7701
|
-
* them.
|
|
7702
|
-
*/
|
|
7703
|
-
/**
|
|
7704
|
-
* The in-memory driver. Its belief state lives for exactly as long as the
|
|
7705
|
-
* process does — use `createPersistentResearchDrivingDriver` when the run must
|
|
7706
|
-
* survive a crash or resume.
|
|
7707
|
-
*/
|
|
7708
|
-
function createResearchDrivingDriver(options = {}) {
|
|
7709
|
-
return buildDriver(options);
|
|
7710
|
-
}
|
|
7711
|
-
/**
|
|
7712
|
-
* The durable driver: same behaviour, plus its claim ledger is read from the
|
|
7713
|
-
* store at construction and written back after every claim and every round.
|
|
7714
|
-
*
|
|
7715
|
-
* Construction is asynchronous because loading is I/O, and loading has to happen
|
|
7716
|
-
* before the caller can read `researchState()` or `isComplete()` — a driver that
|
|
7717
|
-
* loaded lazily would answer "nothing researched, not complete" for a run that
|
|
7718
|
-
* had already corroborated everything.
|
|
7719
|
-
*/
|
|
7720
|
-
async function createPersistentResearchDrivingDriver(options) {
|
|
7721
|
-
const ledgerId = assertClaimLedgerId(options.ledgerId);
|
|
7722
|
-
const existing = await options.store.getClaimLedger(ledgerId);
|
|
7723
|
-
const driver = buildDriver(options, {
|
|
7724
|
-
store: options.store,
|
|
7725
|
-
ledgerId
|
|
7726
|
-
}, existing ?? void 0);
|
|
7727
|
-
if (existing && (existing.preparedRounds ?? existing.rounds) > existing.rounds) await driver.checkpoint();
|
|
7728
|
-
return driver;
|
|
7729
|
-
}
|
|
7730
|
-
function buildDriver(options, persistence, restored) {
|
|
7731
|
-
const minIndependentSources = Math.max(2, options.minIndependentSources ?? 2);
|
|
7732
|
-
const maxQuestionsPerRound = Math.max(1, options.maxQuestionsPerRound ?? 6);
|
|
7733
|
-
const maxClaimsPerSource = Math.max(1, options.maxClaimsPerSource ?? 3);
|
|
7734
|
-
const deterministicFallback = options.deterministicFallback ?? true;
|
|
7735
|
-
const claims = new Map((restored?.claims ?? []).map((claim) => [claim.id, fromRecord(claim)]));
|
|
7736
|
-
const claimEvidence = new Map((restored?.claimEvidence ?? []).map((evidence) => [evidence.id, evidence]));
|
|
7737
|
-
const registeredSources = new Map((restored?.registeredSources ?? []).map((source) => [source.sourceId, source]));
|
|
7738
|
-
const questions = new Map((restored?.questions ?? []).map((question) => [question.id, question]));
|
|
7739
|
-
let rounds = restored?.rounds ?? 0;
|
|
7740
|
-
let preparedRounds = Math.max(rounds, restored?.preparedRounds ?? rounds);
|
|
7741
|
-
let goal = restored?.goal;
|
|
7742
|
-
let lastSteer;
|
|
7743
|
-
if (preparedRounds > rounds) {
|
|
7744
|
-
for (let round = rounds + 1; round <= preparedRounds; round += 1) for (const question of synthesizeDeepQuestions([...claims.values()], round).slice(0, maxQuestionsPerRound)) if (!questions.has(question.id)) questions.set(question.id, question);
|
|
7745
|
-
rounds = preparedRounds;
|
|
7746
|
-
}
|
|
7747
|
-
function toLedger() {
|
|
7748
|
-
return {
|
|
7749
|
-
schemaVersion: 2,
|
|
7750
|
-
id: persistence?.ledgerId ?? "in-memory",
|
|
7751
|
-
...goal === void 0 ? {} : { goal },
|
|
7752
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7753
|
-
rounds,
|
|
7754
|
-
...preparedRounds > rounds ? { preparedRounds } : {},
|
|
7755
|
-
claimEvidence: [...claimEvidence.values()].sort((left, right) => left.id.localeCompare(right.id)),
|
|
7756
|
-
registeredSources: [...registeredSources.values()].sort((left, right) => left.sourceId.localeCompare(right.sourceId)),
|
|
7757
|
-
claims: [...claims.values()].map((claim) => ({
|
|
7758
|
-
...claim,
|
|
7759
|
-
supportingHosts: [...new Set(claim.supportingHosts)].sort(),
|
|
7760
|
-
supportingUris: [...new Set(claim.supportingUris)].sort(),
|
|
7761
|
-
contradicts: [...new Set(claim.contradicts)].sort()
|
|
7762
|
-
})).sort((a, b) => a.id.localeCompare(b.id)),
|
|
7763
|
-
questions: [...questions.values()].map((question) => ({
|
|
7764
|
-
...question,
|
|
7765
|
-
claimIds: [...new Set(question.claimIds)].sort()
|
|
7766
|
-
})).sort((a, b) => a.id.localeCompare(b.id))
|
|
7767
|
-
};
|
|
7768
|
-
}
|
|
7769
|
-
/**
|
|
7770
|
-
* Write this driver's belief state into the stored ledger and adopt the
|
|
7771
|
-
* result.
|
|
7772
|
-
*
|
|
7773
|
-
* It merges rather than overwrites, and then rehydrates from the merged
|
|
7774
|
-
* record, which is the whole of what makes knowledge compound across
|
|
7775
|
-
* workers. Two drivers on one ledger id — a resumed run beside a still-live
|
|
7776
|
-
* one, or two workers researching one goal in parallel — would otherwise each
|
|
7777
|
-
* write a whole record built from what it read before the other wrote, and
|
|
7778
|
-
* the later write would erase the earlier writer's claims. Rehydrating means
|
|
7779
|
-
* a claim another worker corroborated counts toward THIS driver's completion
|
|
7780
|
-
* oracle from the next round onward.
|
|
7781
|
-
*/
|
|
7782
|
-
async function persist() {
|
|
7783
|
-
if (!persistence) return;
|
|
7784
|
-
const mine = toLedger();
|
|
7785
|
-
const merged = await persistence.store.mergeClaimLedger(persistence.ledgerId, (current) => current === null ? mine : mergeClaimLedgers(current, mine));
|
|
7786
|
-
claims.clear();
|
|
7787
|
-
for (const claim of merged.claims) claims.set(claim.id, fromRecord(claim));
|
|
7788
|
-
claimEvidence.clear();
|
|
7789
|
-
for (const evidence of merged.claimEvidence) claimEvidence.set(evidence.id, evidence);
|
|
7790
|
-
registeredSources.clear();
|
|
7791
|
-
for (const source of merged.registeredSources) registeredSources.set(source.sourceId, source);
|
|
7792
|
-
questions.clear();
|
|
7793
|
-
for (const question of merged.questions) questions.set(question.id, question);
|
|
7794
|
-
rounds = Math.max(rounds, merged.rounds);
|
|
7795
|
-
preparedRounds = Math.max(rounds, merged.preparedRounds ?? merged.rounds);
|
|
7796
|
-
goal = merged.goal ?? goal;
|
|
7797
|
-
}
|
|
7798
|
-
async function prepareFold() {
|
|
7799
|
-
if (!persistence) return;
|
|
7800
|
-
preparedRounds = Math.max(preparedRounds, rounds + 1);
|
|
7801
|
-
await persist();
|
|
7802
|
-
}
|
|
7803
|
-
/**
|
|
7804
|
-
* A ledger accumulates evidence FOR a goal. Reusing one id across two goals
|
|
7805
|
-
* merges two runs' beliefs into one corroboration count, which is worse than
|
|
7806
|
-
* losing them, so it fails rather than merging.
|
|
7807
|
-
*/
|
|
7808
|
-
function bindGoal(nextGoal) {
|
|
7809
|
-
if (goal === void 0) {
|
|
7810
|
-
goal = nextGoal;
|
|
7811
|
-
return;
|
|
7812
|
-
}
|
|
7813
|
-
if (goal !== nextGoal) throw new Error(`claim ledger '${persistence?.ledgerId ?? "in-memory"}' accumulated evidence for goal '${goal}' and cannot be reused for '${nextGoal}'`);
|
|
7814
|
-
}
|
|
7815
|
-
function resolveRouter(signal) {
|
|
7816
|
-
if (options.router) return options.router;
|
|
7817
|
-
const configuredSignal = options.router_options?.signal;
|
|
7818
|
-
const requestSignal = configuredSignal && signal ? AbortSignal.any([configuredSignal, signal]) : configuredSignal ?? signal;
|
|
7819
|
-
return createTangleRouterClient({
|
|
7820
|
-
...options.router_options,
|
|
7821
|
-
signal: requestSignal
|
|
7822
|
-
});
|
|
7823
|
-
}
|
|
7824
|
-
function throwIfRouterAborted(signal) {
|
|
7825
|
-
signal?.throwIfAborted();
|
|
7826
|
-
if (!options.router) options.router_options?.signal?.throwIfAborted();
|
|
7827
|
-
}
|
|
7828
|
-
/** Record a claim from a source, growing its independent-source support. */
|
|
7829
|
-
function recordClaim(extracted, sourceUri, round) {
|
|
7830
|
-
const id = claimId(extracted.text);
|
|
7831
|
-
const host = claimSourceHost(sourceUri);
|
|
7832
|
-
const existing = claims.get(id);
|
|
7833
|
-
if (existing) {
|
|
7834
|
-
if (host) existing.supportingHosts.add(host);
|
|
7835
|
-
addUnique(existing.supportingUris, sourceUri);
|
|
7836
|
-
linkContradiction(existing, extracted.contradictsExistingId);
|
|
7837
|
-
return existing;
|
|
7838
|
-
}
|
|
7839
|
-
const tracked = {
|
|
7840
|
-
id,
|
|
7841
|
-
text: extracted.text.trim(),
|
|
7842
|
-
supportingHosts: new Set(host ? [host] : []),
|
|
7843
|
-
supportingUris: [sourceUri],
|
|
7844
|
-
contradicts: /* @__PURE__ */ new Set(),
|
|
7845
|
-
contested: false,
|
|
7846
|
-
firstSeenRound: round
|
|
7847
|
-
};
|
|
7848
|
-
linkContradiction(tracked, extracted.contradictsExistingId);
|
|
7849
|
-
claims.set(id, tracked);
|
|
7850
|
-
return tracked;
|
|
7851
|
-
}
|
|
7852
|
-
/** Persist an extraction observation without treating its source as registered. */
|
|
7853
|
-
function recordEvidence(extracted, sourceVersion, round) {
|
|
7854
|
-
const text = extracted.text.trim();
|
|
7855
|
-
const observedClaimId = claimId(text);
|
|
7856
|
-
const contradictsClaimId = extracted.contradictsExistingId === observedClaimId ? void 0 : extracted.contradictsExistingId;
|
|
7857
|
-
const evidence = {
|
|
7858
|
-
id: claimEvidenceId({
|
|
7859
|
-
claimId: observedClaimId,
|
|
7860
|
-
sourceId: sourceVersion.sourceId,
|
|
7861
|
-
sourceUri: sourceVersion.uri,
|
|
7862
|
-
sourceContentHash: sourceVersion.contentHash,
|
|
7863
|
-
contradictsClaimId
|
|
7864
|
-
}),
|
|
7865
|
-
claimId: observedClaimId,
|
|
7866
|
-
text,
|
|
7867
|
-
sourceId: sourceVersion.sourceId,
|
|
7868
|
-
sourceUri: sourceVersion.uri,
|
|
7869
|
-
sourceContentHash: sourceVersion.contentHash,
|
|
7870
|
-
...contradictsClaimId === void 0 ? {} : { contradictsClaimId },
|
|
7871
|
-
firstSeenRound: round
|
|
7872
|
-
};
|
|
7873
|
-
const existing = claimEvidence.get(evidence.id);
|
|
7874
|
-
if (!existing) {
|
|
7875
|
-
claimEvidence.set(evidence.id, evidence);
|
|
7876
|
-
return evidence;
|
|
7877
|
-
}
|
|
7878
|
-
const merged = {
|
|
7879
|
-
...evidence.firstSeenRound < existing.firstSeenRound || evidence.firstSeenRound === existing.firstSeenRound && evidence.text < existing.text ? evidence : existing,
|
|
7880
|
-
firstSeenRound: Math.min(existing.firstSeenRound, evidence.firstSeenRound)
|
|
7881
|
-
};
|
|
7882
|
-
claimEvidence.set(merged.id, merged);
|
|
7883
|
-
return merged;
|
|
7884
|
-
}
|
|
7885
|
-
/** Materialize evidence for newly confirmed exact source versions into live claims. */
|
|
7886
|
-
function materializeEvidenceFor(sourceVersions) {
|
|
7887
|
-
const evidence = [...claimEvidence.values()].filter((item) => sourceVersions.has(sourceVersionKeyOfEvidence(item)));
|
|
7888
|
-
const texts = [];
|
|
7889
|
-
for (const item of evidence) {
|
|
7890
|
-
recordClaim({
|
|
7891
|
-
text: item.text,
|
|
7892
|
-
contradictsExistingId: item.contradictsClaimId
|
|
7893
|
-
}, item.sourceUri, item.firstSeenRound);
|
|
7894
|
-
texts.push(item.text);
|
|
7895
|
-
}
|
|
7896
|
-
for (const item of evidence) {
|
|
7897
|
-
const claim = claims.get(item.claimId);
|
|
7898
|
-
if (claim) linkContradiction(claim, item.contradictsClaimId);
|
|
7899
|
-
}
|
|
7900
|
-
return texts;
|
|
7901
|
-
}
|
|
7902
|
-
async function commitSources(sources) {
|
|
7903
|
-
const versions = sources.map(sourceVersionOfRecord);
|
|
7904
|
-
const nextRegistered = new Map(registeredSources);
|
|
7905
|
-
const newlyRegistered = /* @__PURE__ */ new Set();
|
|
7906
|
-
for (const version of versions) {
|
|
7907
|
-
const key = researchSourceVersionKey(version);
|
|
7908
|
-
const existing = nextRegistered.get(version.sourceId);
|
|
7909
|
-
if (existing && researchSourceVersionKey(existing) !== key) throw new Error(`registered source '${version.sourceId}' has conflicting immutable content`);
|
|
7910
|
-
if (existing) continue;
|
|
7911
|
-
nextRegistered.set(version.sourceId, version);
|
|
7912
|
-
newlyRegistered.add(key);
|
|
7913
|
-
}
|
|
7914
|
-
registeredSources.clear();
|
|
7915
|
-
for (const [sourceId, version] of nextRegistered) registeredSources.set(sourceId, version);
|
|
7916
|
-
if (newlyRegistered.size > 0) markAddressed(materializeEvidenceFor(newlyRegistered));
|
|
7917
|
-
await persist();
|
|
7918
|
-
}
|
|
7919
|
-
/** Wire a bidirectional contradiction edge and mark BOTH claims contested. */
|
|
7920
|
-
function linkContradiction(claim, otherId) {
|
|
7921
|
-
if (!otherId || otherId === claim.id) return;
|
|
7922
|
-
const other = claims.get(otherId);
|
|
7923
|
-
if (!other) return;
|
|
7924
|
-
claim.contradicts.add(otherId);
|
|
7925
|
-
other.contradicts.add(claim.id);
|
|
7926
|
-
claim.contested = true;
|
|
7927
|
-
other.contested = true;
|
|
7928
|
-
}
|
|
7929
|
-
/** A claim's independent-source count = distinct canonical hosts. */
|
|
7930
|
-
function independentSupport(claim) {
|
|
7931
|
-
return claim.supportingHosts.size;
|
|
7932
|
-
}
|
|
7933
|
-
function isCorroborated(claim) {
|
|
7934
|
-
return independentSupport(claim) >= minIndependentSources;
|
|
7935
|
-
}
|
|
7936
|
-
/** Weakly-supported = NOT corroborated AND NOT contested → an invalidation target. */
|
|
7937
|
-
function isWeak(claim) {
|
|
7938
|
-
return !isCorroborated(claim) && !claim.contested;
|
|
7939
|
-
}
|
|
7940
|
-
function snapshot() {
|
|
7941
|
-
const all = [...claims.values()];
|
|
7942
|
-
const allQuestions = [...questions.values()];
|
|
7943
|
-
return {
|
|
7944
|
-
claims: all,
|
|
7945
|
-
questions: allQuestions,
|
|
7946
|
-
weaklySupported: all.filter(isWeak),
|
|
7947
|
-
corroborated: all.filter(isCorroborated),
|
|
7948
|
-
contested: all.filter((claim) => claim.contested),
|
|
7949
|
-
openQuestions: allQuestions.filter((question) => !question.addressed),
|
|
7950
|
-
rounds
|
|
7951
|
-
};
|
|
7952
|
-
}
|
|
7953
|
-
/**
|
|
7954
|
-
* Mark deep questions addressed when later evidence speaks to them. A question
|
|
7955
|
-
* is addressed once a NEW claim's text shares enough content words with the
|
|
7956
|
-
* question — i.e. the worker brought back evidence on the thing we asked about.
|
|
7957
|
-
* Cheap and deterministic; the LLM is reserved for GENERATING questions, not
|
|
7958
|
-
* grading them, so the oracle stays a non-model check.
|
|
7959
|
-
*/
|
|
7960
|
-
function markAddressed(newClaimTexts) {
|
|
7961
|
-
for (const question of questions.values()) {
|
|
7962
|
-
if (question.addressed) continue;
|
|
7963
|
-
if (question.kind === "contradiction") {
|
|
7964
|
-
if (question.claimIds.some((id) => {
|
|
7965
|
-
const claim = claims.get(id);
|
|
7966
|
-
return claim ? isCorroborated(claim) || claim.contested : false;
|
|
7967
|
-
})) question.addressed = true;
|
|
7968
|
-
continue;
|
|
7969
|
-
}
|
|
7970
|
-
const qWords = contentWordSet(question.text);
|
|
7971
|
-
if (qWords.size === 0) continue;
|
|
7972
|
-
for (const text of newClaimTexts) if (overlapFraction(qWords, contentWordSet(text)) >= .5) {
|
|
7973
|
-
question.addressed = true;
|
|
7974
|
-
break;
|
|
7975
|
-
}
|
|
7976
|
-
}
|
|
7977
|
-
}
|
|
7978
|
-
return {
|
|
7979
|
-
/**
|
|
7980
|
-
* `verifySource` — the per-source hook. This driver does NOT filter for
|
|
7981
|
-
* relevance/dedup (other drivers do that, and the loop already dedups exact
|
|
7982
|
-
* uris). Its job here is to EXTRACT the source's claims and grow the ledger.
|
|
7983
|
-
* It accepts every source that yields at least one extractable claim; it
|
|
7984
|
-
* only rejects a source with NO extractable signal at all (empty/unusable),
|
|
7985
|
-
* because such a source cannot drive the research and pollutes the KB.
|
|
7986
|
-
*/
|
|
7987
|
-
async verifySource(source, ctx) {
|
|
7988
|
-
throwIfRouterAborted(ctx.signal);
|
|
7989
|
-
const sourceSnapshot = snapshotSourceTextInput(source);
|
|
7990
|
-
const goalSnapshot = ctx.goal;
|
|
7991
|
-
const roundSnapshot = ctx.round;
|
|
7992
|
-
const sourceVersion = sourceVersionOfProposal(sourceSnapshot);
|
|
7993
|
-
bindGoal(goalSnapshot);
|
|
7994
|
-
const extracted = await extractClaims(sourceSnapshot, goalSnapshot, ctx.signal);
|
|
7995
|
-
throwIfRouterAborted(ctx.signal);
|
|
7996
|
-
if (extracted.length === 0) return {
|
|
7997
|
-
accept: false,
|
|
7998
|
-
reason: "no extractable claim: source yields nothing to drive the research deeper"
|
|
7999
|
-
};
|
|
8000
|
-
for (const claim of extracted) recordEvidence(claim, sourceVersion, roundSnapshot);
|
|
8001
|
-
if (!persistence) {
|
|
8002
|
-
const key = researchSourceVersionKey(sourceVersion);
|
|
8003
|
-
registeredSources.set(sourceVersion.sourceId, sourceVersion);
|
|
8004
|
-
markAddressed(materializeEvidenceFor(/* @__PURE__ */ new Set([key])));
|
|
8005
|
-
} else {
|
|
8006
|
-
const registered = registeredSources.get(sourceVersion.sourceId);
|
|
8007
|
-
if (registered && researchSourceVersionKey(registered) === researchSourceVersionKey(sourceVersion)) markAddressed(materializeEvidenceFor(/* @__PURE__ */ new Set([researchSourceVersionKey(sourceVersion)])));
|
|
8008
|
-
}
|
|
8009
|
-
await persist();
|
|
8010
|
-
throwIfRouterAborted(ctx.signal);
|
|
8011
|
-
return { accept: true };
|
|
8012
|
-
},
|
|
8013
|
-
/**
|
|
8014
|
-
* `foldGaps` — the DEPTH driver. Runs after the worker's contribution is
|
|
8015
|
-
* applied each round. It builds the next round's steer from (1) the readiness
|
|
8016
|
-
* gaps the loop still reports, (2) freshly generated DEEP sub-questions, and
|
|
8017
|
-
* (3) INVALIDATION challenges for weakly-supported / contradicted claims.
|
|
8018
|
-
*/
|
|
8019
|
-
foldGaps(gaps) {
|
|
8020
|
-
if (persistence && preparedRounds <= rounds) throw new Error("persistent research driver must prepareFold before foldGaps");
|
|
8021
|
-
rounds += 1;
|
|
8022
|
-
const round = rounds;
|
|
8023
|
-
const ledger = [...claims.values()];
|
|
8024
|
-
const invalidationTargets = ledger.filter((claim) => isWeak(claim) || claim.contradicts.size > 0);
|
|
8025
|
-
const deepQuestions = synthesizeDeepQuestions(ledger, round).slice(0, maxQuestionsPerRound);
|
|
8026
|
-
for (const question of deepQuestions) if (!questions.has(question.id)) questions.set(question.id, question);
|
|
8027
|
-
const text = buildSteerText(gaps, deepQuestions, invalidationTargets, minIndependentSources);
|
|
8028
|
-
lastSteer = {
|
|
8029
|
-
round,
|
|
8030
|
-
deepQuestions,
|
|
8031
|
-
invalidationTargets,
|
|
8032
|
-
gaps,
|
|
8033
|
-
text
|
|
8034
|
-
};
|
|
8035
|
-
options.onSteer?.(lastSteer);
|
|
8036
|
-
return text;
|
|
8037
|
-
},
|
|
8038
|
-
researchState: snapshot,
|
|
8039
|
-
isComplete() {
|
|
8040
|
-
const all = [...claims.values()];
|
|
8041
|
-
if (all.length === 0) return false;
|
|
8042
|
-
const everyClaimSettled = all.every((claim) => isCorroborated(claim) || claim.contested);
|
|
8043
|
-
const everyQuestionAddressed = [...questions.values()].every((question) => question.addressed);
|
|
8044
|
-
return everyClaimSettled && everyQuestionAddressed;
|
|
8045
|
-
},
|
|
8046
|
-
lastSteer() {
|
|
8047
|
-
return lastSteer;
|
|
8048
|
-
},
|
|
8049
|
-
checkpoint: persist,
|
|
8050
|
-
prepareFold,
|
|
8051
|
-
commitSources,
|
|
8052
|
-
toLedger
|
|
8053
|
-
};
|
|
8054
|
-
async function extractClaims(source, goal, signal) {
|
|
8055
|
-
const fromLlm = await extractClaimsWithLlm(source, goal, claimsForExtraction(), signal);
|
|
8056
|
-
throwIfRouterAborted(signal);
|
|
8057
|
-
if (fromLlm.length > 0) return fromLlm.slice(0, maxClaimsPerSource);
|
|
8058
|
-
if (deterministicFallback) return deterministicClaims(source).slice(0, maxClaimsPerSource);
|
|
8059
|
-
return [];
|
|
8060
|
-
}
|
|
8061
|
-
function claimsForExtraction() {
|
|
8062
|
-
const known = new Map(claims);
|
|
8063
|
-
for (const evidence of claimEvidence.values()) {
|
|
8064
|
-
if (known.has(evidence.claimId)) continue;
|
|
8065
|
-
known.set(evidence.claimId, {
|
|
8066
|
-
id: evidence.claimId,
|
|
8067
|
-
text: evidence.text,
|
|
8068
|
-
supportingHosts: /* @__PURE__ */ new Set(),
|
|
8069
|
-
supportingUris: [],
|
|
8070
|
-
contradicts: /* @__PURE__ */ new Set(),
|
|
8071
|
-
contested: false,
|
|
8072
|
-
firstSeenRound: evidence.firstSeenRound
|
|
8073
|
-
});
|
|
8074
|
-
}
|
|
8075
|
-
return [...known.values()];
|
|
8076
|
-
}
|
|
8077
|
-
async function extractClaimsWithLlm(source, goal, ledger, signal) {
|
|
8078
|
-
let router;
|
|
8079
|
-
try {
|
|
8080
|
-
router = resolveRouter(signal);
|
|
8081
|
-
} catch (error) {
|
|
8082
|
-
throwIfRouterAborted(signal);
|
|
8083
|
-
if (error?.name === "AbortError") throw error;
|
|
8084
|
-
return [];
|
|
8085
|
-
}
|
|
8086
|
-
const excerpt = source.text.slice(0, 1800);
|
|
8087
|
-
const ledgerLines = ledger.slice(0, 20).map((claim) => `- [${claim.id}] ${claim.text}`).join("\n");
|
|
8088
|
-
const system = `You extract the KEY factual claims a researcher would cite a page for, and flag CONTRADICTIONS with claims already on the ledger. A claim is one concrete, checkable assertion using the page's own terms and numbers. Return ONLY a JSON array; each item is {"claim": string, "contradicts": string|null} where contradicts is the bracketed [id] of a ledger claim this page DIRECTLY contradicts, else null. Return at most ${maxClaimsPerSource} claims. No prose.`;
|
|
8089
|
-
const user = [
|
|
8090
|
-
`Research goal: ${goal}`,
|
|
8091
|
-
`Page title: ${source.title ?? "(none)"}`,
|
|
8092
|
-
ledgerLines ? `Claims already on the ledger:\n${ledgerLines}` : "Ledger is empty.",
|
|
8093
|
-
`Page excerpt:\n${excerpt}`,
|
|
8094
|
-
"Key claims as JSON [{\"claim\": \"...\", \"contradicts\": \"[id]\"|null}]:"
|
|
8095
|
-
].join("\n\n");
|
|
8096
|
-
let raw = "";
|
|
8097
|
-
try {
|
|
8098
|
-
raw = await router.chat([{
|
|
8099
|
-
role: "system",
|
|
8100
|
-
content: system
|
|
8101
|
-
}, {
|
|
8102
|
-
role: "user",
|
|
8103
|
-
content: user
|
|
8104
|
-
}], 1200);
|
|
8105
|
-
} catch (error) {
|
|
8106
|
-
throwIfRouterAborted(signal);
|
|
8107
|
-
if (error?.name === "AbortError") throw error;
|
|
8108
|
-
return [];
|
|
8109
|
-
}
|
|
8110
|
-
throwIfRouterAborted(signal);
|
|
8111
|
-
return parseExtractedClaims(raw, ledger);
|
|
8112
|
-
}
|
|
8113
|
-
/**
|
|
8114
|
-
* Deterministic fallback: pull the leading sentences as candidate claims. Used
|
|
8115
|
-
* only when the model is unavailable, so the driver still drives (and the
|
|
8116
|
-
* offline test runs with no creds). Each sentence becomes a checkable claim.
|
|
8117
|
-
*/
|
|
8118
|
-
function deterministicClaims(source) {
|
|
8119
|
-
return source.text.split(/(?<=[.!?])\s+/).map((sentence) => sentence.trim()).filter((sentence) => contentWordSet(sentence).size >= 3).slice(0, maxClaimsPerSource).map((text) => ({ text }));
|
|
8120
|
-
}
|
|
8121
|
-
/**
|
|
8122
|
-
* Build the four deep-question kinds from the actual ledger claims. This is
|
|
8123
|
-
* intentionally deterministic (not an LLM call): the loop's `foldGaps` contract
|
|
8124
|
-
* is synchronous (`foldGaps(gaps): string`), and a template grounded in the
|
|
8125
|
-
* real claim text gives a faithful, non-fabricated sub-question every round —
|
|
8126
|
-
* comparative, mechanism, gap, and contradiction. Claim EXTRACTION, which runs
|
|
8127
|
-
* inside the awaited `verifySource`, is where the model does the open-ended
|
|
8128
|
-
* work; question generation just interrogates what was extracted.
|
|
8129
|
-
*/
|
|
8130
|
-
function synthesizeDeepQuestions(ledger, round) {
|
|
8131
|
-
if (ledger.length === 0) return [];
|
|
8132
|
-
const out = [];
|
|
8133
|
-
for (const claim of ledger) for (const otherId of claim.contradicts) {
|
|
8134
|
-
const other = ledger.find((entry) => entry.id === otherId);
|
|
8135
|
-
if (!other || other.id < claim.id) continue;
|
|
8136
|
-
out.push(makeQuestion("contradiction", `Does any independent source resolve the contradiction between "${truncate(claim.text)}" and "${truncate(other.text)}"? Find evidence that confirms or refutes one of them.`, [claim.id, other.id], round));
|
|
8137
|
-
}
|
|
8138
|
-
for (const claim of ledger.filter((entry) => !entry.contested)) if (claim.supportingHosts.size < minIndependentSources) out.push(makeQuestion("gap", `Only one independent source supports "${truncate(claim.text)}". What specific corroborating result, dataset, or independent measurement is missing to confirm it?`, [claim.id], round));
|
|
8139
|
-
for (const claim of [...ledger].sort((a, b) => b.supportingHosts.size - a.supportingHosts.size).slice(0, 2)) out.push(makeQuestion("mechanism", `Under what precise condition does "${truncate(claim.text)}" stop holding? Find a source that states the mechanism, limit, or failure mode.`, [claim.id], round));
|
|
8140
|
-
const ranked = [...ledger].sort((a, b) => b.supportingHosts.size - a.supportingHosts.size);
|
|
8141
|
-
if (ranked.length >= 2 && ranked[0] && ranked[1]) out.push(makeQuestion("comparative", `How does the tradeoff in "${truncate(ranked[0].text)}" differ from "${truncate(ranked[1].text)}"? Find a source that compares them directly.`, [ranked[0].id, ranked[1].id], round));
|
|
8142
|
-
const priority = {
|
|
8143
|
-
contradiction: 0,
|
|
8144
|
-
gap: 1,
|
|
8145
|
-
mechanism: 2,
|
|
8146
|
-
comparative: 3
|
|
8147
|
-
};
|
|
8148
|
-
return out.sort((a, b) => priority[a.kind] - priority[b.kind]);
|
|
8149
|
-
}
|
|
8150
|
-
}
|
|
8151
|
-
function sourceVersionOfProposal(source) {
|
|
8152
|
-
const contentHash = sha256(source.text);
|
|
8153
|
-
return {
|
|
8154
|
-
sourceId: textSourceId(source.uri, contentHash),
|
|
8155
|
-
uri: source.uri,
|
|
8156
|
-
contentHash
|
|
8157
|
-
};
|
|
8158
|
-
}
|
|
8159
|
-
function sourceVersionOfRecord(source) {
|
|
8160
|
-
const originalUri = source.metadata?.originalUri;
|
|
8161
|
-
if (typeof originalUri !== "string" || originalUri.length === 0) throw new Error(`registered source '${source.id}' has no originalUri`);
|
|
8162
|
-
if (!/^[a-f0-9]{64}$/.test(source.contentHash)) throw new Error(`registered source '${source.id}' contentHash is not a SHA-256 digest`);
|
|
8163
|
-
const expectedSourceId = textSourceId(originalUri, source.contentHash);
|
|
8164
|
-
if (source.id !== expectedSourceId) throw new Error(`registered source '${source.id}' does not match URI-and-content identity '${expectedSourceId}'`);
|
|
8165
|
-
return {
|
|
8166
|
-
sourceId: source.id,
|
|
8167
|
-
uri: originalUri,
|
|
8168
|
-
contentHash: source.contentHash
|
|
8169
|
-
};
|
|
8170
|
-
}
|
|
8171
|
-
function sourceVersionKeyOfEvidence(evidence) {
|
|
8172
|
-
return researchSourceVersionKey({
|
|
8173
|
-
sourceId: evidence.sourceId,
|
|
8174
|
-
uri: evidence.sourceUri,
|
|
8175
|
-
contentHash: evidence.sourceContentHash
|
|
8176
|
-
});
|
|
8177
|
-
}
|
|
8178
|
-
function makeQuestion(kind, text, claimIds, raisedRound) {
|
|
8179
|
-
return {
|
|
8180
|
-
kind,
|
|
8181
|
-
text,
|
|
8182
|
-
id: deepQuestionId(kind, text),
|
|
8183
|
-
claimIds: [...new Set(claimIds)].sort(),
|
|
8184
|
-
addressed: false,
|
|
8185
|
-
raisedRound
|
|
8186
|
-
};
|
|
8187
|
-
}
|
|
8188
|
-
function fromRecord(claim) {
|
|
8189
|
-
return {
|
|
8190
|
-
...claim,
|
|
8191
|
-
supportingHosts: new Set(claim.supportingHosts),
|
|
8192
|
-
supportingUris: [...claim.supportingUris],
|
|
8193
|
-
contradicts: new Set(claim.contradicts)
|
|
8194
|
-
};
|
|
8195
|
-
}
|
|
8196
|
-
/**
|
|
8197
|
-
* Set-like insert into the array form. The ledger's collections are arrays so
|
|
8198
|
-
* they survive `JSON.stringify`; dedup is enforced here instead of by the type.
|
|
8199
|
-
*/
|
|
8200
|
-
function addUnique(values, value) {
|
|
8201
|
-
if (!values.includes(value)) values.push(value);
|
|
8202
|
-
}
|
|
8203
|
-
const stopwords = /* @__PURE__ */ new Set([
|
|
8204
|
-
"the",
|
|
8205
|
-
"a",
|
|
8206
|
-
"an",
|
|
8207
|
-
"and",
|
|
8208
|
-
"or",
|
|
8209
|
-
"but",
|
|
8210
|
-
"of",
|
|
8211
|
-
"to",
|
|
8212
|
-
"in",
|
|
8213
|
-
"on",
|
|
8214
|
-
"for",
|
|
8215
|
-
"with",
|
|
8216
|
-
"as",
|
|
8217
|
-
"by",
|
|
8218
|
-
"at",
|
|
8219
|
-
"from",
|
|
8220
|
-
"that",
|
|
8221
|
-
"this",
|
|
8222
|
-
"these",
|
|
8223
|
-
"those",
|
|
8224
|
-
"it",
|
|
8225
|
-
"its",
|
|
8226
|
-
"is",
|
|
8227
|
-
"are",
|
|
8228
|
-
"was",
|
|
8229
|
-
"were",
|
|
8230
|
-
"be",
|
|
8231
|
-
"been",
|
|
8232
|
-
"being",
|
|
8233
|
-
"has",
|
|
8234
|
-
"have",
|
|
8235
|
-
"had",
|
|
8236
|
-
"can",
|
|
8237
|
-
"will",
|
|
8238
|
-
"would",
|
|
8239
|
-
"should",
|
|
8240
|
-
"may",
|
|
8241
|
-
"might",
|
|
8242
|
-
"not",
|
|
8243
|
-
"no",
|
|
8244
|
-
"than",
|
|
8245
|
-
"then",
|
|
8246
|
-
"over",
|
|
8247
|
-
"under",
|
|
8248
|
-
"about",
|
|
8249
|
-
"into",
|
|
8250
|
-
"their",
|
|
8251
|
-
"they",
|
|
8252
|
-
"them"
|
|
8253
|
-
]);
|
|
8254
|
-
function contentWordSet(text) {
|
|
8255
|
-
return new Set(normalizeClaimText(text).split(" ").filter((word) => word.length >= 3 && !stopwords.has(word)));
|
|
8256
|
-
}
|
|
8257
|
-
function overlapFraction(a, b) {
|
|
8258
|
-
if (a.size === 0) return 0;
|
|
8259
|
-
let hits = 0;
|
|
8260
|
-
for (const word of a) if (b.has(word)) hits += 1;
|
|
8261
|
-
return hits / a.size;
|
|
8262
|
-
}
|
|
8263
|
-
function truncate(text, max = 140) {
|
|
8264
|
-
const t = text.trim().replace(/\s+/g, " ");
|
|
8265
|
-
return t.length <= max ? t : `${t.slice(0, max - 1)}…`;
|
|
8266
|
-
}
|
|
8267
|
-
/**
|
|
8268
|
-
* Parse the extractor's JSON array of `{claim, contradicts}` items, tolerant of
|
|
8269
|
-
* code fences / surrounding prose. `contradicts` is mapped from a `[id]` token to
|
|
8270
|
-
* a ledger claim id only when that id is actually on the ledger.
|
|
8271
|
-
*/
|
|
8272
|
-
function parseExtractedClaims(raw, ledger) {
|
|
8273
|
-
const text = raw.trim();
|
|
8274
|
-
if (!text) return [];
|
|
8275
|
-
const arrayMatch = text.match(/\[[\s\S]*\]/);
|
|
8276
|
-
if (!arrayMatch) return [];
|
|
8277
|
-
let parsed;
|
|
8278
|
-
try {
|
|
8279
|
-
parsed = JSON.parse(arrayMatch[0]);
|
|
8280
|
-
} catch {
|
|
8281
|
-
return [];
|
|
8282
|
-
}
|
|
8283
|
-
if (!Array.isArray(parsed)) return [];
|
|
8284
|
-
const ledgerIds = new Set(ledger.map((claim) => claim.id));
|
|
8285
|
-
const out = [];
|
|
8286
|
-
for (const item of parsed) {
|
|
8287
|
-
if (!item || typeof item !== "object") continue;
|
|
8288
|
-
const record = item;
|
|
8289
|
-
if (typeof record.claim !== "string" || !record.claim.trim()) continue;
|
|
8290
|
-
const contradictsId = extractBracketId(record.contradicts);
|
|
8291
|
-
out.push({
|
|
8292
|
-
text: record.claim.trim(),
|
|
8293
|
-
contradictsExistingId: contradictsId && ledgerIds.has(contradictsId) ? contradictsId : void 0
|
|
8294
|
-
});
|
|
8295
|
-
}
|
|
8296
|
-
return out;
|
|
8297
|
-
}
|
|
8298
|
-
/** Pull a ledger id out of a `contradicts` field: `"[c_abc]"` or `"c_abc"`. */
|
|
8299
|
-
function extractBracketId(value) {
|
|
8300
|
-
if (typeof value !== "string") return void 0;
|
|
8301
|
-
const trimmed = value.trim();
|
|
8302
|
-
if (!trimmed || trimmed.toLowerCase() === "null") return void 0;
|
|
8303
|
-
const id = (trimmed.match(/\[([^\]]+)\]/)?.[1] ?? trimmed).trim();
|
|
8304
|
-
return id.startsWith("c_") ? id : void 0;
|
|
8305
|
-
}
|
|
8306
|
-
/**
|
|
8307
|
-
* Build the steer string handed to the worker's next prompt. Interleaves the
|
|
8308
|
-
* readiness gaps the loop still reports with the deep sub-questions and the
|
|
8309
|
-
* invalidation challenges — the part that drives DEPTH + VALIDATION.
|
|
8310
|
-
*/
|
|
8311
|
-
function buildSteerText(gaps, deepQuestions, invalidationTargets, minIndependentSources) {
|
|
8312
|
-
const lines = [];
|
|
8313
|
-
lines.push("Do NOT just add more sources. Go DEEPER and VALIDATE. Address the following before adding breadth:");
|
|
8314
|
-
if (deepQuestions.length > 0) {
|
|
8315
|
-
lines.push("", "Deep sub-questions to answer this round:");
|
|
8316
|
-
for (const question of deepQuestions) lines.push(`- (${question.kind}) ${question.text}`);
|
|
8317
|
-
}
|
|
8318
|
-
if (invalidationTargets.length > 0) {
|
|
8319
|
-
lines.push("", `Claims needing corroboration or refutation (each must reach >= ${minIndependentSources} INDEPENDENT sources, or be shown contested):`);
|
|
8320
|
-
for (const claim of invalidationTargets) {
|
|
8321
|
-
const reason = claim.contradicts.size > 0 ? "CONTRADICTED by another source — find evidence that resolves it" : `only ${claim.supportingHosts.size} independent source — find a SECOND, independent corroborating source`;
|
|
8322
|
-
lines.push(`- "${truncate(claim.text)}" — ${reason}`);
|
|
8323
|
-
}
|
|
8324
|
-
}
|
|
8325
|
-
if (gaps.length > 0) {
|
|
8326
|
-
lines.push("", "Readiness gaps still open:");
|
|
8327
|
-
for (const gap of gaps) lines.push(`- (${gap.blocking ? "blocking" : "soft"}) ${gap.description} [${gap.id}]`);
|
|
8328
|
-
}
|
|
8329
|
-
return lines.join("\n");
|
|
8330
|
-
}
|
|
8331
|
-
//#endregion
|
|
8332
|
-
export { AgentMemoryHitSchema, AgentMemoryKindSchema, AgentMemoryLifecycleTimeoutError, AgentMemoryLifecycleUnsafeError, AgentMemoryScopeSchema, AgentMemoryWriteInputSchema, CHECKABLE_RUNG_THRESHOLD, CITES_INVALIDATED_FIELD, ClaimLedgerGoalConflictError, ClaimLedgerMigrationRequiredError, DEADLINE_EXIT_CODE, DEFAULT_KNOWLEDGE_BRIEF_LIMIT, DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, DEFAULT_PAGES_DIRECTORY, DeepQuestionSchema, FileSystemKbStore, FileSystemSearchProvider, HindsightOperationUnknownError, INDUSTRY_MEMORY_BENCHMARKS, INDUSTRY_RAG_BENCHMARKS, IRS_DIMENSION_HINTS, KB_CLAIM_LEDGER_DIR, KB_EVENTS_PATH, KB_INDEX_PATH, KB_STORE_DIR, KNOWLEDGE_EVENT_TYPES, KNOWLEDGE_PROMOTIONS_DIRECTORY, KNOWLEDGE_PROMOTION_DIGEST_ALGORITHM, KNOWLEDGE_PROMOTION_SCHEMA_VERSION, KNOWLEDGE_RECEIPT_DIGEST_ALGORITHM, KNOWLEDGE_SEARCH_RETRIEVER_ID, KNOWLEDGE_USE_RECEIPT_SCHEMA_VERSION, KnowledgeBaseCandidateSchema, KnowledgeCitationAuditError, KnowledgeCitationResolutionError, KnowledgeDuplicateIntakeError, KnowledgeEventSchema, KnowledgeGraphEdgeSchema, KnowledgeGraphNodeSchema, KnowledgeImprovementCandidateRefSchema, KnowledgeImprovementEvidenceSchema, KnowledgeImprovementRunStateSchema, KnowledgeIndexSchema, KnowledgePageInvalidationSchema, KnowledgePageSchema, KnowledgePromotionError, KnowledgeProposalParseError, KnowledgeRelationGraphError, KnowledgeRelationGraphSchema, KnowledgeRelationNodeSchema, KnowledgeRelationSchema, KnowledgeVisibilityUnavailableError, MAX_RESPONSE_BYTES, MIN_REQUEST_GAP_MS, MemoryKbStore, POLITE_USER_AGENT, READINESS_SPEC_DEFAULTS, RUN_LINEAGE_BASENAME, ResearchClaimEvidenceSchema, ResearchClaimLedgerSchema, ResearchClaimRecordSchema, ResearchSourceVersionSchema, RouterError, SCAFFOLD_PAGE_BASENAMES, SourceAnchorSchema, SourceRecordSchema, UncheckableClaimError, WIKILINK_REGEX, __resetHttpThrottle, acquireAgentMemoryRunLease, addSourcePath, addSourceText, agentMemorySequenceJudge, applyKnowledgeWriteBlocks, applyKnowledgeWriteBlocksFile, applyRetrievalHoldout, applySessionStickyRetrievalHoldout, assertClaimLedgerId, assertCurrentRunCitationsResolved, assertDeepQuestionIntegrity, assertGradeableEvidence, assertKnowledgeCitationAudit, assertKnowledgeCitationsResolved, assertKnowledgeRetrievalMatchesVisibility, assertKnowledgeRetrievalMatchesVisibilityArtifact, assertKnowledgeWriteIntake, assertResearchClaimEvidenceIntegrity, assertResearchClaimLedgerIntegrity, assertRunScopedCitationsResolved, assertTrackedClaimIntegrity, auditCurrentRunCitations, auditKnowledgeCitations, buildAgentMemorySequenceScenarios, buildAgentMemorySequencesFromBenchmarkCases, buildEvalKnowledgeBundle, buildFirstPartyMemoryLifecycleBenchmarkCases, buildIndustryMemoryBenchmarkSmokeCases, buildIndustryRagBenchmarkSmokeCases, buildKnowledgeBenchmarkScenarios, buildKnowledgeBrief, buildKnowledgeGraph, buildKnowledgeIndex, buildKnowledgeLexicalIndex, buildKnowledgeRelationGraph, buildRetrievalBenchmarkCasesFromQrels, buildRetrievalEvalDispatch, calibrateRagAnswerJudge, canonicalPathsEqual, canonicalRelativeWithinRoot, canonicalizeUrl, chunkMarkdown, citedClaimKey, citedClaimOf, claimCheckKey, claimEvidenceId, claimId, claimSourceHost, compareAgentMemoryLearning, contentKey, createAdaptiveResearchDriver, createAgentMemoryBranch, createClaimDecorator, createClaimGroundingVerifier, createCollectionResearchDriver, createCornellLiiSource, createD1FreshnessStoreStub, createFileRunLineageAuthority, createFileSystemFreshnessStore, createFileSystemSearchProvider, createGraphitiMemoryAdapter, createHindsightMemoryAdapter, createInMemoryBenchmarkAdapter, createIrsPublicationsSource, createKnowledgeControlLoopAdapter, createKnowledgeEvent, createKnowledgeRetrievalDisposition, createKnowledgeRetrievalReceipt, createKnowledgeTools, createKnowledgeUseReceipt, createKnowledgeVisibilitySnapshot, createLocalDiscoveryDispatcher, createMem0MemoryAdapter, createMemoryExecutionPool, createNeo4jAgentMemoryAdapter, createNoopMemoryBenchmarkAdapter, createPersistentResearchDrivingDriver, createPlayMemoryTools, createQmdKnowledgeTools, createQmdSearchProvider, createRagAnswerQualityHook, createResearchDrivingDriver, createRunScopedStores, createStateSosSource, createTangleRouterClient, createVerifyingResearchDriver, createWebResearchWorker, decodeKnowledgeVisibilitySnapshot, deepQuestionId, defaultGetMemoryContext, defineReadinessSpec, detectChanges, detectNearDuplicatePages, deterministicRng, diagnoseRagAnswerFailure, emitRetrievalHoldoutBypass, emptyClaimLedger, encodeKnowledgeVisibilitySnapshot, evaluateKnowledgeBaseReadiness, explainKnowledgeTarget, extractLinks, extractWikilinks, firstMatch, forkAgentMemoryBranchSnapshot, formatFrontmatter, formatKnowledgeCitationReference, formatKnowledgeInvalidationProposal, fromAgentCandidateKnowledgeRef, gradeClaims, gradeCompanyAgainstText, gradeFactAgainstText, gradeFor, graphitiMemoryAdapterIdentity, groundClaimInText, hashKnowledgeBase, hindsightMemoryBankId, htmlToText, improveKnowledgeBase, improveSelectedKnowledgeCandidate, initKnowledgeBase, innerHtmlById, inspectKnowledgeIndex, inspectPendingKnowledgeMutation, investmentThesisSet, isKernelAnchoredPath, isKnowledgeMemoryBenchmarkCase, isKnowledgeMutationHeld, isKnowledgePagePath, isMissingFile, isReachable, isSafeKnowledgePath, isScaffoldPath, jsonCandidateCodec, jsonObjectCandidateCodec, kbIndexToText, knowledgeBenchmarkJudge, knowledgeCitationAuditFindings, knowledgeImprovementCandidateRef, knowledgeImprovementRunDir, knowledgeImprovementRunId, knowledgePageDigest, knowledgePageFromMarkdown, knowledgePageRelations, knowledgeReleaseReport, knowledgeVisibilityArtifactRef, layoutFor, lensDistribution, linkClaimContradictions, lintCurrentRunCitations, lintKnowledgeIndex, listRegularFilesWithinRoot, loadKnowledgeImprovementActivationResult, loadKnowledgeImprovementEvents, loadKnowledgeImprovementState, loadKnowledgePages, loadKnowledgePromotionRecord, loadSourceRegistry, looksLikeBlockPage, materialFactsSurfaced, materialFactsSurfacedInText, materializeRegisteredClaimEvidence, mediaTypeFor, mem0MemoryAdapterIdentity, memoryHitToSourceRecord, memoryRecoveryDelayMs, memoryWriteResultToSourceRecord, mergeClaimLedgers, mergeTrackedClaims, neighbors, normalizeClaimText, normalizeExternalRagScores, normalizeKnowledgeStateScope, normalizeLinkTarget, normalizePageText, normalizePagesDirectory, optimizeKnowledgeBasePolicy, originatedPages, parseFrontmatter, parseKnowledgeBenchmarkJsonl, parseKnowledgeBenchmarkQrels, parseKnowledgeCitationReference, parseKnowledgeWriteBlocks, partitionRetrievalScenarios, planInvalidationPropagation, politeFetch, promoteKnowledgeCandidate, promoteRunScopedPages, proposeFromFinding, proposeFromFindings, ragAnswerQualityJudge, readRegularFileNoFollow, readRegularFileWithinRoot, reciprocalRankFusion, recoverPendingKnowledgeMutation, relativeWithinRoot, removeDurable, renameDurable, renderKnowledgeBenchmarkReportMarkdown, renderMemoryContext, researchSourceVersionKey, resetRetrievalHoldoutRegistry, resolveKnowledgeCitation, resolveKnowledgeCitations, resolveMemoryCleanupTimeoutMs, resolveRunScopedCitation, resolveRunScopedCitations, respondToIndustryMemoryBenchmarkSmokeCase, respondToIndustryRagBenchmarkSmokeCase, restoreKnowledgeCandidateBaseline, retrievalConfigFromSurface, retrievalConfigSurface, retrievalHoldoutConfigHash, retrievalRecallJudge, runAgentMemoryExperiment, runAgentMemoryImprovement, runAgentMemoryLearningExperiment, runBoundedMemoryLifecycle, runDiscoveryLoop, runInKnowledgeMutationScope, runInvestmentThesisTask, runKnowledgeBenchmarkSuite, runKnowledgeResearchLoop, runMemoryAdapterBenchmark, runRagKnowledgeImprovementLoop, runRagOptimization, runRetrievalImprovementLoop, runSerializedKnowledgeOptimization, runVerifiedResearchLoop, scenarioContentFingerprint, scoreBm25, scoreKnowledgeBaseIndex, scoreKnowledgeBenchmarkArtifact, scoreMemoryBenchmarkArtifact, scoreRagAnswerArtifact, scoreRetrievalArtifact, searchKnowledge, searchKnowledgePages, sha256, sleepForMemoryRecovery, slugify, snapshotSourceTextInput, sourceMatchesGaps, sourceRegistryPath, stableId, stripFrontmatter, summarizeKnowledgeBenchmarkCampaign, syncDirectory, textSourceAdapter, textSourceId, thesisReadinessSpecs, toAgentCandidateKnowledgeRef, toDeepEvalTestCases, toOffPolicyTrajectory, toRagCheckerRecords, toRagasEvaluationRows, toTruLensRecords, tokenizeQuery, tokenizeText, totalMaterialFacts, triageSource, validateKnowledgeIndex, verdictFor, verifyGradeableEvidence, verifyKnowledgeRetrievalDisposition, verifyKnowledgeRetrievalReceipt, verifyKnowledgeUseReceipt, verifyKnowledgeVisibilitySnapshot, walk, withCitedClaim, withKnowledgeImprovementCandidate, withKnowledgeImprovementComparison, withKnowledgeMutation, withKnowledgeRead, withSafeDescendant, withSafeDirectory, writeFileDurable, writeFileDurableWithinRoot, writeJson, writeJsonDurable, writeJsonDurableWithinRoot, writeKnowledgeIndex, writeSourceRegistry };
|
|
5240
|
+
export { AgentMemoryHitSchema, AgentMemoryKindSchema, AgentMemoryLifecycleTimeoutError, AgentMemoryLifecycleUnsafeError, AgentMemoryScopeSchema, AgentMemoryWriteInputSchema, CHECKABLE_RUNG_THRESHOLD, ClaimLedgerGoalConflictError, ClaimLedgerMigrationRequiredError, DEADLINE_EXIT_CODE, DEFAULT_KNOWLEDGE_BRIEF_LIMIT, DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, DEFAULT_PAGES_DIRECTORY, DeepQuestionSchema, FileSystemKbStore, FileSystemSearchProvider, HindsightOperationUnknownError, KB_CLAIM_LEDGER_DIR, KB_EVENTS_PATH, KB_INDEX_PATH, KB_STORE_DIR, KNOWLEDGE_EVENT_TYPES, KNOWLEDGE_PROMOTIONS_DIRECTORY, KNOWLEDGE_PROMOTION_DIGEST_ALGORITHM, KNOWLEDGE_PROMOTION_SCHEMA_VERSION, KNOWLEDGE_RECEIPT_DIGEST_ALGORITHM, KNOWLEDGE_SEARCH_RETRIEVER_ID, KNOWLEDGE_USE_RECEIPT_SCHEMA_VERSION, KnowledgeBaseCandidateSchema, KnowledgeCitationAuditError, KnowledgeCitationResolutionError, KnowledgeDuplicateIntakeError, KnowledgeEventSchema, KnowledgeGraphEdgeSchema, KnowledgeGraphNodeSchema, KnowledgeImprovementCandidateRefSchema, KnowledgeImprovementEvidenceSchema, KnowledgeImprovementRunStateSchema, KnowledgeIndexSchema, KnowledgePageInvalidationSchema, KnowledgePageSchema, KnowledgePromotionError, KnowledgeProposalParseError, KnowledgeRelationSchema, KnowledgeVisibilityUnavailableError, MemoryKbStore, READINESS_SPEC_DEFAULTS, RUN_LINEAGE_BASENAME, ResearchClaimEvidenceSchema, ResearchClaimLedgerSchema, ResearchClaimRecordSchema, ResearchSourceVersionSchema, SCAFFOLD_PAGE_BASENAMES, SourceAnchorSchema, SourceRecordSchema, UncheckableClaimError, WIKILINK_REGEX, addSourcePath, addSourceText, applyKnowledgeWriteBlocks, applyKnowledgeWriteBlocksFile, applyRetrievalHoldout, applySessionStickyRetrievalHoldout, assertClaimLedgerId, assertCurrentRunCitationsResolved, assertDeepQuestionIntegrity, assertGradeableEvidence, assertKnowledgeCitationAudit, assertKnowledgeCitationsResolved, assertKnowledgeRetrievalMatchesVisibility, assertKnowledgeRetrievalMatchesVisibilityArtifact, assertKnowledgeWriteIntake, assertResearchClaimEvidenceIntegrity, assertResearchClaimLedgerIntegrity, assertRunScopedCitationsResolved, assertTrackedClaimIntegrity, auditCurrentRunCitations, auditKnowledgeCitations, buildEvalKnowledgeBundle, buildKnowledgeBrief, buildKnowledgeGraph, buildKnowledgeIndex, buildKnowledgeLexicalIndex, buildRetrievalEvalDispatch, canonicalPathsEqual, canonicalRelativeWithinRoot, chunkMarkdown, claimCheckKey, claimEvidenceId, claimId, claimSourceHost, createAgentMemoryBranch, createFileRunLineageAuthority, createFileSystemSearchProvider, createGraphitiMemoryAdapter, createHindsightMemoryAdapter, createInMemoryBenchmarkAdapter, createKnowledgeControlLoopAdapter, createKnowledgeEvent, createKnowledgeRetrievalDisposition, createKnowledgeRetrievalReceipt, createKnowledgeTools, createKnowledgeUseReceipt, createKnowledgeVisibilitySnapshot, createLocalDiscoveryDispatcher, createMem0MemoryAdapter, createMemoryExecutionPool, createNeo4jAgentMemoryAdapter, createNoopMemoryBenchmarkAdapter, createPlayMemoryTools, createQmdKnowledgeTools, createQmdSearchProvider, createRunScopedStores, decodeKnowledgeVisibilitySnapshot, deepQuestionId, defaultGetMemoryContext, defineReadinessSpec, detectNearDuplicatePages, deterministicRng, diagnoseRagAnswerFailure, emitRetrievalHoldoutBypass, emptyClaimLedger, encodeKnowledgeVisibilitySnapshot, evaluateKnowledgeBaseReadiness, explainKnowledgeTarget, extractWikilinks, forkAgentMemoryBranchSnapshot, formatFrontmatter, formatKnowledgeCitationReference, fromAgentCandidateKnowledgeRef, gradeClaims, gradeFor, graphitiMemoryAdapterIdentity, groundClaimInText, hashKnowledgeBase, hindsightMemoryBankId, improveKnowledgeBase, initKnowledgeBase, inspectKnowledgeIndex, inspectPendingKnowledgeMutation, isKernelAnchoredPath, isKnowledgeMutationHeld, isKnowledgePagePath, isMissingFile, isSafeKnowledgePath, isScaffoldPath, jsonCandidateCodec, jsonObjectCandidateCodec, knowledgeImprovementCandidateRef, knowledgeImprovementRunDir, knowledgeImprovementRunId, knowledgePageDigest, knowledgePageFromMarkdown, knowledgePageRelations, knowledgeReleaseReport, knowledgeVisibilityArtifactRef, layoutFor, linkClaimContradictions, lintKnowledgeIndex, listRegularFilesWithinRoot, loadKnowledgeImprovementActivationResult, loadKnowledgeImprovementEvents, loadKnowledgeImprovementState, loadKnowledgePages, loadKnowledgePromotionRecord, loadSourceRegistry, materializeRegisteredClaimEvidence, mediaTypeFor, mem0MemoryAdapterIdentity, memoryHitToSourceRecord, memoryRecoveryDelayMs, memoryWriteResultToSourceRecord, mergeClaimLedgers, mergeTrackedClaims, normalizeClaimText, normalizeExternalRagScores, normalizeKnowledgeStateScope, normalizeLinkTarget, normalizePageText, normalizePagesDirectory, originatedPages, parseFrontmatter, parseKnowledgeCitationReference, parseKnowledgeWriteBlocks, partitionRetrievalScenarios, promoteKnowledgeCandidate, promoteRunScopedPages, proposeFromFinding, proposeFromFindings, ragAnswerQualityJudge, readRegularFileNoFollow, readRegularFileWithinRoot, reciprocalRankFusion, recoverPendingKnowledgeMutation, relativeWithinRoot, removeDurable, renameDurable, renderMemoryContext, researchSourceVersionKey, resetRetrievalHoldoutRegistry, resolveKnowledgeCitation, resolveKnowledgeCitations, resolveMemoryCleanupTimeoutMs, resolveRunScopedCitation, resolveRunScopedCitations, restoreKnowledgeCandidateBaseline, retrievalConfigFromSurface, retrievalConfigSurface, retrievalHoldoutConfigHash, retrievalRecallJudge, runBoundedMemoryLifecycle, runDiscoveryLoop, runInKnowledgeMutationScope, runKnowledgeResearchLoop, runRagKnowledgeImprovementLoop, runRagOptimization, runRetrievalImprovementLoop, runSerializedKnowledgeOptimization, scenarioContentFingerprint, scoreBm25, scoreKnowledgeBaseIndex, scoreRagAnswerArtifact, scoreRetrievalArtifact, searchKnowledge, searchKnowledgePages, sha256, sleepForMemoryRecovery, slugify, snapshotSourceTextInput, sourceRegistryPath, stableId, stripFrontmatter, syncDirectory, textSourceAdapter, textSourceId, toAgentCandidateKnowledgeRef, toDeepEvalTestCases, toOffPolicyTrajectory, toRagCheckerRecords, toRagasEvaluationRows, toTruLensRecords, tokenizeQuery, tokenizeText, validateKnowledgeIndex, verdictFor, verifyGradeableEvidence, verifyKnowledgeRetrievalDisposition, verifyKnowledgeRetrievalReceipt, verifyKnowledgeUseReceipt, verifyKnowledgeVisibilitySnapshot, withKnowledgeImprovementCandidate, withKnowledgeImprovementComparison, withKnowledgeMutation, withKnowledgeRead, withSafeDescendant, withSafeDirectory, writeFileDurable, writeFileDurableWithinRoot, writeJson, writeJsonDurable, writeJsonDurableWithinRoot, writeKnowledgeIndex, writeSourceRegistry };
|
|
8333
5241
|
|
|
8334
5242
|
//# sourceMappingURL=index.js.map
|