@tangle-network/agent-knowledge 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +250 -74
- package/dist/benchmarks/index.d.ts +5 -0
- package/dist/benchmarks/index.js +52 -0
- package/dist/chunk-DQ3PDMDP.js +115 -0
- package/dist/chunk-DQ3PDMDP.js.map +1 -0
- package/dist/chunk-ULXZI235.js +1855 -0
- package/dist/chunk-ULXZI235.js.map +1 -0
- package/dist/{chunk-U6KQ4FS3.js → chunk-VN2OGUUP.js} +306 -40
- package/dist/chunk-VN2OGUUP.js.map +1 -0
- package/dist/{chunk-WWY5FTKQ.js → chunk-W6VWYTNH.js} +10 -115
- package/dist/chunk-W6VWYTNH.js.map +1 -0
- package/dist/chunk-XVU5FFQA.js +49 -0
- package/dist/chunk-XVU5FFQA.js.map +1 -0
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/index-Cj5jRXOz.d.ts +426 -0
- package/dist/index.d.ts +1036 -169
- package/dist/index.js +3372 -558
- package/dist/index.js.map +1 -1
- package/dist/memory/index.d.ts +5 -70
- package/dist/memory/index.js +20 -4
- package/dist/types-CjqPcTTK.d.ts +319 -0
- package/docs/architecture.md +7 -5
- package/docs/eval/investment-material-facts.md +252 -0
- package/docs/eval/rag-eval-roadmap.md +119 -0
- package/docs/results/investment-thesis.md +306 -0
- package/docs/results/research-driving.md +265 -0
- package/docs/two-agent-research-ab.md +121 -10
- package/package.json +8 -9
- package/dist/chunk-MU5CEOGE.js +0 -387
- package/dist/chunk-MU5CEOGE.js.map +0 -1
- package/dist/chunk-U6KQ4FS3.js.map +0 -1
- package/dist/chunk-WWY5FTKQ.js.map +0 -1
- package/dist/profiles/index.d.ts +0 -194
- package/dist/profiles/index.js +0 -11
- package/docs/recursive-research-leaf.md +0 -86
- package/docs/supervisor-profiles.md +0 -295
- /package/dist/{profiles → benchmarks}/index.js.map +0 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { S as SourceRecord, c as KnowledgeIndex, d as KnowledgeSearchResult, e as SourceRegistry, f as KnowledgeEvent, g as KnowledgeEventType, h as KnowledgePage, b as KnowledgeGraph, i as KnowledgeLintFinding, j as KnowledgeWriteBlock, k as KnowledgeClaim, l as KnowledgeRelease, m as KnowledgeWriteParseResult } from './types-C17sAROL.js';
|
|
2
2
|
export { C as ClaimRef, n as KnowledgeBaseCandidate, K as KnowledgeGraphEdge, a as KnowledgeGraphNode, o as KnowledgeId, p as KnowledgePolicy, q as KnowledgeRelation, r as KnowledgeUnit, s as SourceAnchor } from './types-C17sAROL.js';
|
|
3
|
-
import { KnowledgeRequirementCategory, KnowledgeAcquisitionMode, KnowledgeImportance, KnowledgeFreshness, KnowledgeSensitivity, KnowledgeRequirement, KnowledgeBundle, KnowledgeReadinessReport, UserQuestion, DataAcquisitionPlan, AnalystSeverity, AnalystFinding, RunRecord, ReleaseTraceEvidence, GateDecision, ReleaseConfidenceScorecard
|
|
3
|
+
import { KnowledgeRequirementCategory, KnowledgeAcquisitionMode, KnowledgeImportance, KnowledgeFreshness, KnowledgeSensitivity, KnowledgeRequirement, KnowledgeBundle, KnowledgeReadinessReport, UserQuestion, DataAcquisitionPlan, ControlRuntimeConfig, ControlEvalResult, AnalystSeverity, AnalystFinding, RunRecord, ReleaseTraceEvidence, GateDecision, ReleaseConfidenceScorecard } from '@tangle-network/agent-eval';
|
|
4
|
+
import { R as RunRetrievalImprovementLoopResult, a as RunRetrievalImprovementLoopOptions, b as RetrievalConfig } from './index-Cj5jRXOz.js';
|
|
5
|
+
export { B as BuildRetrievalBenchmarkCasesFromQrelsOptions, c as BuildRetrievalEvalDispatchOptions, d as BuildRetrievalParameterCandidatesOptions, I as INDUSTRY_MEMORY_BENCHMARKS, e as INDUSTRY_RAG_BENCHMARKS, K as KnowledgeAnswerBenchmarkCase, f as KnowledgeAnswerBenchmarkTaskKind, g as KnowledgeBenchmarkArtifact, h as KnowledgeBenchmarkCase, i as KnowledgeBenchmarkCaseBase, j as KnowledgeBenchmarkDistribution, k as KnowledgeBenchmarkEvaluation, l as KnowledgeBenchmarkFamily, m as KnowledgeBenchmarkReport, n as KnowledgeBenchmarkResponder, o as KnowledgeBenchmarkScenario, p as KnowledgeBenchmarkSliceSummary, q as KnowledgeBenchmarkSource, r as KnowledgeBenchmarkSpec, s as KnowledgeBenchmarkSplit, t as KnowledgeBenchmarkTaskKind, u as KnowledgeClaimMatcher, v as KnowledgeMemoryBenchmarkCase, w as KnowledgeMemoryBenchmarkTaskKind, x as KnowledgeMemoryEvent, y as KnowledgeMemoryFactMatcher, z as KnowledgeRetrievalBenchmarkCase, A as KnowledgeRetrievalBenchmarkQrel, C as KnowledgeRetrievalBenchmarkQuery, M as MemoryAdapterBenchmarkCandidate, D as MemoryAdapterBenchmarkRankingRow, E as RetrievalEvalArtifact, F as RetrievalEvalRetriever, G as RetrievalEvalRetrieverInput, H as RetrievalEvalRetrieverResult, J as RetrievalEvalScenario, L as RetrievalGoldTarget, N as RetrievalMetricSummary, O as RetrievalMetricWeights, P as RetrievalParameterSearchSpace, Q as RetrievalParameterSweepProposerOptions, S as RetrievalRecallJudgeOptions, T as RetrievedKnowledgeHit, U as RetrievedSourceSpan, V as RunKnowledgeBenchmarkSuiteOptions, W as RunKnowledgeBenchmarkSuiteResult, X as RunMemoryAdapterBenchmarkOptions, Y as RunMemoryAdapterBenchmarkResult, Z as buildFirstPartyMemoryLifecycleBenchmarkCases, _ as buildIndustryMemoryBenchmarkSmokeCases, $ as buildIndustryRagBenchmarkSmokeCases, a0 as buildKnowledgeBenchmarkScenarios, a1 as buildRetrievalBenchmarkCasesFromQrels, a2 as buildRetrievalEvalDispatch, a3 as buildRetrievalParameterCandidates, a4 as createInMemoryBenchmarkAdapter, a5 as createMemoryAdapterBenchmarkResponder, a6 as createNoopMemoryBenchmarkAdapter, a7 as isKnowledgeMemoryBenchmarkCase, a8 as knowledgeBenchmarkJudge, a9 as parseKnowledgeBenchmarkJsonl, aa as parseKnowledgeBenchmarkQrels, ab as renderKnowledgeBenchmarkReportMarkdown, ac as respondToIndustryMemoryBenchmarkSmokeCase, ad as respondToIndustryRagBenchmarkSmokeCase, ae as retrievalConfigFromSurface, af as retrievalConfigSurface, ag as retrievalParameterSweepProposer, ah as retrievalRecallJudge, ai as runKnowledgeBenchmarkSuite, aj as runMemoryAdapterBenchmark, ak as runRetrievalImprovementLoop, al as scoreKnowledgeBenchmarkArtifact, am as scoreMemoryBenchmarkArtifact, an as scoreRetrievalArtifact, ao as summarizeKnowledgeBenchmarkCampaign } from './index-Cj5jRXOz.js';
|
|
4
6
|
import { KnowledgeFragment } from './sources/index.js';
|
|
5
7
|
export { CornellLiiSelector, CornellLiiSourceOptions, FetchOpts, FragmentProvenance, IRS_DIMENSION_HINTS, IrsPublicationsSourceOptions, KnowledgeSource, MAX_RESPONSE_BYTES, MIN_REQUEST_GAP_MS, POLITE_USER_AGENT, PoliteFetchOptions, PoliteFetchResult, StateSosEntity, StateSosSourceConfig, __resetHttpThrottle, createCornellLiiSource, createIrsPublicationsSource, createStateSosSource, extractLinks, firstMatch, htmlToText, innerHtmlById, looksLikeBlockPage, politeFetch } from './sources/index.js';
|
|
6
|
-
|
|
7
|
-
|
|
8
|
+
import { JsonValue, Scenario, JudgeConfig } from '@tangle-network/agent-eval/campaign';
|
|
9
|
+
export { AgentMemoryHitSchema, AgentMemoryKindSchema, AgentMemoryScopeSchema, AgentMemoryWriteInputSchema, Neo4jAgentMemoryAdapterOptions, createNeo4jAgentMemoryAdapter, defaultGetMemoryContext, memoryHitToSourceRecord, memoryWriteResultToSourceRecord, renderMemoryContext } from './memory/index.js';
|
|
10
|
+
export { A as AgentMemoryAdapter, b as AgentMemoryContext, c as AgentMemoryHit, d as AgentMemoryKind, a as AgentMemoryScope, e as AgentMemorySearchOptions, f as AgentMemoryWriteInput, g as AgentMemoryWriteResult, R as RetrievalHoldoutBypassReason, h as RetrievalHoldoutCallContext, i as RetrievalHoldoutConfig, j as RetrievalHoldoutEligibleItem, k as RetrievalHoldoutEvent, l as RetrievalHoldoutOffPolicyOptions, m as RetrievalHoldoutOffPolicyResult, n as RetrievalHoldoutResult, o as RetrievalHoldoutSessionState, p as RetrievalHoldoutSessionSummary, q as applyRetrievalHoldout, r as applySessionStickyRetrievalHoldout, s as deterministicRng, t as emitRetrievalHoldoutBypass, u as resetRetrievalHoldoutRegistry, v as retrievalHoldoutConfigHash, w as toOffPolicyTrajectory } from './types-CjqPcTTK.js';
|
|
8
11
|
import { z } from 'zod';
|
|
12
|
+
import '@tangle-network/agent-eval/rl';
|
|
9
13
|
|
|
10
14
|
interface SourceAdapterInput {
|
|
11
15
|
uri: string;
|
|
@@ -248,7 +252,7 @@ type SourceVerdict = {
|
|
|
248
252
|
};
|
|
249
253
|
/** The worker: primary research targeting the round's gaps. */
|
|
250
254
|
type ResearchWorker = (ctx: WorkerResearchContext) => Promise<ResearchContribution> | ResearchContribution;
|
|
251
|
-
interface
|
|
255
|
+
interface VerifiedResearchLoopOptions {
|
|
252
256
|
root: string;
|
|
253
257
|
goal: string;
|
|
254
258
|
worker: ResearchWorker;
|
|
@@ -268,9 +272,9 @@ interface TwoAgentResearchLoopOptions {
|
|
|
268
272
|
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
269
273
|
sourceOptions?: Pick<AddSourceOptions, 'adapters' | 'now'>;
|
|
270
274
|
signal?: AbortSignal;
|
|
271
|
-
onRound?: (round:
|
|
275
|
+
onRound?: (round: VerifiedResearchRound) => Promise<void> | void;
|
|
272
276
|
}
|
|
273
|
-
interface
|
|
277
|
+
interface VerifiedResearchRound {
|
|
274
278
|
round: number;
|
|
275
279
|
/** Gaps reported at the START of the round (what the worker targeted). */
|
|
276
280
|
gaps: KnowledgeGap[];
|
|
@@ -291,14 +295,14 @@ interface TwoAgentResearchRound {
|
|
|
291
295
|
driver?: string;
|
|
292
296
|
};
|
|
293
297
|
}
|
|
294
|
-
interface
|
|
298
|
+
interface VerifiedResearchLoopResult {
|
|
295
299
|
root: string;
|
|
296
300
|
goal: string;
|
|
297
301
|
rounds: number;
|
|
298
302
|
ready: boolean;
|
|
299
303
|
index: KnowledgeIndex;
|
|
300
304
|
readiness?: EvalKnowledgeBundleBuildResult;
|
|
301
|
-
steps:
|
|
305
|
+
steps: VerifiedResearchRound[];
|
|
302
306
|
}
|
|
303
307
|
/**
|
|
304
308
|
* Two-agent (driver + worker) sibling of `runKnowledgeResearchLoop`.
|
|
@@ -322,16 +326,24 @@ interface TwoAgentResearchLoopResult {
|
|
|
322
326
|
* `applyKnowledgeWriteBlocks`, `buildEvalKnowledgeBundle` (the readiness gate),
|
|
323
327
|
* and `searchKnowledge` — and reinvents none of them.
|
|
324
328
|
*/
|
|
325
|
-
declare function
|
|
329
|
+
declare function runVerifiedResearchLoop(options: VerifiedResearchLoopOptions): Promise<VerifiedResearchLoopResult>;
|
|
326
330
|
/**
|
|
327
331
|
* Helper for verifiers: does the candidate source's text/title overlap any page
|
|
328
332
|
* the readiness search returns for a gap query? A cheap relevance heuristic the
|
|
329
333
|
* driver can compose into `verifySource` (real verifiers can do more).
|
|
330
334
|
*/
|
|
331
335
|
declare function sourceMatchesGaps(source: ResearchSourceProposal, index: KnowledgeIndex, gaps: KnowledgeGap[]): KnowledgeSearchResult[];
|
|
336
|
+
/** @deprecated Renamed to {@link runVerifiedResearchLoop}. */
|
|
337
|
+
declare const runTwoAgentResearchLoop: typeof runVerifiedResearchLoop;
|
|
338
|
+
/** @deprecated Renamed to {@link VerifiedResearchLoopOptions}. */
|
|
339
|
+
type TwoAgentResearchLoopOptions = VerifiedResearchLoopOptions;
|
|
340
|
+
/** @deprecated Renamed to {@link VerifiedResearchLoopResult}. */
|
|
341
|
+
type TwoAgentResearchLoopResult = VerifiedResearchLoopResult;
|
|
342
|
+
/** @deprecated Renamed to {@link VerifiedResearchRound}. */
|
|
343
|
+
type TwoAgentResearchRound = VerifiedResearchRound;
|
|
332
344
|
|
|
333
345
|
/**
|
|
334
|
-
* Real web-research worker + verifying driver for `
|
|
346
|
+
* Real web-research worker + verifying driver for `runVerifiedResearchLoop`.
|
|
335
347
|
*
|
|
336
348
|
* This is the GENERAL, any-topic implementation behind the two-agent research
|
|
337
349
|
* loop's live arm. Given the open knowledge gaps the readiness gate surfaces,
|
|
@@ -407,6 +419,16 @@ interface TangleRouterOptions {
|
|
|
407
419
|
model?: string;
|
|
408
420
|
/** Optional preferred search provider (exa | you | perplexity | …). */
|
|
409
421
|
searchProvider?: string;
|
|
422
|
+
/**
|
|
423
|
+
* Retries on a TRANSIENT upstream status (502/503/504/429) with exponential
|
|
424
|
+
* backoff. Default 4. A 4xx that isn't 429, and a 401, are NOT retried — those
|
|
425
|
+
* are not transient. After the budget is exhausted the call still fails loud
|
|
426
|
+
* with the original `RouterError`, so the fail-closed contract holds; this only
|
|
427
|
+
* stops a single upstream-capacity blip from voiding a whole multi-topic run.
|
|
428
|
+
*/
|
|
429
|
+
maxRetries?: number;
|
|
430
|
+
/** Base backoff in ms (doubled each retry, ±25% jitter). Default 1500. */
|
|
431
|
+
retryBaseMs?: number;
|
|
410
432
|
signal?: AbortSignal;
|
|
411
433
|
}
|
|
412
434
|
/** A small error so a failed router call fails loud rather than returning junk. */
|
|
@@ -466,7 +488,7 @@ interface VerifyingDriverOptions {
|
|
|
466
488
|
declare function createVerifyingResearchDriver(options?: VerifyingDriverOptions): ResearchDriver;
|
|
467
489
|
|
|
468
490
|
/**
|
|
469
|
-
* Adaptive verifier mode for `
|
|
491
|
+
* Adaptive verifier mode for `runVerifiedResearchLoop`.
|
|
470
492
|
*
|
|
471
493
|
* The cost/quality A/B (`docs/results/cost-quality.md`) found the LLM relevance
|
|
472
494
|
* verifier's cleanliness win is dominated by DE-DUPLICATION — which a
|
|
@@ -715,7 +737,7 @@ declare function stripFrontmatter(content: string): {
|
|
|
715
737
|
};
|
|
716
738
|
|
|
717
739
|
/**
|
|
718
|
-
* Claim-grounding mode for `
|
|
740
|
+
* Claim-grounding mode for `runVerifiedResearchLoop`.
|
|
719
741
|
*
|
|
720
742
|
* The two-agent loop's existing verifier judges a source's on-topic RELEVANCE
|
|
721
743
|
* (is this page about the goal?). On the topic sets we have measured, its
|
|
@@ -828,7 +850,7 @@ interface ClaimGroundingDriverOptions extends GroundClaimOptions {
|
|
|
828
850
|
* composes a relevance verifier after grounding passes.
|
|
829
851
|
*
|
|
830
852
|
* The returned function matches `ResearchDriver['verifySource']`, so it drops
|
|
831
|
-
* straight into `
|
|
853
|
+
* straight into `runVerifiedResearchLoop` as `{ verifySource: createClaimGroundingVerifier(...) }`.
|
|
832
854
|
*/
|
|
833
855
|
declare function createClaimGroundingVerifier(options?: ClaimGroundingDriverOptions): (source: ResearchSourceProposal, ctx: SourceVerificationContext) => Promise<SourceVerdict>;
|
|
834
856
|
interface WorkerClaimDecorationOptions {
|
|
@@ -852,6 +874,41 @@ interface WorkerClaimDecorationOptions {
|
|
|
852
874
|
*/
|
|
853
875
|
declare function createClaimDecorator(options?: WorkerClaimDecorationOptions): (source: ResearchSourceProposal, goal: string) => Promise<ResearchSourceProposal>;
|
|
854
876
|
|
|
877
|
+
/**
|
|
878
|
+
* The SINGLE-AGENT COLLECTION driver — the blind-collection baseline (Arm A).
|
|
879
|
+
*
|
|
880
|
+
* This is the honest null the depth A/B is measured against. The other drivers
|
|
881
|
+
* spend extra inference to do something differentiated:
|
|
882
|
+
* - `createVerifyingResearchDriver` runs an LLM gate per source (Arm B),
|
|
883
|
+
* - `createResearchDrivingDriver` extracts claims, tracks corroboration, and
|
|
884
|
+
* synthesizes deep follow-up questions to drive depth (Arm C).
|
|
885
|
+
*
|
|
886
|
+
* This driver does NONE of that. It is a pass-through: it accepts every source
|
|
887
|
+
* the worker proposes and contributes no research, no gating, and no steering of
|
|
888
|
+
* its own. The loop still dedups exact-uri duplicates before calling
|
|
889
|
+
* `verifySource` (that is the loop's job, not the driver's), and the default
|
|
890
|
+
* `foldGaps` (a plain bulleted list of the still-open readiness gaps) still folds
|
|
891
|
+
* the gaps into the worker's next prompt — so the worker keeps researching, but
|
|
892
|
+
* NOTHING intelligent sits between the worker and the knowledge base.
|
|
893
|
+
*
|
|
894
|
+
* In other words: ONE agent (the worker) collects sources round after round, and
|
|
895
|
+
* the "driver" is an inert rubber stamp. That is exactly what "single-agent
|
|
896
|
+
* collection" means — the topology with zero coordinator intelligence — so its
|
|
897
|
+
* material-facts score is the floor every other arm must beat to justify its
|
|
898
|
+
* extra inference cost.
|
|
899
|
+
*
|
|
900
|
+
* It adds NO router calls of its own: `verifySource` is a synchronous accept and
|
|
901
|
+
* `foldGaps` is omitted so the loop uses its built-in gap list. So Arm A's cost
|
|
902
|
+
* is the worker's cost alone — the cleanest possible blind-collection baseline.
|
|
903
|
+
*/
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* Build the single-agent collection driver. Accepts every source; never gates,
|
|
907
|
+
* never researches, never steers beyond the loop's default open-gap list. The
|
|
908
|
+
* worker is the only agent that thinks.
|
|
909
|
+
*/
|
|
910
|
+
declare function createCollectionResearchDriver(): ResearchDriver;
|
|
911
|
+
|
|
855
912
|
interface DiscoveryTask {
|
|
856
913
|
id: string;
|
|
857
914
|
goal: string;
|
|
@@ -1065,6 +1122,740 @@ interface KnowledgeExplanation {
|
|
|
1065
1122
|
}
|
|
1066
1123
|
declare function explainKnowledgeTarget(index: KnowledgeIndex, target: string): KnowledgeExplanation;
|
|
1067
1124
|
|
|
1125
|
+
/**
|
|
1126
|
+
* HELD-OUT INVESTMENT-RESEARCH EVAL SET.
|
|
1127
|
+
*
|
|
1128
|
+
* The point of this file is the same FIREWALL the deep-question exam uses
|
|
1129
|
+
* (`tests/loops/held-out-exam.ts`): the material facts and their checkable
|
|
1130
|
+
* fragments are NEVER shown to a research loop. A loop is told only the company
|
|
1131
|
+
* + ticker + a research-as-of CUTOFF date, and asked to write an investment
|
|
1132
|
+
* thesis. AFTER it finishes, we grade the thesis it produced against THESE
|
|
1133
|
+
* facts — facts it never saw — so a high score is thesis QUALITY (it surfaced
|
|
1134
|
+
* the buried, material, non-obvious drivers) and not teaching-to-the-test.
|
|
1135
|
+
*
|
|
1136
|
+
* Each fact is a DEPTH fact by construction: a single ticker / company-name web
|
|
1137
|
+
* search does NOT surface it. They are the things buried in the filings or
|
|
1138
|
+
* knowable from then-available primary sources that a thorough analyst flags and
|
|
1139
|
+
* a one-shot search misses — customer/revenue/deposit concentration, a debt
|
|
1140
|
+
* maturity wall, a margin-trend reversal, a governance / related-party item, a
|
|
1141
|
+
* specific competitive or regulatory risk, an off-balance-sheet loss.
|
|
1142
|
+
*
|
|
1143
|
+
* THREE HARD RULES, enforced by how the data was gathered (see
|
|
1144
|
+
* docs/eval/investment-material-facts.md for the per-item provenance + the
|
|
1145
|
+
* curation-bias disclosure):
|
|
1146
|
+
*
|
|
1147
|
+
* 1. SPECIFIC + CHECKABLE. Each fact carries `expected` keyword groups — the
|
|
1148
|
+
* specific number / name / phrase — so a deterministic, model-free
|
|
1149
|
+
* substring grader (`gradeFactAgainstText`) can score "did the thesis
|
|
1150
|
+
* surface it". $0, reproducible, and it cannot leak the answer key into a
|
|
1151
|
+
* model the loop could observe.
|
|
1152
|
+
*
|
|
1153
|
+
* 2. DERIVED FROM REAL FETCHED EVIDENCE. Every fact records the primary source
|
|
1154
|
+
* it came from (`sourceUrl`, an SEC EDGAR 10-K) and the literal `evidence`
|
|
1155
|
+
* value read out of that document. Nothing here is invented; an item that
|
|
1156
|
+
* could not be independently sourced was DROPPED, not guessed (the drop log
|
|
1157
|
+
* is in the doc).
|
|
1158
|
+
*
|
|
1159
|
+
* 3. KNOWABLE AT THE CUTOFF. Every fact was disclosed in, or computable from, a
|
|
1160
|
+
* document available on or before the company's `cutoff` date. Post-cutoff
|
|
1161
|
+
* hindsight (the eventual bankruptcy / collapse) is NOT a checklist item —
|
|
1162
|
+
* it is recorded separately as `knownOutcome`, purely for the reader, and is
|
|
1163
|
+
* never graded.
|
|
1164
|
+
*
|
|
1165
|
+
* Grading mirrors the deep-question exam exactly: a fact is SURFACED when the
|
|
1166
|
+
* thesis text contains at least `minGroups` of its expected groups; a group is
|
|
1167
|
+
* satisfied when ANY of its `anyOf` fragments appears (case-insensitive
|
|
1168
|
+
* substring), so a faithful thesis phrased in its own words still grades as a
|
|
1169
|
+
* hit. `anyOf` groups model synonyms; the load-bearing tokens are the specific
|
|
1170
|
+
* numbers / names.
|
|
1171
|
+
*/
|
|
1172
|
+
/** A required answer component: satisfied when any synonym fragment is present. */
|
|
1173
|
+
interface ExpectedGroup {
|
|
1174
|
+
/** Human label for the component (for the doc / audit). */
|
|
1175
|
+
label: string;
|
|
1176
|
+
/** Case-insensitive substring fragments; any one present satisfies the group. */
|
|
1177
|
+
anyOf: string[];
|
|
1178
|
+
}
|
|
1179
|
+
/** Lens the fact belongs to — so a set can be checked for category coverage. */
|
|
1180
|
+
type MaterialFactLens = 'concentration' | 'leverage' | 'margin-trend' | 'liquidity' | 'capital-return' | 'governance' | 'off-balance-sheet' | 'regulatory';
|
|
1181
|
+
/** One held-out material fact with a checkable expected answer + its provenance. */
|
|
1182
|
+
interface MaterialFact {
|
|
1183
|
+
/** Stable id, `ticker/fN`. */
|
|
1184
|
+
id: string;
|
|
1185
|
+
/** Which analyst lens this fact exercises. For coverage + the doc. */
|
|
1186
|
+
lens: MaterialFactLens;
|
|
1187
|
+
/**
|
|
1188
|
+
* The material fact, in plain words — for the doc/audit. NEVER shown to a loop.
|
|
1189
|
+
* This is the thing a thorough analyst would flag and a ticker search misses.
|
|
1190
|
+
*/
|
|
1191
|
+
fact: string;
|
|
1192
|
+
/**
|
|
1193
|
+
* The checkable answer as required keyword GROUPS. The thesis text must contain
|
|
1194
|
+
* at least `minGroups` of these groups (default: all). A group is satisfied
|
|
1195
|
+
* when ANY of its `anyOf` fragments appears (case-insensitive substring).
|
|
1196
|
+
*/
|
|
1197
|
+
expected: ExpectedGroup[];
|
|
1198
|
+
/**
|
|
1199
|
+
* Minimum number of `expected` groups the thesis must contain to count the
|
|
1200
|
+
* fact SURFACED. Default = all groups (the strict bar). Lowered (and documented
|
|
1201
|
+
* inline) only when the fact is genuinely satisfiable by a subset.
|
|
1202
|
+
*/
|
|
1203
|
+
minGroups?: number;
|
|
1204
|
+
/**
|
|
1205
|
+
* PROVENANCE. The primary source URL this fact was read from — an SEC EDGAR
|
|
1206
|
+
* 10-K primary document, fetched live during curation.
|
|
1207
|
+
*/
|
|
1208
|
+
sourceUrl: string;
|
|
1209
|
+
/**
|
|
1210
|
+
* The literal value / phrase read out of `sourceUrl` that grounds the fact.
|
|
1211
|
+
* This is the "cite the actual filing + the value" requirement — verbatim or
|
|
1212
|
+
* near-verbatim from the filing, with the figure.
|
|
1213
|
+
*/
|
|
1214
|
+
evidence: string;
|
|
1215
|
+
}
|
|
1216
|
+
/** A company + the cutoff a loop researches as-of + its held-out material facts. */
|
|
1217
|
+
interface CompanyEvalCase {
|
|
1218
|
+
/** Ticker as of the cutoff. */
|
|
1219
|
+
ticker: string;
|
|
1220
|
+
/** Legal name as of the cutoff (what the loop is told to research). */
|
|
1221
|
+
company: string;
|
|
1222
|
+
/** SEC Central Index Key (CIK), zero-stripped — the EDGAR filer id. */
|
|
1223
|
+
cik: string;
|
|
1224
|
+
/**
|
|
1225
|
+
* Research-as-of date (ISO). The loop must reason as if it is this date; every
|
|
1226
|
+
* `evidence` value was knowable on or before it. >= 18 months before this set
|
|
1227
|
+
* was curated, so the outcome is known but is NOT a checklist item.
|
|
1228
|
+
*/
|
|
1229
|
+
cutoff: string;
|
|
1230
|
+
/** Sector, for coverage / the curation-bias disclosure. */
|
|
1231
|
+
sector: string;
|
|
1232
|
+
/**
|
|
1233
|
+
* The known POST-cutoff outcome — recorded for the reader ONLY, never graded.
|
|
1234
|
+
* Keeping it out of `facts` is what makes the set hindsight-free.
|
|
1235
|
+
*/
|
|
1236
|
+
knownOutcome: string;
|
|
1237
|
+
/** The held-out material facts for this company. */
|
|
1238
|
+
facts: MaterialFact[];
|
|
1239
|
+
}
|
|
1240
|
+
/**
|
|
1241
|
+
* The eval set. 5 public companies, 5-8 held-out material facts each, every fact
|
|
1242
|
+
* grounded in a primary SEC EDGAR 10-K filed on or before the cutoff.
|
|
1243
|
+
*
|
|
1244
|
+
* CURATION-BIAS DISCLOSURE (full version in the doc): all five are companies
|
|
1245
|
+
* whose buried risks later materialized, because that is where the material-vs-
|
|
1246
|
+
* surface distinction is sharpest AND where the figures are easy to verify after
|
|
1247
|
+
* the fact. This biases the set toward downside risks (two of the eight lenses,
|
|
1248
|
+
* concentration + leverage, dominate) and toward distressed names. A production
|
|
1249
|
+
* eval would balance these with companies whose buried facts were POSITIVE
|
|
1250
|
+
* drivers and with survivors. This set is honest about that and reports the lens
|
|
1251
|
+
* distribution so the bias is measurable, not hidden.
|
|
1252
|
+
*/
|
|
1253
|
+
declare const investmentThesisSet: CompanyEvalCase[];
|
|
1254
|
+
/**
|
|
1255
|
+
* Grade ONE material fact against an investment thesis's full text. Returns
|
|
1256
|
+
* whether the thesis SURFACED it plus which expected groups were found. The
|
|
1257
|
+
* check is a deterministic case-insensitive substring scan — $0, model-free,
|
|
1258
|
+
* reproducible — so the eval never leaks into a model the loop could observe.
|
|
1259
|
+
*/
|
|
1260
|
+
declare function gradeFactAgainstText(fact: MaterialFact, thesisText: string): {
|
|
1261
|
+
surfaced: boolean;
|
|
1262
|
+
groupsFound: number;
|
|
1263
|
+
groupsTotal: number;
|
|
1264
|
+
foundLabels: string[];
|
|
1265
|
+
};
|
|
1266
|
+
/** Grade a whole company's thesis text: how many of its held-out facts it surfaces. */
|
|
1267
|
+
declare function gradeCompanyAgainstText(company: CompanyEvalCase, thesisText: string): {
|
|
1268
|
+
surfaced: number;
|
|
1269
|
+
total: number;
|
|
1270
|
+
perFact: ReturnType<typeof gradeFactAgainstText>[];
|
|
1271
|
+
};
|
|
1272
|
+
/** Total held-out facts across the set (the denominator the doc reports). */
|
|
1273
|
+
declare function totalMaterialFacts(set?: CompanyEvalCase[]): number;
|
|
1274
|
+
/** Count facts per lens across the set — used to report (and bound) curation bias. */
|
|
1275
|
+
declare function lensDistribution(set?: CompanyEvalCase[]): Record<MaterialFactLens, number>;
|
|
1276
|
+
|
|
1277
|
+
/**
|
|
1278
|
+
* The INVESTMENT-THESIS research task.
|
|
1279
|
+
*
|
|
1280
|
+
* Given `{ company, ticker, cik, cutoff }`, drive the SAME two-agent research
|
|
1281
|
+
* loop the ML deep-question A/B uses (`runVerifiedResearchLoop` + the real web
|
|
1282
|
+
* worker) to research the company AS OF the cutoff — web + SEC EDGAR, both public
|
|
1283
|
+
* — and produce an investment-thesis PAGE in the knowledge base: a judgment, the
|
|
1284
|
+
* drivers, and the risks, grounded in what it fetched.
|
|
1285
|
+
*
|
|
1286
|
+
* This file builds NOTHING new for the loop: it composes the existing worker +
|
|
1287
|
+
* driver + loop, supplies the readiness specs that steer the worker toward the
|
|
1288
|
+
* filing-level evidence (the analyst lenses), then writes a synthesis thesis page
|
|
1289
|
+
* the metric (`materialFactsSurfaced`) grades against the HELD-OUT checklist.
|
|
1290
|
+
*
|
|
1291
|
+
* THE FIREWALL: the task is told ONLY company + ticker + cutoff (+ the generic
|
|
1292
|
+
* analyst-lens readiness specs every company gets). It is NEVER shown the
|
|
1293
|
+
* checklist. The checklist is read only afterward, by the metric. So a high score
|
|
1294
|
+
* is research depth, not teaching-to-the-test.
|
|
1295
|
+
*/
|
|
1296
|
+
|
|
1297
|
+
/** The minimal brief a thesis run is given — the firewall boundary. */
|
|
1298
|
+
interface ThesisTaskInput {
|
|
1299
|
+
/** Legal name as of the cutoff — what the loop researches. */
|
|
1300
|
+
company: string;
|
|
1301
|
+
/** Ticker as of the cutoff. */
|
|
1302
|
+
ticker: string;
|
|
1303
|
+
/** SEC Central Index Key (CIK), zero-stripped — the EDGAR filer id. */
|
|
1304
|
+
cik: string;
|
|
1305
|
+
/** Research-as-of date (ISO). The loop must reason as if it is this date. */
|
|
1306
|
+
cutoff: string;
|
|
1307
|
+
/** Sector, for the readiness query context (NOT a checklist hint). */
|
|
1308
|
+
sector?: string;
|
|
1309
|
+
}
|
|
1310
|
+
/**
|
|
1311
|
+
* The generic analyst-lens readiness specs every company gets. They are the ONLY
|
|
1312
|
+
* thing the loop is told about WHAT to look for, and they name the LENSES a
|
|
1313
|
+
* thorough analyst checks (balance-sheet risk, concentration, leverage, margins,
|
|
1314
|
+
* liquidity, governance, regulatory) and where they live (the latest SEC 10-K) —
|
|
1315
|
+
* NOT the answers. They steer the worker's web/EDGAR search toward the filing,
|
|
1316
|
+
* not toward the held-out facts (which the loop never sees).
|
|
1317
|
+
*
|
|
1318
|
+
* `minSources` is set above 1 so the readiness gate stays UNMET after a single
|
|
1319
|
+
* fetch and the loop runs multiple rounds — the depth-driving driver needs >1
|
|
1320
|
+
* round to steer, exactly as the ML-exam multi-round probe established.
|
|
1321
|
+
*/
|
|
1322
|
+
declare function thesisReadinessSpecs(input: ThesisTaskInput): KnowledgeReadinessSpec[];
|
|
1323
|
+
interface ThesisRunOptions {
|
|
1324
|
+
/** The KB root the loop writes into. */
|
|
1325
|
+
root: string;
|
|
1326
|
+
/** Shared router client (web search + chat). Defaults to env creds. */
|
|
1327
|
+
router: RouterClient;
|
|
1328
|
+
/** The driver — verify/dedup or research-driving. The loop's coordinator. */
|
|
1329
|
+
driver: ResearchDriver;
|
|
1330
|
+
/** Round budget. Default 3 (the depth-driving driver needs >1). */
|
|
1331
|
+
maxRounds?: number;
|
|
1332
|
+
/** Worker tuning forwarded to `createWebResearchWorker`. */
|
|
1333
|
+
workerOptions?: Omit<WebResearchWorkerOptions, 'router'>;
|
|
1334
|
+
/** Max tokens for the synthesis pass. Default 1600 (above glm-5.2's reasoning floor). */
|
|
1335
|
+
synthesisMaxTokens?: number;
|
|
1336
|
+
signal?: AbortSignal;
|
|
1337
|
+
}
|
|
1338
|
+
interface ThesisRunResult {
|
|
1339
|
+
loop: VerifiedResearchLoopResult;
|
|
1340
|
+
/** The synthesized thesis text. */
|
|
1341
|
+
thesis: string;
|
|
1342
|
+
/** Path of the thesis page written into the KB. */
|
|
1343
|
+
thesisPath: string;
|
|
1344
|
+
}
|
|
1345
|
+
/**
|
|
1346
|
+
* Run the full thesis task: drive the two-agent loop to research the company AS
|
|
1347
|
+
* OF the cutoff, then synthesize + write the thesis page. Returns the loop result
|
|
1348
|
+
* + the thesis text + the page path. The caller grades the KB with
|
|
1349
|
+
* `materialFactsSurfaced(root, checklist)` — the checklist is never passed here.
|
|
1350
|
+
*/
|
|
1351
|
+
declare function runInvestmentThesisTask(input: ThesisTaskInput, options: ThesisRunOptions): Promise<ThesisRunResult>;
|
|
1352
|
+
|
|
1353
|
+
interface ApplyWriteBlocksResult {
|
|
1354
|
+
written: string[];
|
|
1355
|
+
warnings: string[];
|
|
1356
|
+
}
|
|
1357
|
+
declare function applyKnowledgeWriteBlocks(root: string, proposalText: string): Promise<ApplyWriteBlocksResult>;
|
|
1358
|
+
declare function applyKnowledgeWriteBlocksFile(root: string, proposalPath: string): Promise<ApplyWriteBlocksResult>;
|
|
1359
|
+
|
|
1360
|
+
interface ValidateKnowledgeOptions {
|
|
1361
|
+
strict?: boolean;
|
|
1362
|
+
}
|
|
1363
|
+
interface ValidateKnowledgeResult {
|
|
1364
|
+
ok: boolean;
|
|
1365
|
+
findings: KnowledgeLintFinding[];
|
|
1366
|
+
}
|
|
1367
|
+
declare function validateKnowledgeIndex(index: KnowledgeIndex, options?: ValidateKnowledgeOptions): ValidateKnowledgeResult;
|
|
1368
|
+
|
|
1369
|
+
interface KnowledgeResearchLoopContext {
|
|
1370
|
+
root: string;
|
|
1371
|
+
goal: string;
|
|
1372
|
+
iteration: number;
|
|
1373
|
+
index: KnowledgeIndex;
|
|
1374
|
+
lintFindings: KnowledgeLintFinding[];
|
|
1375
|
+
validation: ValidateKnowledgeResult;
|
|
1376
|
+
readiness?: EvalKnowledgeBundleBuildResult;
|
|
1377
|
+
previousSteps: KnowledgeResearchLoopStep[];
|
|
1378
|
+
signal?: AbortSignal;
|
|
1379
|
+
}
|
|
1380
|
+
interface KnowledgeResearchLoopDecision {
|
|
1381
|
+
/**
|
|
1382
|
+
* Free-form notes from the researcher. Keep this human-readable; products can
|
|
1383
|
+
* store it as the research transcript.
|
|
1384
|
+
*/
|
|
1385
|
+
notes?: string;
|
|
1386
|
+
/**
|
|
1387
|
+
* Local files to register as immutable sources before applying proposals.
|
|
1388
|
+
*/
|
|
1389
|
+
sourcePaths?: string[];
|
|
1390
|
+
/**
|
|
1391
|
+
* Textual source artifacts discovered by an agent, browser worker, connector,
|
|
1392
|
+
* or deep-research process.
|
|
1393
|
+
*/
|
|
1394
|
+
sourceTexts?: AddSourceTextInput[];
|
|
1395
|
+
/**
|
|
1396
|
+
* Safe write protocol text. The loop parses and applies only accepted
|
|
1397
|
+
* `---FILE: knowledge/...---` blocks.
|
|
1398
|
+
*/
|
|
1399
|
+
proposalText?: string;
|
|
1400
|
+
/**
|
|
1401
|
+
* The researcher decides when the wiki is good enough. The loop deliberately
|
|
1402
|
+
* does not encode a domain-specific definition of "done".
|
|
1403
|
+
*/
|
|
1404
|
+
done?: boolean;
|
|
1405
|
+
metadata?: Record<string, unknown>;
|
|
1406
|
+
}
|
|
1407
|
+
interface KnowledgeResearchLoopStep {
|
|
1408
|
+
iteration: number;
|
|
1409
|
+
notes?: string;
|
|
1410
|
+
addedSources: SourceRecord[];
|
|
1411
|
+
applied?: ApplyWriteBlocksResult;
|
|
1412
|
+
lintFindings: KnowledgeLintFinding[];
|
|
1413
|
+
validation: ValidateKnowledgeResult;
|
|
1414
|
+
readiness?: EvalKnowledgeBundleBuildResult;
|
|
1415
|
+
event: KnowledgeEvent;
|
|
1416
|
+
done: boolean;
|
|
1417
|
+
metadata?: Record<string, unknown>;
|
|
1418
|
+
}
|
|
1419
|
+
interface RunKnowledgeResearchLoopOptions {
|
|
1420
|
+
root: string;
|
|
1421
|
+
goal: string;
|
|
1422
|
+
maxIterations?: number;
|
|
1423
|
+
actor?: string;
|
|
1424
|
+
strict?: ValidateKnowledgeOptions['strict'];
|
|
1425
|
+
readinessSpecs?: KnowledgeReadinessSpec[];
|
|
1426
|
+
readinessTaskId?: string;
|
|
1427
|
+
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
1428
|
+
sourceOptions?: Pick<AddSourceOptions, 'adapters' | 'now'>;
|
|
1429
|
+
signal?: AbortSignal;
|
|
1430
|
+
step(context: KnowledgeResearchLoopContext): Promise<KnowledgeResearchLoopDecision> | KnowledgeResearchLoopDecision;
|
|
1431
|
+
onStep?: (step: KnowledgeResearchLoopStep) => Promise<void> | void;
|
|
1432
|
+
}
|
|
1433
|
+
interface KnowledgeResearchLoopResult {
|
|
1434
|
+
root: string;
|
|
1435
|
+
goal: string;
|
|
1436
|
+
iterations: number;
|
|
1437
|
+
done: boolean;
|
|
1438
|
+
index: KnowledgeIndex;
|
|
1439
|
+
lintFindings: KnowledgeLintFinding[];
|
|
1440
|
+
validation: ValidateKnowledgeResult;
|
|
1441
|
+
readiness?: EvalKnowledgeBundleBuildResult;
|
|
1442
|
+
steps: KnowledgeResearchLoopStep[];
|
|
1443
|
+
}
|
|
1444
|
+
type KnowledgeControlLoopState = KnowledgeResearchLoopContext;
|
|
1445
|
+
type KnowledgeControlLoopAction = KnowledgeResearchLoopDecision;
|
|
1446
|
+
type KnowledgeControlLoopActionResult = KnowledgeResearchLoopStep;
|
|
1447
|
+
interface KnowledgeControlLoopAdapterOptions {
|
|
1448
|
+
root: string;
|
|
1449
|
+
goal: string;
|
|
1450
|
+
actor?: string;
|
|
1451
|
+
strict?: ValidateKnowledgeOptions['strict'];
|
|
1452
|
+
readinessSpecs?: KnowledgeReadinessSpec[];
|
|
1453
|
+
readinessTaskId?: string;
|
|
1454
|
+
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
1455
|
+
sourceOptions?: Pick<AddSourceOptions, 'adapters' | 'now'>;
|
|
1456
|
+
}
|
|
1457
|
+
type KnowledgeControlLoopAdapter = Pick<ControlRuntimeConfig<KnowledgeControlLoopState, KnowledgeControlLoopAction, KnowledgeControlLoopActionResult, ControlEvalResult>, 'intent' | 'observe' | 'validate' | 'act' | 'shouldStop'>;
|
|
1458
|
+
/**
|
|
1459
|
+
* Adapter for running knowledge growth through `agent-eval`'s generic control
|
|
1460
|
+
* runtime. The caller still owns `decide`: that can be a proposer agent,
|
|
1461
|
+
* reviewer agent, deterministic policy, or a composition of all three.
|
|
1462
|
+
*/
|
|
1463
|
+
declare function createKnowledgeControlLoopAdapter(options: KnowledgeControlLoopAdapterOptions): KnowledgeControlLoopAdapter;
|
|
1464
|
+
declare function runKnowledgeResearchLoop(options: RunKnowledgeResearchLoopOptions): Promise<KnowledgeResearchLoopResult>;
|
|
1465
|
+
|
|
1466
|
+
type RagKnowledgeImprovementPhase = 'retrieval-tuning' | 'gap-diagnosis' | 'knowledge-acquisition' | 'knowledge-update' | 'answer-quality' | 'promotion';
|
|
1467
|
+
type RagKnowledgeImprovementPhaseStatus = 'completed' | 'skipped' | 'failed';
|
|
1468
|
+
type RagGapKind = 'missing-source' | 'stale-source' | 'retrieval-miss' | 'retrieval-noise' | 'chunking-mismatch' | 'missing-multihop-evidence' | 'generator-unsupported-claim' | 'citation-mismatch' | 'incorrect-abstention' | 'unknown';
|
|
1469
|
+
type RagGapSeverity = 'info' | 'warning' | 'error' | 'critical';
|
|
1470
|
+
interface RagGapFinding {
|
|
1471
|
+
id: string;
|
|
1472
|
+
kind: RagGapKind;
|
|
1473
|
+
severity: RagGapSeverity;
|
|
1474
|
+
message: string;
|
|
1475
|
+
scenarioId?: string;
|
|
1476
|
+
evidence?: Record<string, JsonValue>;
|
|
1477
|
+
}
|
|
1478
|
+
interface RagKnowledgeImprovementPhaseResult {
|
|
1479
|
+
phase: RagKnowledgeImprovementPhase;
|
|
1480
|
+
status: RagKnowledgeImprovementPhaseStatus;
|
|
1481
|
+
summary: string;
|
|
1482
|
+
startedAt: string;
|
|
1483
|
+
finishedAt: string;
|
|
1484
|
+
metadata?: Record<string, JsonValue>;
|
|
1485
|
+
}
|
|
1486
|
+
interface RagPhaseInputBase {
|
|
1487
|
+
goal: string;
|
|
1488
|
+
phases: readonly RagKnowledgeImprovementPhaseResult[];
|
|
1489
|
+
signal?: AbortSignal;
|
|
1490
|
+
}
|
|
1491
|
+
interface RagDiagnosisInput extends RagPhaseInputBase {
|
|
1492
|
+
retrieval?: RunRetrievalImprovementLoopResult;
|
|
1493
|
+
}
|
|
1494
|
+
interface RagKnowledgeAcquisitionInput extends RagPhaseInputBase {
|
|
1495
|
+
retrieval?: RunRetrievalImprovementLoopResult;
|
|
1496
|
+
findings: readonly RagGapFinding[];
|
|
1497
|
+
}
|
|
1498
|
+
interface RagKnowledgeUpdateInput extends RagPhaseInputBase {
|
|
1499
|
+
retrieval?: RunRetrievalImprovementLoopResult;
|
|
1500
|
+
findings: readonly RagGapFinding[];
|
|
1501
|
+
acquisition?: KnowledgeResearchLoopDecision;
|
|
1502
|
+
}
|
|
1503
|
+
interface RagKnowledgeUpdateResult {
|
|
1504
|
+
applied: boolean;
|
|
1505
|
+
summary: string;
|
|
1506
|
+
research?: KnowledgeResearchLoopResult;
|
|
1507
|
+
metadata?: Record<string, JsonValue>;
|
|
1508
|
+
}
|
|
1509
|
+
interface RagAnswerQualityInput extends RagPhaseInputBase {
|
|
1510
|
+
retrieval?: RunRetrievalImprovementLoopResult;
|
|
1511
|
+
findings: readonly RagGapFinding[];
|
|
1512
|
+
acquisition?: KnowledgeResearchLoopDecision;
|
|
1513
|
+
knowledgeUpdate?: RagKnowledgeUpdateResult;
|
|
1514
|
+
}
|
|
1515
|
+
interface RagAnswerQualityResult {
|
|
1516
|
+
passed: boolean;
|
|
1517
|
+
metrics: Record<string, number>;
|
|
1518
|
+
findings?: readonly RagGapFinding[];
|
|
1519
|
+
metadata?: Record<string, JsonValue>;
|
|
1520
|
+
}
|
|
1521
|
+
interface RagPromotionInput extends RagPhaseInputBase {
|
|
1522
|
+
retrieval?: RunRetrievalImprovementLoopResult;
|
|
1523
|
+
findings: readonly RagGapFinding[];
|
|
1524
|
+
acquisition?: KnowledgeResearchLoopDecision;
|
|
1525
|
+
knowledgeUpdate?: RagKnowledgeUpdateResult;
|
|
1526
|
+
answerQuality?: RagAnswerQualityResult;
|
|
1527
|
+
}
|
|
1528
|
+
interface RagPromotionResult {
|
|
1529
|
+
promoted: boolean;
|
|
1530
|
+
reason: string;
|
|
1531
|
+
metadata?: Record<string, JsonValue>;
|
|
1532
|
+
}
|
|
1533
|
+
interface RagKnowledgeResearchOptions extends Omit<RunKnowledgeResearchLoopOptions, 'goal' | 'signal' | 'step'> {
|
|
1534
|
+
goal?: string;
|
|
1535
|
+
step?: RunKnowledgeResearchLoopOptions['step'];
|
|
1536
|
+
}
|
|
1537
|
+
interface RunRagKnowledgeImprovementLoopOptions {
|
|
1538
|
+
goal: string;
|
|
1539
|
+
retrieval?: RunRetrievalImprovementLoopOptions;
|
|
1540
|
+
diagnose?: (input: RagDiagnosisInput) => MaybePromise$1<readonly RagGapFinding[]>;
|
|
1541
|
+
acquireKnowledge?: (input: RagKnowledgeAcquisitionInput) => MaybePromise$1<KnowledgeResearchLoopDecision>;
|
|
1542
|
+
knowledgeResearch?: RagKnowledgeResearchOptions;
|
|
1543
|
+
updateKnowledge?: (input: RagKnowledgeUpdateInput) => MaybePromise$1<RagKnowledgeUpdateResult>;
|
|
1544
|
+
evaluateAnswers?: (input: RagAnswerQualityInput) => MaybePromise$1<RagAnswerQualityResult>;
|
|
1545
|
+
promote?: (input: RagPromotionInput) => MaybePromise$1<RagPromotionResult>;
|
|
1546
|
+
enabledPhases?: readonly RagKnowledgeImprovementPhase[];
|
|
1547
|
+
requiredPhases?: readonly RagKnowledgeImprovementPhase[];
|
|
1548
|
+
signal?: AbortSignal;
|
|
1549
|
+
now?: () => Date;
|
|
1550
|
+
}
|
|
1551
|
+
interface RunRagKnowledgeImprovementLoopResult {
|
|
1552
|
+
goal: string;
|
|
1553
|
+
phases: readonly RagKnowledgeImprovementPhaseResult[];
|
|
1554
|
+
retrieval?: RunRetrievalImprovementLoopResult;
|
|
1555
|
+
findings: readonly RagGapFinding[];
|
|
1556
|
+
acquisition?: KnowledgeResearchLoopDecision;
|
|
1557
|
+
knowledgeUpdate?: RagKnowledgeUpdateResult;
|
|
1558
|
+
answerQuality?: RagAnswerQualityResult;
|
|
1559
|
+
promotion?: RagPromotionResult;
|
|
1560
|
+
}
|
|
1561
|
+
type MaybePromise$1<T> = T | Promise<T>;
|
|
1562
|
+
declare function runRagKnowledgeImprovementLoop(options: RunRagKnowledgeImprovementLoopOptions): Promise<RunRagKnowledgeImprovementLoopResult>;
|
|
1563
|
+
|
|
1564
|
+
type RagEvalProvider = 'agent-knowledge' | 'ragas' | 'deepeval' | 'trulens' | 'ragchecker' | 'custom';
|
|
1565
|
+
type RagEvalMetricKey = 'context_precision' | 'context_recall' | 'context_relevance' | 'context_sufficiency' | 'faithfulness' | 'groundedness' | 'answer_relevance' | 'answer_correctness' | 'citation_support' | 'abstention' | 'unsupported_answer_rate';
|
|
1566
|
+
type RagEvalSlice = 'known-answer' | 'paraphrase' | 'distractor' | 'freshness' | 'multi-source' | 'unanswerable' | 'long-tail' | 'custom';
|
|
1567
|
+
interface RagEvalContext {
|
|
1568
|
+
id: string;
|
|
1569
|
+
text: string;
|
|
1570
|
+
rank?: number;
|
|
1571
|
+
pageId?: string;
|
|
1572
|
+
sourceId?: string;
|
|
1573
|
+
anchorId?: string;
|
|
1574
|
+
stale?: boolean;
|
|
1575
|
+
metadata?: Record<string, JsonValue>;
|
|
1576
|
+
}
|
|
1577
|
+
interface RagEvalCitation {
|
|
1578
|
+
id: string;
|
|
1579
|
+
claimId?: string;
|
|
1580
|
+
contextId?: string;
|
|
1581
|
+
pageId?: string;
|
|
1582
|
+
sourceId?: string;
|
|
1583
|
+
anchorId?: string;
|
|
1584
|
+
quote?: string;
|
|
1585
|
+
metadata?: Record<string, JsonValue>;
|
|
1586
|
+
}
|
|
1587
|
+
interface RagEvalClaim {
|
|
1588
|
+
id: string;
|
|
1589
|
+
text: string;
|
|
1590
|
+
citationIds?: readonly string[];
|
|
1591
|
+
metadata?: Record<string, JsonValue>;
|
|
1592
|
+
}
|
|
1593
|
+
interface RagRequiredContext {
|
|
1594
|
+
id?: string;
|
|
1595
|
+
text?: string;
|
|
1596
|
+
pageId?: string;
|
|
1597
|
+
sourceId?: string;
|
|
1598
|
+
anchorId?: string;
|
|
1599
|
+
}
|
|
1600
|
+
interface RagAnswerEvalScenario extends Scenario {
|
|
1601
|
+
kind: 'rag-answer-eval';
|
|
1602
|
+
query: string;
|
|
1603
|
+
referenceAnswer?: string;
|
|
1604
|
+
expectedClaims?: readonly string[];
|
|
1605
|
+
forbiddenClaims?: readonly string[];
|
|
1606
|
+
requiredContext?: readonly RagRequiredContext[];
|
|
1607
|
+
unanswerable?: boolean;
|
|
1608
|
+
requireCitations?: boolean;
|
|
1609
|
+
slices?: readonly RagEvalSlice[];
|
|
1610
|
+
thresholds?: Partial<Record<RagEvalMetricKey, number>>;
|
|
1611
|
+
}
|
|
1612
|
+
interface ExternalRagEvalScore {
|
|
1613
|
+
provider: RagEvalProvider | string;
|
|
1614
|
+
scores: Record<string, number>;
|
|
1615
|
+
reasons?: Record<string, string>;
|
|
1616
|
+
metadata?: Record<string, JsonValue>;
|
|
1617
|
+
}
|
|
1618
|
+
interface RagAnswerEvalArtifact {
|
|
1619
|
+
query: string;
|
|
1620
|
+
answer: string;
|
|
1621
|
+
contexts: readonly RagEvalContext[];
|
|
1622
|
+
claims?: readonly RagEvalClaim[];
|
|
1623
|
+
citations?: readonly RagEvalCitation[];
|
|
1624
|
+
abstained?: boolean;
|
|
1625
|
+
durationMs?: number;
|
|
1626
|
+
costUsd?: number;
|
|
1627
|
+
externalScores?: readonly ExternalRagEvalScore[];
|
|
1628
|
+
metadata?: Record<string, JsonValue>;
|
|
1629
|
+
}
|
|
1630
|
+
interface RagAnswerMetricSummary {
|
|
1631
|
+
metrics: Record<RagEvalMetricKey, number>;
|
|
1632
|
+
composite: number;
|
|
1633
|
+
passed: boolean;
|
|
1634
|
+
findings: readonly RagGapFinding[];
|
|
1635
|
+
claimCount: number;
|
|
1636
|
+
supportedClaimCount: number;
|
|
1637
|
+
citedClaimCount: number;
|
|
1638
|
+
supportedCitationCount: number;
|
|
1639
|
+
matchedRequiredContextCount: number;
|
|
1640
|
+
requiredContextCount: number;
|
|
1641
|
+
providerScores: Record<string, Record<RagEvalMetricKey, number>>;
|
|
1642
|
+
}
|
|
1643
|
+
interface RagAnswerQualityJudgeOptions {
|
|
1644
|
+
name?: string;
|
|
1645
|
+
thresholds?: Partial<Record<RagEvalMetricKey, number>>;
|
|
1646
|
+
weights?: Partial<Record<RagEvalMetricKey, number>>;
|
|
1647
|
+
externalScorePolicy?: 'prefer-external' | 'deterministic-first';
|
|
1648
|
+
minClaimSupport?: number;
|
|
1649
|
+
}
|
|
1650
|
+
interface RagAnswerEvalCase {
|
|
1651
|
+
scenario: RagAnswerEvalScenario;
|
|
1652
|
+
artifact: RagAnswerEvalArtifact;
|
|
1653
|
+
}
|
|
1654
|
+
interface RagAnswerQualityHookOptions {
|
|
1655
|
+
scenarios: readonly RagAnswerEvalScenario[];
|
|
1656
|
+
run: (scenario: RagAnswerEvalScenario) => MaybePromise<RagAnswerEvalArtifact>;
|
|
1657
|
+
externalEvaluator?: (item: RagAnswerEvalCase) => MaybePromise<ExternalRagEvalScore | readonly ExternalRagEvalScore[] | undefined>;
|
|
1658
|
+
thresholds?: Partial<Record<RagEvalMetricKey, number>>;
|
|
1659
|
+
weights?: Partial<Record<RagEvalMetricKey, number>>;
|
|
1660
|
+
}
|
|
1661
|
+
interface RagCalibrationOptions {
|
|
1662
|
+
scenario: RagAnswerEvalScenario;
|
|
1663
|
+
strong: RagAnswerEvalArtifact;
|
|
1664
|
+
weak: RagAnswerEvalArtifact;
|
|
1665
|
+
judge?: JudgeConfig<RagAnswerEvalArtifact, RagAnswerEvalScenario>;
|
|
1666
|
+
minStrongScore?: number;
|
|
1667
|
+
maxWeakScore?: number;
|
|
1668
|
+
signal?: AbortSignal;
|
|
1669
|
+
}
|
|
1670
|
+
interface RagCalibrationResult {
|
|
1671
|
+
passed: boolean;
|
|
1672
|
+
strongScore: number;
|
|
1673
|
+
weakScore: number;
|
|
1674
|
+
gap: number;
|
|
1675
|
+
}
|
|
1676
|
+
interface KnowledgeBaseQualityOptions {
|
|
1677
|
+
now?: Date;
|
|
1678
|
+
strict?: boolean;
|
|
1679
|
+
minCitationRate?: number;
|
|
1680
|
+
maxStaleSourceRate?: number;
|
|
1681
|
+
}
|
|
1682
|
+
interface KnowledgeBaseQualityReport {
|
|
1683
|
+
ok: boolean;
|
|
1684
|
+
metrics: {
|
|
1685
|
+
page_count: number;
|
|
1686
|
+
source_count: number;
|
|
1687
|
+
citation_rate: number;
|
|
1688
|
+
source_backed_page_rate: number;
|
|
1689
|
+
stale_source_rate: number;
|
|
1690
|
+
duplicate_source_hash_rate: number;
|
|
1691
|
+
lint_error_count: number;
|
|
1692
|
+
lint_warning_count: number;
|
|
1693
|
+
};
|
|
1694
|
+
findings: readonly RagGapFinding[];
|
|
1695
|
+
}
|
|
1696
|
+
type MaybePromise<T> = T | Promise<T>;
|
|
1697
|
+
declare function ragAnswerQualityJudge(options?: RagAnswerQualityJudgeOptions): JudgeConfig<RagAnswerEvalArtifact, RagAnswerEvalScenario>;
|
|
1698
|
+
declare function scoreRagAnswerArtifact(artifact: RagAnswerEvalArtifact, scenario: RagAnswerEvalScenario, options?: RagAnswerQualityJudgeOptions): RagAnswerMetricSummary;
|
|
1699
|
+
declare function diagnoseRagAnswerFailure(metrics: Record<RagEvalMetricKey, number>, scenario: RagAnswerEvalScenario, thresholds?: Partial<Record<RagEvalMetricKey, number>>): RagGapFinding[];
|
|
1700
|
+
declare function createRagAnswerQualityHook(options: RagAnswerQualityHookOptions): () => Promise<RagAnswerQualityResult>;
|
|
1701
|
+
declare function calibrateRagAnswerJudge(options: RagCalibrationOptions): Promise<RagCalibrationResult>;
|
|
1702
|
+
declare function normalizeExternalRagScores(scores: readonly ExternalRagEvalScore[]): Record<string, Record<RagEvalMetricKey, number>>;
|
|
1703
|
+
declare function toRagasEvaluationRows(cases: readonly RagAnswerEvalCase[]): {
|
|
1704
|
+
user_input: string;
|
|
1705
|
+
response: string;
|
|
1706
|
+
retrieved_contexts: string[];
|
|
1707
|
+
reference: string | undefined;
|
|
1708
|
+
reference_contexts: string[];
|
|
1709
|
+
}[];
|
|
1710
|
+
declare function toDeepEvalTestCases(cases: readonly RagAnswerEvalCase[]): {
|
|
1711
|
+
input: string;
|
|
1712
|
+
actual_output: string;
|
|
1713
|
+
expected_output: string | undefined;
|
|
1714
|
+
retrieval_context: string[];
|
|
1715
|
+
context: string[];
|
|
1716
|
+
}[];
|
|
1717
|
+
declare function toTruLensRecords(cases: readonly RagAnswerEvalCase[]): {
|
|
1718
|
+
input: string;
|
|
1719
|
+
output: string;
|
|
1720
|
+
context: string;
|
|
1721
|
+
}[];
|
|
1722
|
+
declare function toRagCheckerRecords(cases: readonly RagAnswerEvalCase[]): {
|
|
1723
|
+
query_id: string;
|
|
1724
|
+
query: string;
|
|
1725
|
+
gt_answer: string | undefined;
|
|
1726
|
+
response: string;
|
|
1727
|
+
retrieved_context: {
|
|
1728
|
+
doc_id: string;
|
|
1729
|
+
text: string;
|
|
1730
|
+
}[];
|
|
1731
|
+
claims: string[];
|
|
1732
|
+
}[];
|
|
1733
|
+
declare function scoreKnowledgeBaseIndex(index: KnowledgeIndex, options?: KnowledgeBaseQualityOptions): KnowledgeBaseQualityReport;
|
|
1734
|
+
|
|
1735
|
+
type KnowledgeImprovementStatus = 'running' | 'candidate-ready' | 'promoted' | 'rejected' | 'blocked';
|
|
1736
|
+
interface KnowledgeImprovementMetric {
|
|
1737
|
+
score: number;
|
|
1738
|
+
passed: boolean;
|
|
1739
|
+
dimensions?: Record<string, number>;
|
|
1740
|
+
notes?: string;
|
|
1741
|
+
}
|
|
1742
|
+
interface KnowledgeImprovementEvaluationInput {
|
|
1743
|
+
runId: string;
|
|
1744
|
+
iteration: number;
|
|
1745
|
+
root: string;
|
|
1746
|
+
baselineRoot: string;
|
|
1747
|
+
candidateRoot: string;
|
|
1748
|
+
baselineIndex: KnowledgeIndex;
|
|
1749
|
+
candidateIndex: KnowledgeIndex;
|
|
1750
|
+
baseHash: string;
|
|
1751
|
+
candidateHash: string;
|
|
1752
|
+
validation: ValidateKnowledgeResult;
|
|
1753
|
+
readiness?: EvalKnowledgeBundleBuildResult;
|
|
1754
|
+
kbQuality: KnowledgeBaseQualityReport;
|
|
1755
|
+
lifecycle?: RunRagKnowledgeImprovementLoopResult;
|
|
1756
|
+
signal?: AbortSignal;
|
|
1757
|
+
}
|
|
1758
|
+
type KnowledgeImprovementEvaluator = (input: KnowledgeImprovementEvaluationInput) => Promise<KnowledgeImprovementMetric> | KnowledgeImprovementMetric;
|
|
1759
|
+
interface KnowledgeImprovementLifecycleRecord {
|
|
1760
|
+
stage: 'candidate-update' | 'candidate-evaluation';
|
|
1761
|
+
phases: readonly RagKnowledgeImprovementPhaseResult[];
|
|
1762
|
+
findingCount: number;
|
|
1763
|
+
retrievalWinnerConfig?: RetrievalConfig;
|
|
1764
|
+
answerQuality?: RagAnswerQualityResult;
|
|
1765
|
+
promotionDecision?: RagPromotionResult;
|
|
1766
|
+
}
|
|
1767
|
+
interface KnowledgeImprovementCandidateRecord {
|
|
1768
|
+
iteration: number;
|
|
1769
|
+
candidateId: string;
|
|
1770
|
+
candidateRoot: string;
|
|
1771
|
+
baseHash: string;
|
|
1772
|
+
candidateHash?: string;
|
|
1773
|
+
status: KnowledgeImprovementStatus;
|
|
1774
|
+
createdAt: string;
|
|
1775
|
+
updatedAt: string;
|
|
1776
|
+
validation?: ValidateKnowledgeResult;
|
|
1777
|
+
kbQuality?: KnowledgeBaseQualityReport;
|
|
1778
|
+
readinessBlockingMissing?: number;
|
|
1779
|
+
evaluation?: KnowledgeImprovementMetric;
|
|
1780
|
+
lifecycle?: readonly KnowledgeImprovementLifecycleRecord[];
|
|
1781
|
+
retrievalWinnerConfig?: RetrievalConfig;
|
|
1782
|
+
answerQuality?: RagAnswerQualityResult;
|
|
1783
|
+
promotionDecision?: RagPromotionResult;
|
|
1784
|
+
notes?: string;
|
|
1785
|
+
}
|
|
1786
|
+
interface KnowledgeImprovementRunState {
|
|
1787
|
+
version: 1;
|
|
1788
|
+
runId: string;
|
|
1789
|
+
root: string;
|
|
1790
|
+
goal: string;
|
|
1791
|
+
status: KnowledgeImprovementStatus;
|
|
1792
|
+
baseHash: string;
|
|
1793
|
+
createdAt: string;
|
|
1794
|
+
updatedAt: string;
|
|
1795
|
+
ownerId?: string;
|
|
1796
|
+
candidates: KnowledgeImprovementCandidateRecord[];
|
|
1797
|
+
promotedCandidateId?: string;
|
|
1798
|
+
blockedReason?: string;
|
|
1799
|
+
}
|
|
1800
|
+
interface KnowledgeImprovementResult {
|
|
1801
|
+
runId: string;
|
|
1802
|
+
runDir: string;
|
|
1803
|
+
state: KnowledgeImprovementRunState;
|
|
1804
|
+
candidate?: KnowledgeImprovementCandidateRecord;
|
|
1805
|
+
lifecycle?: RunRagKnowledgeImprovementLoopResult;
|
|
1806
|
+
promoted: boolean;
|
|
1807
|
+
blocked: boolean;
|
|
1808
|
+
}
|
|
1809
|
+
interface KnowledgeImprovementRetrievalOptions extends Omit<RunRetrievalImprovementLoopOptions, 'index' | 'runDir'> {
|
|
1810
|
+
runDir?: RunRetrievalImprovementLoopOptions['runDir'];
|
|
1811
|
+
}
|
|
1812
|
+
interface KnowledgeImprovementUpdateInput extends RagKnowledgeUpdateInput {
|
|
1813
|
+
runId: string;
|
|
1814
|
+
iteration: number;
|
|
1815
|
+
candidateId: string;
|
|
1816
|
+
root: string;
|
|
1817
|
+
baselineRoot: string;
|
|
1818
|
+
candidateRoot: string;
|
|
1819
|
+
baseHash: string;
|
|
1820
|
+
}
|
|
1821
|
+
type KnowledgeImprovementUpdate = (input: KnowledgeImprovementUpdateInput) => Promise<RagKnowledgeUpdateResult> | RagKnowledgeUpdateResult;
|
|
1822
|
+
interface KnowledgeImprovementOptions {
|
|
1823
|
+
root: string;
|
|
1824
|
+
goal: string;
|
|
1825
|
+
runId?: string;
|
|
1826
|
+
runDir?: string;
|
|
1827
|
+
ownerId?: string;
|
|
1828
|
+
leaseTtlMs?: number;
|
|
1829
|
+
resume?: boolean;
|
|
1830
|
+
promote?: boolean;
|
|
1831
|
+
maxCandidates?: number;
|
|
1832
|
+
candidateResearchIterations?: number;
|
|
1833
|
+
strict?: ValidateKnowledgeOptions['strict'];
|
|
1834
|
+
readinessSpecs?: KnowledgeReadinessSpec[];
|
|
1835
|
+
readinessTaskId?: string;
|
|
1836
|
+
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
1837
|
+
kbQuality?: KnowledgeBaseQualityOptions;
|
|
1838
|
+
step?: RunKnowledgeResearchLoopOptions['step'];
|
|
1839
|
+
knowledgeResearch?: Omit<RagKnowledgeResearchOptions, 'root'>;
|
|
1840
|
+
retrieval?: KnowledgeImprovementRetrievalOptions;
|
|
1841
|
+
diagnose?: NonNullable<RunRagKnowledgeImprovementLoopOptions['diagnose']>;
|
|
1842
|
+
acquireKnowledge?: NonNullable<RunRagKnowledgeImprovementLoopOptions['acquireKnowledge']>;
|
|
1843
|
+
updateKnowledge?: KnowledgeImprovementUpdate;
|
|
1844
|
+
evaluateAnswers?: NonNullable<RunRagKnowledgeImprovementLoopOptions['evaluateAnswers']>;
|
|
1845
|
+
decidePromotion?: NonNullable<RunRagKnowledgeImprovementLoopOptions['promote']>;
|
|
1846
|
+
enabledPhases?: readonly RagKnowledgeImprovementPhase[];
|
|
1847
|
+
requiredPhases?: readonly RagKnowledgeImprovementPhase[];
|
|
1848
|
+
evaluate?: KnowledgeImprovementEvaluator;
|
|
1849
|
+
signal?: AbortSignal;
|
|
1850
|
+
now?: () => Date;
|
|
1851
|
+
onState?: (state: KnowledgeImprovementRunState) => Promise<void> | void;
|
|
1852
|
+
}
|
|
1853
|
+
declare function knowledgeImprovementRunId(root: string, goal: string): string;
|
|
1854
|
+
declare function knowledgeImprovementRunDir(root: string, runId: string): string;
|
|
1855
|
+
declare function loadKnowledgeImprovementState(root: string, runId: string, runDir?: string): Promise<KnowledgeImprovementRunState | null>;
|
|
1856
|
+
declare function improveKnowledgeBase(options: KnowledgeImprovementOptions): Promise<KnowledgeImprovementResult>;
|
|
1857
|
+
declare function hashKnowledgeBase(root: string): Promise<string>;
|
|
1858
|
+
|
|
1068
1859
|
interface KbStore {
|
|
1069
1860
|
putSource(source: SourceRecord): Promise<void>;
|
|
1070
1861
|
getSource(id: string): Promise<SourceRecord | null>;
|
|
@@ -1103,12 +1894,79 @@ declare class FileSystemKbStore extends MemoryKbStore {
|
|
|
1103
1894
|
|
|
1104
1895
|
declare function lintKnowledgeIndex(index: KnowledgeIndex): KnowledgeLintFinding[];
|
|
1105
1896
|
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1897
|
+
/**
|
|
1898
|
+
* `materialFactsSurfaced` — the held-out investment-research METRIC.
|
|
1899
|
+
*
|
|
1900
|
+
* Given a knowledge base a research loop built for a company and the company's
|
|
1901
|
+
* HELD-OUT material-fact checklist (`tests/eval/investment-thesis-set.ts`, never
|
|
1902
|
+
* shown to the loop), this returns the FRACTION of checklist items the KB's
|
|
1903
|
+
* pages surface + ground. The check is the same `$0`, model-free, deterministic
|
|
1904
|
+
* substring grader the loop's checklist already ships (`gradeFactAgainstText` /
|
|
1905
|
+
* `gradeCompanyAgainstText`) — so the answer key never reaches a model the loop
|
|
1906
|
+
* could observe, exactly the firewall the ML deep-question exam uses.
|
|
1907
|
+
*
|
|
1908
|
+
* The ONLY thing this file adds over the raw grader is the KB→text join: it reads
|
|
1909
|
+
* the curated pages (and the raw source text) the loop wrote and hands their
|
|
1910
|
+
* concatenation to the grader. That join mirrors `kbText` in the research-quality
|
|
1911
|
+
* A/B (research-driving-ab.test.ts) so the thesis metric and the ML-exam metric
|
|
1912
|
+
* read a KB the same way.
|
|
1913
|
+
*
|
|
1914
|
+
* WHY pages AND source text: an honest thesis surfaces a buried fact in its
|
|
1915
|
+
* curated thesis PAGE (the judgment), but a loop whose page is thin while its
|
|
1916
|
+
* fetched filings are rich should still get credit for what it actually pulled.
|
|
1917
|
+
* Grading the union is the faithful, not the lenient, choice — it rewards the
|
|
1918
|
+
* loop that REACHED the filing even if its synthesis was terse, and it cannot
|
|
1919
|
+
* manufacture a hit the underlying evidence does not contain.
|
|
1920
|
+
*/
|
|
1921
|
+
|
|
1922
|
+
/** Per-fact grade plus the fact's id/lens, for the audit trail. */
|
|
1923
|
+
interface FactResult {
|
|
1924
|
+
id: string;
|
|
1925
|
+
lens: CompanyEvalCase['facts'][number]['lens'];
|
|
1926
|
+
surfaced: boolean;
|
|
1927
|
+
groupsFound: number;
|
|
1928
|
+
groupsTotal: number;
|
|
1929
|
+
foundLabels: string[];
|
|
1930
|
+
}
|
|
1931
|
+
/** The metric's result for one company: the surfaced fraction + the per-fact trail. */
|
|
1932
|
+
interface MaterialFactsResult {
|
|
1933
|
+
ticker: string;
|
|
1934
|
+
company: string;
|
|
1935
|
+
/** Held-out facts the KB surfaced + grounded. */
|
|
1936
|
+
surfaced: number;
|
|
1937
|
+
/** Total held-out facts for this company (the denominator). */
|
|
1938
|
+
total: number;
|
|
1939
|
+
/** `surfaced / total` in [0, 1]. */
|
|
1940
|
+
fraction: number;
|
|
1941
|
+
/** Per-fact grade, in checklist order, for the doc / audit. */
|
|
1942
|
+
perFact: FactResult[];
|
|
1109
1943
|
}
|
|
1110
|
-
|
|
1111
|
-
|
|
1944
|
+
/**
|
|
1945
|
+
* Join a KB index into the single text blob the grader scans: every curated PAGE
|
|
1946
|
+
* (title + body) followed by every raw SOURCE (title + fetched text). This is the
|
|
1947
|
+
* text read AFTER the loop finished — it is never handed to the loop. Identical
|
|
1948
|
+
* in spirit to `kbText` in the research-quality A/B so both metrics read a KB the
|
|
1949
|
+
* same way.
|
|
1950
|
+
*/
|
|
1951
|
+
declare function kbIndexToText(index: KnowledgeIndex): string;
|
|
1952
|
+
/**
|
|
1953
|
+
* Grade one company's KB against its held-out material-fact checklist, given the
|
|
1954
|
+
* KB's already-joined text. The pure core — no I/O — so calibration can score a
|
|
1955
|
+
* hand-written shallow/deep thesis string directly and the live path can score a
|
|
1956
|
+
* real KB. Returns the surfaced FRACTION plus the per-fact audit trail.
|
|
1957
|
+
*/
|
|
1958
|
+
declare function materialFactsSurfacedInText(company: CompanyEvalCase, kbText: string): MaterialFactsResult;
|
|
1959
|
+
/**
|
|
1960
|
+
* `materialFactsSurfaced(kb, checklist)` — the metric in its KB-reading form.
|
|
1961
|
+
*
|
|
1962
|
+
* `kb` is EITHER a knowledge-base root directory (the loop wrote pages there) or
|
|
1963
|
+
* an already-built `KnowledgeIndex`. `checklist` is the company's held-out
|
|
1964
|
+
* `CompanyEvalCase`. Returns the surfaced fraction + per-fact trail.
|
|
1965
|
+
*
|
|
1966
|
+
* The checklist is HELD OUT by construction: it lives in the test eval set, is
|
|
1967
|
+
* never passed to the loop, and is read only here, after the loop finished.
|
|
1968
|
+
*/
|
|
1969
|
+
declare function materialFactsSurfaced(kb: string | KnowledgeIndex, checklist: CompanyEvalCase): Promise<MaterialFactsResult>;
|
|
1112
1970
|
|
|
1113
1971
|
/**
|
|
1114
1972
|
* Bridge from `AnalystFinding` (agent-eval) to knowledge proposals.
|
|
@@ -1196,6 +2054,30 @@ interface ProposeFromFindingsResult {
|
|
|
1196
2054
|
}
|
|
1197
2055
|
declare function proposeFromFindings(findings: ReadonlyArray<AnalystFinding>): ProposeFromFindingsResult;
|
|
1198
2056
|
|
|
2057
|
+
interface EvaluateKnowledgeBaseReadinessOptions {
|
|
2058
|
+
root: string;
|
|
2059
|
+
goal: string;
|
|
2060
|
+
readinessSpecs?: readonly KnowledgeReadinessSpec[];
|
|
2061
|
+
readinessTaskId?: string;
|
|
2062
|
+
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
2063
|
+
strict?: ValidateKnowledgeOptions['strict'];
|
|
2064
|
+
kbQuality?: KnowledgeBaseQualityOptions;
|
|
2065
|
+
}
|
|
2066
|
+
interface KnowledgeBaseReadinessEvaluation {
|
|
2067
|
+
ready: boolean;
|
|
2068
|
+
summary: string;
|
|
2069
|
+
index: KnowledgeIndex;
|
|
2070
|
+
validation: ValidateKnowledgeResult;
|
|
2071
|
+
readiness?: EvalKnowledgeBundleBuildResult;
|
|
2072
|
+
kbQuality: KnowledgeBaseQualityReport;
|
|
2073
|
+
dimensions: {
|
|
2074
|
+
validation: number;
|
|
2075
|
+
kb_quality: number;
|
|
2076
|
+
blocking_readiness: number;
|
|
2077
|
+
};
|
|
2078
|
+
}
|
|
2079
|
+
declare function evaluateKnowledgeBaseReadiness(options: EvaluateKnowledgeBaseReadinessOptions): Promise<KnowledgeBaseReadinessEvaluation>;
|
|
2080
|
+
|
|
1199
2081
|
interface KnowledgeReleaseReport {
|
|
1200
2082
|
release: KnowledgeRelease;
|
|
1201
2083
|
scorecard: ReleaseConfidenceScorecard;
|
|
@@ -1234,172 +2116,157 @@ interface KnowledgeReleaseInput {
|
|
|
1234
2116
|
}
|
|
1235
2117
|
declare function knowledgeReleaseReport(input: KnowledgeReleaseInput): KnowledgeReleaseReport;
|
|
1236
2118
|
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
2119
|
+
/**
|
|
2120
|
+
* Research-DRIVING driver for `runVerifiedResearchLoop`.
|
|
2121
|
+
*
|
|
2122
|
+
* The shipped drivers all FILTER the worker's sources:
|
|
2123
|
+
* - `createVerifyingResearchDriver` judges on-topic relevance,
|
|
2124
|
+
* - `createAdaptiveResearchDriver` dedups then triages then escalates,
|
|
2125
|
+
* - `createClaimGroundingVerifier` rejects misattributed citations.
|
|
2126
|
+
*
|
|
2127
|
+
* This driver does the OPPOSITE job: instead of narrowing the worker's output,
|
|
2128
|
+
* it DRIVES the research DEEPER each round. Its value is not "fewer sources" —
|
|
2129
|
+
* it is "more answered, better-corroborated sub-questions". Concretely, each
|
|
2130
|
+
* round it:
|
|
2131
|
+
*
|
|
2132
|
+
* 1. EXTRACTS the key claims from the worker's new sources (one LLM pass per
|
|
2133
|
+
* source, in `verifySource`; falls back to a deterministic sentence-pull
|
|
2134
|
+
* when the model is unavailable so a round never silently extracts nothing).
|
|
2135
|
+
* 2. TRACKS each claim's support — the set of INDEPENDENT sources (by canonical
|
|
2136
|
+
* host) that assert it — and detects CONTRADICTIONS between a new claim and
|
|
2137
|
+
* one already on the ledger.
|
|
2138
|
+
* 3. GENERATES the next round's DEEP sub-questions from the accumulated claims,
|
|
2139
|
+
* in four kinds — comparative ("how does X's tradeoff differ from Y's?"),
|
|
2140
|
+
* mechanism ("under what precise condition does X fail?"), gap ("what
|
|
2141
|
+
* specific result is missing?"), and contradiction ("does any source
|
|
2142
|
+
* challenge claim Z?").
|
|
2143
|
+
* 4. FLAGS weakly-supported claims (only ONE independent source) and
|
|
2144
|
+
* contradicted claims as INVALIDATION targets and demands the worker find
|
|
2145
|
+
* corroborating / refuting evidence for them.
|
|
2146
|
+
* 5. FOLDS the deep sub-questions + invalidation challenges into the worker's
|
|
2147
|
+
* next prompt via the loop's `foldGaps` → `steer` channel — that is the
|
|
2148
|
+
* mechanism that drives DEPTH and VALIDATION rather than breadth.
|
|
2149
|
+
*
|
|
2150
|
+
* COMPLETION (`isComplete` / the `done` judgment the caller gates on) does NOT
|
|
2151
|
+
* look at source COUNT. It is done only when every deep sub-question it raised
|
|
2152
|
+
* has been addressed AND every key claim is either supported by >= 2 independent
|
|
2153
|
+
* sources OR explicitly marked CONTESTED (a contradiction the loop surfaced and
|
|
2154
|
+
* could not resolve). A KB with twenty sources all asserting one unchallenged
|
|
2155
|
+
* claim is NOT done; a KB whose handful of claims are each corroborated or
|
|
2156
|
+
* contested IS.
|
|
2157
|
+
*
|
|
2158
|
+
* It reuses `runVerifiedResearchLoop` (it is a plain `ResearchDriver`), the web
|
|
2159
|
+
* worker, `sha256` (claim identity), `canonicalizeUrl` (independent-source
|
|
2160
|
+
* identity), and the `RouterClient` chat surface; it reinvents none of them.
|
|
2161
|
+
*/
|
|
1245
2162
|
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
/**
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
2163
|
+
/** The four deep sub-question kinds the driver generates to drive depth. */
|
|
2164
|
+
type DeepQuestionKind = 'comparative' | 'mechanism' | 'gap' | 'contradiction';
|
|
2165
|
+
/** A deep sub-question the driver folds into the worker's next prompt. */
|
|
2166
|
+
interface DeepQuestion {
|
|
2167
|
+
kind: DeepQuestionKind;
|
|
2168
|
+
text: string;
|
|
2169
|
+
/** sha256-derived stable id, so "addressed" can be tracked across rounds. */
|
|
2170
|
+
id: string;
|
|
2171
|
+
/** Claim id(s) this question interrogates (for contradiction/mechanism kinds). */
|
|
2172
|
+
claimIds: string[];
|
|
2173
|
+
/** True once a later round's evidence addressed it (see `markAddressed`). */
|
|
2174
|
+
addressed: boolean;
|
|
2175
|
+
/** The round this question was raised in. */
|
|
2176
|
+
raisedRound: number;
|
|
2177
|
+
}
|
|
2178
|
+
/** One tracked claim plus the independent sources that assert it. */
|
|
2179
|
+
interface TrackedClaim {
|
|
2180
|
+
id: string;
|
|
2181
|
+
/** The claim text as first extracted (kept for prompts/audit). */
|
|
2182
|
+
text: string;
|
|
2183
|
+
/** Canonical hosts of the INDEPENDENT sources that assert this claim. */
|
|
2184
|
+
supportingHosts: Set<string>;
|
|
2185
|
+
/** Source URIs that assert this claim (provenance; may share a host). */
|
|
2186
|
+
supportingUris: string[];
|
|
2187
|
+
/** Claim ids this claim was found to CONTRADICT (and vice versa). */
|
|
2188
|
+
contradicts: Set<string>;
|
|
1263
2189
|
/**
|
|
1264
|
-
*
|
|
2190
|
+
* CONTESTED = a contradiction the loop surfaced but could not resolve to a
|
|
2191
|
+
* single supported claim. A contested claim counts as "settled enough to be
|
|
2192
|
+
* done" (we report the disagreement) even with < 2 independent sources.
|
|
1265
2193
|
*/
|
|
1266
|
-
|
|
2194
|
+
contested: boolean;
|
|
2195
|
+
firstSeenRound: number;
|
|
2196
|
+
}
|
|
2197
|
+
/** The driver's accumulated research state — the completion oracle reads this. */
|
|
2198
|
+
interface ResearchDrivingState {
|
|
2199
|
+
/** Every claim extracted from the worker's sources, by id. */
|
|
2200
|
+
claims: TrackedClaim[];
|
|
2201
|
+
/** Every deep sub-question raised, by id. */
|
|
2202
|
+
questions: DeepQuestion[];
|
|
2203
|
+
/** Claims with exactly one independent source AND not contested. */
|
|
2204
|
+
weaklySupported: TrackedClaim[];
|
|
2205
|
+
/** Claims supported by >= 2 independent sources. */
|
|
2206
|
+
corroborated: TrackedClaim[];
|
|
2207
|
+
/** Claims marked contested (a surfaced, unresolved contradiction). */
|
|
2208
|
+
contested: TrackedClaim[];
|
|
2209
|
+
/** Deep questions still unaddressed. */
|
|
2210
|
+
openQuestions: DeepQuestion[];
|
|
2211
|
+
/** How many rounds the driver has folded steer for. */
|
|
2212
|
+
rounds: number;
|
|
2213
|
+
}
|
|
2214
|
+
interface ResearchDrivingDriverOptions {
|
|
2215
|
+
/** Router client for claim extraction + deep-question generation. */
|
|
2216
|
+
router?: RouterClient;
|
|
2217
|
+
router_options?: TangleRouterOptions;
|
|
1267
2218
|
/**
|
|
1268
|
-
*
|
|
1269
|
-
*
|
|
2219
|
+
* A claim is CORROBORATED at this many INDEPENDENT supporting sources (distinct
|
|
2220
|
+
* canonical hosts). Default 2 — the task's ">= 2 independent sources" bar.
|
|
1270
2221
|
*/
|
|
1271
|
-
|
|
2222
|
+
minIndependentSources?: number;
|
|
2223
|
+
/** Max deep sub-questions to fold into one round's steer. Default 6. */
|
|
2224
|
+
maxQuestionsPerRound?: number;
|
|
2225
|
+
/** Max claims to extract from a single source. Default 3. */
|
|
2226
|
+
maxClaimsPerSource?: number;
|
|
1272
2227
|
/**
|
|
1273
|
-
*
|
|
1274
|
-
*
|
|
2228
|
+
* When the extractor LLM is unavailable, fall back to a deterministic claim
|
|
2229
|
+
* pull (the source's leading sentences) so the driver still drives. Default
|
|
2230
|
+
* true. Set false to require the model (claims will be empty without it).
|
|
1275
2231
|
*/
|
|
1276
|
-
|
|
1277
|
-
/**
|
|
1278
|
-
|
|
1279
|
-
* does not encode a domain-specific definition of "done".
|
|
1280
|
-
*/
|
|
1281
|
-
done?: boolean;
|
|
1282
|
-
metadata?: Record<string, unknown>;
|
|
2232
|
+
deterministicFallback?: boolean;
|
|
2233
|
+
/** Observe each round's generated steer (for instrumentation / the script). */
|
|
2234
|
+
onSteer?: (steer: ResearchDrivingSteer) => void;
|
|
1283
2235
|
}
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
metadata?: Record<string, unknown>;
|
|
1295
|
-
}
|
|
1296
|
-
interface RunKnowledgeResearchLoopOptions {
|
|
1297
|
-
root: string;
|
|
1298
|
-
goal: string;
|
|
1299
|
-
maxIterations?: number;
|
|
1300
|
-
actor?: string;
|
|
1301
|
-
strict?: ValidateKnowledgeOptions['strict'];
|
|
1302
|
-
readinessSpecs?: KnowledgeReadinessSpec[];
|
|
1303
|
-
readinessTaskId?: string;
|
|
1304
|
-
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
1305
|
-
sourceOptions?: Pick<AddSourceOptions, 'adapters' | 'now'>;
|
|
1306
|
-
signal?: AbortSignal;
|
|
1307
|
-
step(context: KnowledgeResearchLoopContext): Promise<KnowledgeResearchLoopDecision> | KnowledgeResearchLoopDecision;
|
|
1308
|
-
onStep?: (step: KnowledgeResearchLoopStep) => Promise<void> | void;
|
|
1309
|
-
}
|
|
1310
|
-
interface KnowledgeResearchLoopResult {
|
|
1311
|
-
root: string;
|
|
1312
|
-
goal: string;
|
|
1313
|
-
iterations: number;
|
|
1314
|
-
done: boolean;
|
|
1315
|
-
index: KnowledgeIndex;
|
|
1316
|
-
lintFindings: KnowledgeLintFinding[];
|
|
1317
|
-
validation: ValidateKnowledgeResult;
|
|
1318
|
-
readiness?: EvalKnowledgeBundleBuildResult;
|
|
1319
|
-
steps: KnowledgeResearchLoopStep[];
|
|
1320
|
-
}
|
|
1321
|
-
type KnowledgeControlLoopState = KnowledgeResearchLoopContext;
|
|
1322
|
-
type KnowledgeControlLoopAction = KnowledgeResearchLoopDecision;
|
|
1323
|
-
type KnowledgeControlLoopActionResult = KnowledgeResearchLoopStep;
|
|
1324
|
-
interface KnowledgeControlLoopAdapterOptions {
|
|
1325
|
-
root: string;
|
|
1326
|
-
goal: string;
|
|
1327
|
-
actor?: string;
|
|
1328
|
-
strict?: ValidateKnowledgeOptions['strict'];
|
|
1329
|
-
readinessSpecs?: KnowledgeReadinessSpec[];
|
|
1330
|
-
readinessTaskId?: string;
|
|
1331
|
-
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
1332
|
-
sourceOptions?: Pick<AddSourceOptions, 'adapters' | 'now'>;
|
|
2236
|
+
/** What the driver folded into one round's worker prompt, surfaced for audit. */
|
|
2237
|
+
interface ResearchDrivingSteer {
|
|
2238
|
+
round: number;
|
|
2239
|
+
deepQuestions: DeepQuestion[];
|
|
2240
|
+
/** Claims it demanded corroborating/refuting evidence for this round. */
|
|
2241
|
+
invalidationTargets: TrackedClaim[];
|
|
2242
|
+
/** The readiness gaps it interleaved (passed through from the loop). */
|
|
2243
|
+
gaps: KnowledgeGap[];
|
|
2244
|
+
/** The full steer text handed to the worker. */
|
|
2245
|
+
text: string;
|
|
1333
2246
|
}
|
|
1334
|
-
type KnowledgeControlLoopAdapter = Pick<ControlRuntimeConfig<KnowledgeControlLoopState, KnowledgeControlLoopAction, KnowledgeControlLoopActionResult, ControlEvalResult>, 'intent' | 'observe' | 'validate' | 'act' | 'shouldStop'>;
|
|
1335
2247
|
/**
|
|
1336
|
-
*
|
|
1337
|
-
*
|
|
1338
|
-
*
|
|
2248
|
+
* The research-driving driver. It is a `ResearchDriver` (drops straight into
|
|
2249
|
+
* `runVerifiedResearchLoop`) PLUS a completion oracle and live state, mirroring
|
|
2250
|
+
* how `createAdaptiveResearchDriver` exposes `stats()`.
|
|
1339
2251
|
*/
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
/**
|
|
1344
|
-
* Default standing instructions for a research supervisor. It does NOT solve the
|
|
1345
|
-
* research itself — it DECOMPOSES the goal into sub-topics and spawns one
|
|
1346
|
-
* researcher per sub-topic over the live `Scope`, widening/narrowing until the
|
|
1347
|
-
* readiness gate (the deliverable check) reports the knowledge base is ready.
|
|
1348
|
-
*/
|
|
1349
|
-
declare const RESEARCH_SUPERVISOR_SYSTEM_PROMPT: string;
|
|
1350
|
-
interface ResearchSupervisorOptions {
|
|
1351
|
-
/** Where the knowledge base lives. The deliverable check reads readiness here. */
|
|
1352
|
-
root: string;
|
|
1353
|
-
/** The research goal handed to the supervisor as its task. */
|
|
1354
|
-
goal: string;
|
|
2252
|
+
interface ResearchDrivingDriver extends ResearchDriver {
|
|
2253
|
+
/** Live snapshot of the claim ledger + deep questions. */
|
|
2254
|
+
researchState(): ResearchDrivingState;
|
|
1355
2255
|
/**
|
|
1356
|
-
*
|
|
1357
|
-
*
|
|
1358
|
-
*
|
|
2256
|
+
* The completion oracle — gate `done` on THIS, not on source count. True when
|
|
2257
|
+
* every deep sub-question is addressed AND every claim is corroborated
|
|
2258
|
+
* (>= `minIndependentSources` independent sources) or explicitly contested.
|
|
2259
|
+
* False while any claim is weakly-supported or any deep question is open.
|
|
2260
|
+
* Returns false before any claim has been seen (nothing researched yet).
|
|
1359
2261
|
*/
|
|
1360
|
-
|
|
1361
|
-
readinessTaskId?: string;
|
|
1362
|
-
readiness?: Omit<BuildEvalKnowledgeBundleOptions, 'taskId' | 'index' | 'specs'>;
|
|
1363
|
-
/** The conserved compute pool for the whole supervised run. */
|
|
1364
|
-
budget: Budget;
|
|
2262
|
+
isComplete(): boolean;
|
|
1365
2263
|
/**
|
|
1366
|
-
*
|
|
1367
|
-
*
|
|
1368
|
-
* on this backend; assembling the seam (creds, sandbox SDK) is the caller's,
|
|
1369
|
-
* so there is no fabricated default.
|
|
2264
|
+
* The last round's generated steer, or undefined before the first fold. Useful
|
|
2265
|
+
* to assert the driver produced deeper questions / invalidation challenges.
|
|
1370
2266
|
*/
|
|
1371
|
-
|
|
1372
|
-
/** Harness for the researcher worker profile (default `researcherProfile`'s). */
|
|
1373
|
-
harness?: string;
|
|
1374
|
-
/** The supervisor brain's router model. */
|
|
1375
|
-
supervisorModel?: string;
|
|
1376
|
-
/** Custom supervisor instructions. Defaults to {@link RESEARCH_SUPERVISOR_SYSTEM_PROMPT}. */
|
|
1377
|
-
supervisorSystemPrompt?: string;
|
|
1378
|
-
/** Extra `supervise()` knobs (perWorker, maxLiveWorkers, router, allowedModels, …). */
|
|
1379
|
-
superviseOptions?: Partial<Omit<SuperviseOptions, 'budget' | 'backend' | 'deliverable'>>;
|
|
2267
|
+
lastSteer(): ResearchDrivingSteer | undefined;
|
|
1380
2268
|
}
|
|
1381
|
-
|
|
1382
|
-
* Build the deliverable (completion oracle) for a research supervisor: "settled
|
|
1383
|
-
* ⟺ the knowledge base is ready". It re-reads the KB from disk and runs the
|
|
1384
|
-
* readiness gate, so the supervisor stops on the REAL grounded state — not on a
|
|
1385
|
-
* worker's self-report.
|
|
1386
|
-
*/
|
|
1387
|
-
declare function knowledgeReadinessDeliverable(options: Pick<ResearchSupervisorOptions, 'root' | 'goal' | 'readinessSpecs' | 'readinessTaskId' | 'readiness'>): DeliverableSpec<unknown>;
|
|
1388
|
-
/**
|
|
1389
|
-
* A research supervisor: a SUPERVISOR brain creates the topology dynamically —
|
|
1390
|
-
* how many researchers, split by sub-topic — over the `Scope`, with the
|
|
1391
|
-
* knowledge-readiness gate as its stop condition.
|
|
1392
|
-
*
|
|
1393
|
-
* This is a thin wrapper over `supervise(profile, goal, opts)`. It composes the
|
|
1394
|
-
* existing pieces and builds nothing new: the researcher `AgentProfile` (from
|
|
1395
|
-
* `researcherProfile`) is the worker shape the supervisor spawns, and
|
|
1396
|
-
* `knowledgeReadinessDeliverable` (over `scoreKnowledgeReadiness`) is the
|
|
1397
|
-
* completion oracle that decides when the KB is ready.
|
|
1398
|
-
*
|
|
1399
|
-
* Needs creds (a real supervisor router brain + a worker backend), so it is the
|
|
1400
|
-
* LIVE path. The offline two-agent loop is `runTwoAgentResearchLoop`.
|
|
1401
|
-
*/
|
|
1402
|
-
declare function runResearchSupervisor(options: ResearchSupervisorOptions): Promise<SupervisedResult<unknown>>;
|
|
2269
|
+
declare function createResearchDrivingDriver(options?: ResearchDrivingDriverOptions): ResearchDrivingDriver;
|
|
1403
2270
|
|
|
1404
2271
|
declare const SourceAnchorSchema: z.ZodObject<{
|
|
1405
2272
|
id: z.ZodString;
|
|
@@ -1613,4 +2480,4 @@ declare function normalizeLinkTarget(target: string): string;
|
|
|
1613
2480
|
declare function isSafeKnowledgePath(path: string, allowedPrefixes?: string[]): boolean;
|
|
1614
2481
|
declare function parseKnowledgeWriteBlocks(text: string, allowedPrefixes?: string[]): KnowledgeWriteParseResult;
|
|
1615
2482
|
|
|
1616
|
-
export { type AdaptiveDecision, type AdaptiveDriverOptions, type AdaptiveResearchDriver, type AdaptiveStats, type AddSourceOptions, type AddSourceTextInput, type ApplyWriteBlocksResult, type BuildEvalKnowledgeBundleOptions, type ChunkingOptions, type ClaimGroundingDriverOptions, type D1Adapter, type DedupReason, type DefineReadinessSpecInput, type DetectChangesOptions, type DetectChangesResult, type DiscoveryResult, type DiscoveryTask, type DriverResearchContext, type EvalKnowledgeBundleBuildResult, type FileSystemFreshnessStoreOptions, FileSystemKbStore, type FreshnessKey, type FreshnessMark, type FreshnessRecord, type FreshnessTtl, type GroundClaimOptions, type GroundingResult, type KbStore, KnowledgeBaseCandidateSchema, type KnowledgeChange, type KnowledgeChangeKind, type KnowledgeChunk, KnowledgeClaim, type KnowledgeControlLoopAction, type KnowledgeControlLoopActionResult, type KnowledgeControlLoopAdapter, type KnowledgeControlLoopAdapterOptions, type KnowledgeControlLoopState, type KnowledgeDiscoveryDispatcher, type KnowledgeDiscoveryWorker, KnowledgeEvent, type KnowledgeEventQuery, KnowledgeEventSchema, KnowledgeEventType, type KnowledgeExplanation, KnowledgeFragment, type KnowledgeFreshnessStore, type KnowledgeGap, KnowledgeGraph, KnowledgeGraphEdgeSchema, KnowledgeGraphNodeSchema, KnowledgeIndex, KnowledgeIndexSchema, type KnowledgeInspection, type KnowledgeLayout, KnowledgeLintFinding, KnowledgePage, KnowledgePageSchema, type KnowledgeProposal, KnowledgeProposalParseError, type KnowledgeReadinessSpec, KnowledgeRelease, type KnowledgeReleaseInput, type KnowledgeReleaseReport, type KnowledgeResearchLoopContext, type KnowledgeResearchLoopDecision, type KnowledgeResearchLoopResult, type KnowledgeResearchLoopStep, KnowledgeSearchResult, KnowledgeWriteBlock, KnowledgeWriteParseResult, MemoryKbStore, type ParsedFrontmatter, type ProposeFromFindingsResult, READINESS_SPEC_DEFAULTS,
|
|
2483
|
+
export { type AdaptiveDecision, type AdaptiveDriverOptions, type AdaptiveResearchDriver, type AdaptiveStats, type AddSourceOptions, type AddSourceTextInput, type ApplyWriteBlocksResult, type BuildEvalKnowledgeBundleOptions, type ChunkingOptions, type ClaimGroundingDriverOptions, type CompanyEvalCase, type D1Adapter, type DedupReason, type DeepQuestion, type DeepQuestionKind, type DefineReadinessSpecInput, type DetectChangesOptions, type DetectChangesResult, type DiscoveryResult, type DiscoveryTask, type DriverResearchContext, type EvalKnowledgeBundleBuildResult, type EvaluateKnowledgeBaseReadinessOptions, type ExpectedGroup, type ExternalRagEvalScore, type FactResult, type FileSystemFreshnessStoreOptions, FileSystemKbStore, type FreshnessKey, type FreshnessMark, type FreshnessRecord, type FreshnessTtl, type GroundClaimOptions, type GroundingResult, type KbStore, KnowledgeBaseCandidateSchema, type KnowledgeBaseQualityOptions, type KnowledgeBaseQualityReport, type KnowledgeBaseReadinessEvaluation, type KnowledgeChange, type KnowledgeChangeKind, type KnowledgeChunk, KnowledgeClaim, type KnowledgeControlLoopAction, type KnowledgeControlLoopActionResult, type KnowledgeControlLoopAdapter, type KnowledgeControlLoopAdapterOptions, type KnowledgeControlLoopState, type KnowledgeDiscoveryDispatcher, type KnowledgeDiscoveryWorker, KnowledgeEvent, type KnowledgeEventQuery, KnowledgeEventSchema, KnowledgeEventType, type KnowledgeExplanation, KnowledgeFragment, type KnowledgeFreshnessStore, type KnowledgeGap, KnowledgeGraph, KnowledgeGraphEdgeSchema, KnowledgeGraphNodeSchema, type KnowledgeImprovementCandidateRecord, type KnowledgeImprovementEvaluationInput, type KnowledgeImprovementEvaluator, type KnowledgeImprovementLifecycleRecord, type KnowledgeImprovementMetric, type KnowledgeImprovementOptions, type KnowledgeImprovementResult, type KnowledgeImprovementRetrievalOptions, type KnowledgeImprovementRunState, type KnowledgeImprovementStatus, type KnowledgeImprovementUpdate, type KnowledgeImprovementUpdateInput, KnowledgeIndex, KnowledgeIndexSchema, type KnowledgeInspection, type KnowledgeLayout, KnowledgeLintFinding, KnowledgePage, KnowledgePageSchema, type KnowledgeProposal, KnowledgeProposalParseError, type KnowledgeReadinessSpec, KnowledgeRelease, type KnowledgeReleaseInput, type KnowledgeReleaseReport, type KnowledgeResearchLoopContext, type KnowledgeResearchLoopDecision, type KnowledgeResearchLoopResult, type KnowledgeResearchLoopStep, KnowledgeSearchResult, KnowledgeWriteBlock, KnowledgeWriteParseResult, type MaterialFact, type MaterialFactLens, type MaterialFactsResult, MemoryKbStore, type ParsedFrontmatter, type ProposeFromFindingsResult, READINESS_SPEC_DEFAULTS, type RagAnswerEvalArtifact, type RagAnswerEvalCase, type RagAnswerEvalScenario, type RagAnswerMetricSummary, type RagAnswerQualityHookOptions, type RagAnswerQualityInput, type RagAnswerQualityJudgeOptions, type RagAnswerQualityResult, type RagCalibrationOptions, type RagCalibrationResult, type RagDiagnosisInput, type RagEvalCitation, type RagEvalClaim, type RagEvalContext, type RagEvalMetricKey, type RagEvalProvider, type RagEvalSlice, type RagGapFinding, type RagGapKind, type RagGapSeverity, type RagKnowledgeAcquisitionInput, type RagKnowledgeImprovementPhase, type RagKnowledgeImprovementPhaseResult, type RagKnowledgeImprovementPhaseStatus, type RagKnowledgeResearchOptions, type RagKnowledgeUpdateInput, type RagKnowledgeUpdateResult, type RagPhaseInputBase, type RagPromotionInput, type RagPromotionResult, type RagRequiredContext, type RejectedSource, type ResearchContribution, type ResearchDriver, type ResearchDrivingDriver, type ResearchDrivingDriverOptions, type ResearchDrivingState, type ResearchDrivingSteer, type ResearchSourceProposal, type ResearchWorker, RetrievalConfig, type RouterClient, RouterError, type RouterUsage, type RunKnowledgeResearchLoopOptions, type RunRagKnowledgeImprovementLoopOptions, type RunRagKnowledgeImprovementLoopResult, RunRetrievalImprovementLoopOptions, RunRetrievalImprovementLoopResult, SCAFFOLD_PAGE_BASENAMES, type SourceAdapter, type SourceAdapterInput, type SourceAdapterOutput, SourceAnchorSchema, type SourceFreshnessInspection, SourceRecord, SourceRecordSchema, SourceRegistry, type SourceVerdict, type SourceVerificationContext, type TangleRouterOptions, type ThesisRunOptions, type ThesisRunResult, type ThesisTaskInput, type TrackedClaim, type TriageClass, type TwoAgentResearchLoopOptions, type TwoAgentResearchLoopResult, type TwoAgentResearchRound, type ValidateKnowledgeOptions, type ValidateKnowledgeResult, type VerifiedResearchLoopOptions, type VerifiedResearchLoopResult, type VerifiedResearchRound, type VerifyingDriverOptions, WIKILINK_REGEX, type WebResearchWorkerOptions, type WebSearchHit, type WorkerClaimDecorationOptions, type WorkerResearchContext, addSourcePath, addSourceText, applyKnowledgeWriteBlocks, applyKnowledgeWriteBlocksFile, buildEvalKnowledgeBundle, buildKnowledgeGraph, buildKnowledgeIndex, calibrateRagAnswerJudge, canonicalizeUrl, chunkMarkdown, citedClaimKey, citedClaimOf, contentKey, createAdaptiveResearchDriver, createClaimDecorator, createClaimGroundingVerifier, createCollectionResearchDriver, createD1FreshnessStoreStub, createFileSystemFreshnessStore, createKnowledgeControlLoopAdapter, createKnowledgeEvent, createLocalDiscoveryDispatcher, createRagAnswerQualityHook, createResearchDrivingDriver, createTangleRouterClient, createVerifyingResearchDriver, createWebResearchWorker, defineReadinessSpec, detectChanges, diagnoseRagAnswerFailure, evaluateKnowledgeBaseReadiness, explainKnowledgeTarget, extractWikilinks, formatFrontmatter, gradeCompanyAgainstText, gradeFactAgainstText, groundClaimInText, hashKnowledgeBase, improveKnowledgeBase, initKnowledgeBase, inspectKnowledgeIndex, investmentThesisSet, isSafeKnowledgePath, isScaffoldPath, kbIndexToText, knowledgeImprovementRunDir, knowledgeImprovementRunId, knowledgeReleaseReport, layoutFor, lensDistribution, lintKnowledgeIndex, loadKnowledgeImprovementState, loadKnowledgePages, loadSourceRegistry, materialFactsSurfaced, materialFactsSurfacedInText, mediaTypeFor, normalizeExternalRagScores, normalizeLinkTarget, parseFrontmatter, parseKnowledgeWriteBlocks, proposeFromFinding, proposeFromFindings, ragAnswerQualityJudge, reciprocalRankFusion, runInvestmentThesisTask, runKnowledgeResearchLoop, runRagKnowledgeImprovementLoop, runTwoAgentResearchLoop, runVerifiedResearchLoop, scoreKnowledgeBaseIndex, scoreRagAnswerArtifact, searchKnowledge, sha256, slugify, sourceMatchesGaps, sourceRegistryPath, stableId, stripFrontmatter, textSourceAdapter, thesisReadinessSpecs, toDeepEvalTestCases, toRagCheckerRecords, toRagasEvaluationRows, toTruLensRecords, tokenizeQuery, totalMaterialFacts, triageSource, validateKnowledgeIndex, withCitedClaim, writeJson, writeKnowledgeIndex, writeSourceRegistry };
|