@tangle-network/agent-knowledge 4.0.1 → 5.0.0

package/dist/memory/index.js

@@ -24,7 +24,7 @@ import {
   runAgentMemoryExperiment,
   runAgentMemoryImprovement,
   toOffPolicyTrajectory
-} from "../chunk-3CAAGJCE.js";
+} from "../chunk-LMR53POQ.js";
 import {
   AgentMemoryLifecycleTimeoutError,
   AgentMemoryLifecycleUnsafeError,
@@ -37,7 +37,7 @@ import {
   resolveMemoryCleanupTimeoutMs,
   runBoundedMemoryLifecycle,
   sleepForMemoryRecovery
-} from "../chunk-BBCIB4UL.js";
+} from "../chunk-EYIA5PLQ.js";
 import "../chunk-DQ3PDMDP.js";
 import "../chunk-YMKHCTS2.js";
 export {

package/dist/sources/index.d.ts

@@ -15,9 +15,9 @@
  * Sources MUST be pure with respect to local filesystem state outside the
  * cache directory the caller hands them — they read remote authorities and
  * return data. They MUST mark `verifiable: false` on any fragment they could
- * not authenticate (block page, 4xx, parse failure) rather than silently
- * substituting empty/partial content. The control loop downstream uses
- * `verifiable` to refuse promotion of un-grounded content.
+ * not fetch and extract (block page, 4xx, parse failure) rather than silently
+ * substituting empty or partial content. The control loop downstream uses
+ * `verifiable` to refuse promotion of unusable content.
  *
  * @stable
  */
@@ -77,10 +77,11 @@ interface FragmentProvenance {
      */
     jurisdiction?: string;
     /**
-     * True iff the source could authenticate the fetched content (HTTP 200,
-     * expected selectors present, parse succeeded). False on any block page,
-     * rate-limit response, 4xx/5xx, or selector miss. Consumers MUST refuse
-     * to promote `verifiable: false` fragments into citable knowledge.
+     * True iff the configured URL returned an acceptable response and the
+     * expected content was extracted. False on a block page, rate-limit
+     * response, 4xx/5xx, or selector miss. This is not publisher authentication
+     * or cryptographic content verification. Consumers MUST refuse to promote
+     * `verifiable: false` fragments into citable knowledge.
      */
     verifiable: boolean;
     /** If `verifiable === false`, the reason — surfaced to operators. */
@@ -222,33 +223,16 @@ declare function extractLinks(html: string, hrefPattern: RegExp, baseUrl: string
 }[];
 
 /**
- * Polite HTTP fetcher used by every shipped source.
+ * Polite HTTP fetcher shared by remote sources.
  *
- * Three invariants this enforces — each was a bug found while wiring real
- * authorities; do not regress:
- *
- *   1. Per-host throttling. Cornell LII serves under 1 req/s/origin
- *      politely and will start serving block pages above that. The lock
- *      is per-host (`hostThrottle`) rather than per-source so that two
- *      independent sources targeting the same authority still cooperate.
- *
- *   2. On-disk content cache keyed by URL. Production sources are called
- *      from a cron loop; without a cache, every run re-hits the same
- *      pages and inflates change-detection false-positives (the authority
- *      occasionally serves slightly different boilerplate). The cache is
- *      content-addressed by URL, not by ETag — authorities like IRS.gov
- *      do not consistently send ETag/Last-Modified.
- *
- *   3. Block-page detection on success. A 200 with a captcha body still
- *      means "we couldn't authenticate." Sources downstream rely on
- *      `verifiable` to refuse promotion — losing that signal because the
- *      fetcher said "well, the status code was 200" is the bug class
- *      this exists to prevent.
- *
- * @stable
+ * Independent sources share a per-origin throttle because rate-limited sites
+ * may return block pages instead of 429 responses. Responses are cached by URL
+ * because many publishers omit reliable ETag and Last-Modified headers. Bodies
+ * are checked even after a 2xx response because captcha and block pages often
+ * use successful status codes.
  */
 /** User-Agent string sent on every outbound request. */
-declare const POLITE_USER_AGENT = "agent-knowledge/0.2.0 (+https://github.com/tangle-network/agent-knowledge)";
+declare const POLITE_USER_AGENT = "agent-knowledge (+https://github.com/tangle-network/agent-knowledge)";
 /** Minimum gap between successive requests to the same origin (ms). */
 declare const MIN_REQUEST_GAP_MS = 1000;
 /** Maximum response body we will buffer in memory (bytes). */
@@ -257,7 +241,7 @@ interface PoliteFetchOptions {
     signal?: AbortSignal;
     cacheDir?: string;
     /**
-     * Cache age beyond which we re-fetch. Default 1 hour — long enough to
+     * Cache age beyond which we re-fetch. Default 1 hour, long enough to
      * batch a cron sweep across many selectors, short enough that hourly
      * authoritative-page changes get picked up next tick.
      */
@@ -291,7 +275,7 @@ interface PoliteFetchResult {
 }
 /**
  * Fetch one URL with per-host throttling, on-disk cache, and block-page
- * detection. Never throws on network/HTTP failure — returns a result with
+ * detection. Never throws on network/HTTP failure. It returns a result with
  * `verifiable: false` and `unverifiableReason` set so the caller can decide
  * whether to skip, retry, or surface.
  *

package/dist/sources/index.js

@@ -13,7 +13,7 @@ import {
   innerHtmlById,
   looksLikeBlockPage,
   politeFetch
-} from "../chunk-WCYW2GDA.js";
+} from "../chunk-MYFM6LKH.js";
 import "../chunk-YMKHCTS2.js";
 export {
   IRS_DIMENSION_HINTS,

package/dist/{types-ZzY_x0r7.d.ts → types-BY-xLVw-.d.ts}

@@ -1,8 +1,8 @@
-import { JsonValue, Scenario, DispatchContext, RunImprovementLoopResult, ParameterCandidate, JudgeConfig, Gate, RunImprovementLoopOptions, MutableSurface, SurfaceProposer, CampaignStorage, CostLedgerHandle, CampaignResult } from '@tangle-network/agent-eval/campaign';
+import { Scenario, DispatchContext, MutableSurface, JudgeConfig, CampaignStorage, CostLedgerHandle, CampaignResult } from '@tangle-network/agent-eval/campaign';
 import { c as KnowledgeIndex, S as SourceRecord } from './types-6x0OpfW6.js';
+import { AgentCandidateJsonValue } from '@tangle-network/agent-interface';
 
-type RetrievalConfig = Record<string, JsonValue>;
-type RetrievalParameterSearchSpace = Record<string, readonly JsonValue[]>;
+type RetrievalConfig = Record<string, AgentCandidateJsonValue>;
 type RetrievalGoldTarget = {
     kind: 'page';
     pageId: string;
@@ -44,7 +44,7 @@ interface RetrievedKnowledgeHit {
     sourceIds?: readonly string[];
     sourceSpans?: readonly RetrievedSourceSpan[];
     snippet?: string;
-    metadata?: Record<string, JsonValue>;
+    metadata?: Record<string, AgentCandidateJsonValue>;
 }
 interface RetrievalEvalArtifact {
     config: RetrievalConfig;
@@ -54,7 +54,7 @@ interface RetrievalEvalArtifact {
     durationMs: number;
     /** Informational copy. Billable retrievers account through context.cost.runPaidCall. */
     costUsd?: number;
-    metadata?: Record<string, JsonValue>;
+    metadata?: Record<string, AgentCandidateJsonValue>;
 }
 interface RetrievalMetricSummary {
     recall: number;
@@ -79,7 +79,7 @@ interface RetrievalEvalRetrieverResult {
     hits: readonly RetrievedKnowledgeHit[];
     /** Informational copy. Billable retrievers account through context.cost.runPaidCall. */
     costUsd?: number;
-    metadata?: Record<string, JsonValue>;
+    metadata?: Record<string, AgentCandidateJsonValue>;
 }
 type RetrievalEvalRetriever = (input: RetrievalEvalRetrieverInput) => Promise<readonly RetrievedKnowledgeHit[] | RetrievalEvalRetrieverResult>;
 interface BuildRetrievalEvalDispatchOptions {
@@ -97,65 +97,22 @@ interface RetrievalRecallJudgeOptions {
     name?: string;
     weights?: RetrievalMetricWeights;
 }
-interface BuildRetrievalParameterCandidatesOptions {
-    baseline?: RetrievalConfig;
-}
-interface RetrievalParameterSweepProposerOptions {
-    candidates?: readonly ParameterCandidate[];
-    searchSpace?: RetrievalParameterSearchSpace;
-    baseline?: RetrievalConfig;
+interface PartitionRetrievalScenariosOptions {
+    selectionFraction?: number;
+    finalFraction?: number;
+    seed?: number;
 }
-type RetrievalLoopBaseOptions = RunImprovementLoopOptions<RetrievalEvalScenario, RetrievalEvalArtifact>;
-interface RunRetrievalImprovementLoopOptions {
-    baseline: RetrievalConfig;
-    scenarios: readonly RetrievalEvalScenario[];
-    holdoutScenarios?: readonly RetrievalEvalScenario[];
-    index?: KnowledgeIndex;
-    defaultK?: number;
-    retrieve?: RetrievalEvalRetriever;
-    candidates?: readonly ParameterCandidate[];
-    searchSpace?: RetrievalParameterSearchSpace;
-    judges?: readonly JudgeConfig<RetrievalEvalArtifact, RetrievalEvalScenario>[];
-    gate?: Gate<RetrievalEvalArtifact, RetrievalEvalScenario>;
-    metricWeights?: RetrievalMetricWeights;
-    targetRecall?: number;
-    holdoutFraction?: number;
-    splitSeed?: number;
-    deltaThreshold?: number;
-    runDir?: RetrievalLoopBaseOptions['runDir'];
-    seed?: RetrievalLoopBaseOptions['seed'];
-    reps?: RetrievalLoopBaseOptions['reps'];
-    resumable?: RetrievalLoopBaseOptions['resumable'];
-    costCeiling?: RetrievalLoopBaseOptions['costCeiling'];
-    maxConcurrency?: RetrievalLoopBaseOptions['maxConcurrency'];
-    dispatchTimeoutMs?: RetrievalLoopBaseOptions['dispatchTimeoutMs'];
-    expectUsage?: RetrievalLoopBaseOptions['expectUsage'];
-    tracing?: RetrievalLoopBaseOptions['tracing'];
-    storage?: RetrievalLoopBaseOptions['storage'];
-    populationSize?: RetrievalLoopBaseOptions['populationSize'];
-    maxGenerations?: RetrievalLoopBaseOptions['maxGenerations'];
-    promoteTopK?: RetrievalLoopBaseOptions['promoteTopK'];
-    maxImprovementShots?: RetrievalLoopBaseOptions['maxImprovementShots'];
-    report?: RetrievalLoopBaseOptions['report'];
-    findings?: RetrievalLoopBaseOptions['findings'];
-    now?: RetrievalLoopBaseOptions['now'];
-}
-interface RunRetrievalImprovementLoopResult extends RunImprovementLoopResult<RetrievalEvalArtifact, RetrievalEvalScenario> {
-    baselineConfig: RetrievalConfig;
-    winnerConfig: RetrievalConfig;
-    trainScenarios: readonly RetrievalEvalScenario[];
-    holdoutScenarios: readonly RetrievalEvalScenario[];
-    candidates: readonly ParameterCandidate[];
-    targetRecall?: number;
+interface RetrievalScenarioPartitions {
+    trainScenarios: RetrievalEvalScenario[];
+    selectionScenarios: RetrievalEvalScenario[];
+    finalScenarios: RetrievalEvalScenario[];
 }
 declare function retrievalConfigSurface(config: RetrievalConfig): string;
 declare function retrievalConfigFromSurface(surface: MutableSurface): RetrievalConfig;
 declare function buildRetrievalEvalDispatch(options: BuildRetrievalEvalDispatchOptions): (surface: MutableSurface, scenario: RetrievalEvalScenario, context: DispatchContext) => Promise<RetrievalEvalArtifact>;
 declare function retrievalRecallJudge(options?: RetrievalRecallJudgeOptions): JudgeConfig<RetrievalEvalArtifact, RetrievalEvalScenario>;
 declare function scoreRetrievalArtifact(artifact: RetrievalEvalArtifact, scenario: RetrievalEvalScenario): RetrievalMetricSummary;
-declare function buildRetrievalParameterCandidates(searchSpace: RetrievalParameterSearchSpace, options?: BuildRetrievalParameterCandidatesOptions): ParameterCandidate[];
-declare function retrievalParameterSweepProposer(options: RetrievalParameterSweepProposerOptions): SurfaceProposer;
-declare function runRetrievalImprovementLoop(options: RunRetrievalImprovementLoopOptions): Promise<RunRetrievalImprovementLoopResult>;
+declare function partitionRetrievalScenarios(scenarios: readonly RetrievalEvalScenario[], options?: PartitionRetrievalScenariosOptions): RetrievalScenarioPartitions;
 
 type AgentMemoryKind = 'message' | 'entity' | 'fact' | 'preference' | 'observation' | 'reasoning-trace';
 interface AgentMemoryScope {
@@ -252,6 +209,11 @@ interface AgentMemoryAdapter {
     flush?(): Promise<void>;
     close?(): Promise<void>;
 }
+/**
+ * Optional session-level retrieval dropout for estimating whether delivered memories affect
+ * task outcomes. The feature is disabled unless configured, and consumers persist events
+ * through `onEvent`.
+ */
 interface RetrievalHoldoutConfig {
     /** Per-session probability that one eligible watchlist item is suppressed. 0 logs the full schema without ever dropping. */
     epsilon: number;
@@ -268,20 +230,20 @@ interface RetrievalHoldoutConfig {
      * sessionIdHash/scopeHash, so PII-bearing identifiers (tenantId/userId/tags) never reach a
      * consumer-controlled sink unless the consumer explicitly owns that decision. Note that
      * replaying assignment draws from logs alone needs the plaintext sessionId, so
-     * privacy-default logs require the consumer's own sessionId mapping for replay audits.
+     * privacy-preserving logs require the consumer's own sessionId mapping for replay.
      */
     includePlaintextIdentifiers?: boolean;
     /**
      * Cap on tracked sessions per experiment config in the sticky wrapper's registry.
-     * Exists so tests can exercise eviction; production should keep the default (10,000).
+     * The default is 10,000.
      */
     maxTrackedSessions?: number;
     /**
      * Uniform-[0,1) generator keyed by a string. Defaults to a sha256-derived deterministic
-     * generator so every assignment is replayable from the logged keys alone (design rule D5).
+     * generator so every assignment is replayable from the logged keys alone.
      */
     rng?: (key: string) => number;
-    /** Receives one event per retrieval call, INCLUDING no-drop calls: control-arm membership is half the data. */
+    /** Receives one event per retrieval call, including calls where nothing is dropped. */
     onEvent: (event: RetrievalHoldoutEvent) => void;
 }
 interface RetrievalHoldoutEligibleItem {
@@ -298,19 +260,18 @@ interface RetrievalHoldoutEvent {
     eventId: string;
     ts: string;
     adapterId?: string;
-    /** Plaintext session id — emitted ONLY when config.includePlaintextIdentifiers is true. */
+    /** Plaintext session id, emitted only when `includePlaintextIdentifiers` is true. */
     sessionId?: string;
     /** Consumer-supplied experiment/outcome join id (scope.tags.taskId); deliberately plaintext. */
     taskId?: string;
     /** 1-based call counter within the session; 0 when the call is outside session randomization. */
     callIndex: number;
     /**
-     * sha256(sessionId) prefix — the default privacy-preserving session join key AND the seed-key
-     * reference for the assignment draws (previously named rngKey; identical derivation, deduped).
+     * sha256(sessionId) prefix used as the privacy-preserving join key and assignment seed.
      */
     sessionIdHash?: string;
     queryHash?: string;
-    /** Verbatim scope — emitted ONLY when config.includePlaintextIdentifiers is true (PII risk). */
+    /** Verbatim scope, emitted only when `includePlaintextIdentifiers` is true. */
     scope?: AgentMemoryScope;
     /** sha256 prefix of the canonical-JSON scope (keys sorted, undefined stripped). */
     scopeHash?: string;
@@ -658,4 +619,4 @@ interface BuildRetrievalBenchmarkCasesFromQrelsOptions {
     splitOf?: (queryId: string) => KnowledgeBenchmarkSplit;
 }
 
-export { type RetrievalHoldoutConfig as $, type AgentMemoryAcquireRunLease as A, type BuildRetrievalBenchmarkCasesFromQrelsOptions as B, type KnowledgeBenchmarkSpec as C, type KnowledgeBenchmarkSplit as D, type KnowledgeBenchmarkTaskKind as E, type KnowledgeClaimMatcher as F, type KnowledgeMemoryBenchmarkCase as G, type KnowledgeMemoryBenchmarkTaskKind as H, type KnowledgeMemoryEvent as I, type KnowledgeMemoryFactMatcher as J, type KnowledgeAnswerBenchmarkCase as K, type KnowledgeRetrievalBenchmarkCase as L, type KnowledgeRetrievalBenchmarkQrel as M, type KnowledgeRetrievalBenchmarkQuery as N, type MemoryAdapterBenchmarkCandidate as O, type MemoryAdapterBenchmarkRankingRow as P, type OwnedAgentMemoryRunLease as Q, type RunRetrievalImprovementLoopResult as R, type RetrievalConfig as S, type RetrievalEvalArtifact as T, type RetrievalEvalRetriever as U, type RetrievalEvalRetrieverInput as V, type RetrievalEvalRetrieverResult as W, type RetrievalEvalScenario as X, type RetrievalGoldTarget as Y, type RetrievalHoldoutBypassReason as Z, type RetrievalHoldoutCallContext as _, type RunRetrievalImprovementLoopOptions as a, type RetrievalHoldoutEligibleItem as a0, type RetrievalHoldoutEvent as a1, type RetrievalHoldoutResult as a2, type RetrievalHoldoutSessionState as a3, type RetrievalMetricSummary as a4, type RetrievalMetricWeights as a5, type RetrievalParameterSearchSpace as a6, type RetrievalParameterSweepProposerOptions as a7, type RetrievalRecallJudgeOptions as a8, type RetrievedKnowledgeHit as a9, type RetrievedSourceSpan as aa, type RunKnowledgeBenchmarkSuiteOptions as ab, type RunKnowledgeBenchmarkSuiteResult as ac, type RunMemoryAdapterBenchmarkOptions as ad, type RunMemoryAdapterBenchmarkResult as ae, acquireAgentMemoryRunLease as af, buildRetrievalEvalDispatch as ag, buildRetrievalParameterCandidates as ah, retrievalConfigFromSurface as ai, retrievalConfigSurface as aj, retrievalParameterSweepProposer as ak, retrievalRecallJudge as al, runRetrievalImprovementLoop as am, scoreRetrievalArtifact as an, type AgentMemoryAdapter as b, type AgentMemoryBranchIsolation as c, type AgentMemoryContext as d, type AgentMemoryControllerMode as e, type AgentMemoryHit as f, type AgentMemoryKind as g, type AgentMemoryRunLease as h, type AgentMemoryScope as i, type AgentMemorySearchOptions as j, type AgentMemoryWriteInput as k, type AgentMemoryWriteResult as l, type BuildRetrievalEvalDispatchOptions as m, type BuildRetrievalParameterCandidatesOptions as n, type KnowledgeAnswerBenchmarkTaskKind as o, type KnowledgeBenchmarkArtifact as p, type KnowledgeBenchmarkCase as q, type KnowledgeBenchmarkCaseBase as r, type KnowledgeBenchmarkDistribution as s, type KnowledgeBenchmarkEvaluation as t, type KnowledgeBenchmarkFamily as u, type KnowledgeBenchmarkReport as v, type KnowledgeBenchmarkResponder as w, type KnowledgeBenchmarkScenario as x, type KnowledgeBenchmarkSliceSummary as y, type KnowledgeBenchmarkSource as z };
+export { type RetrievalHoldoutConfig as $, type AgentMemoryAcquireRunLease as A, type BuildRetrievalBenchmarkCasesFromQrelsOptions as B, type KnowledgeBenchmarkScenario as C, type KnowledgeBenchmarkSliceSummary as D, type KnowledgeBenchmarkSource as E, type KnowledgeBenchmarkSpec as F, type KnowledgeBenchmarkSplit as G, type KnowledgeBenchmarkTaskKind as H, type KnowledgeClaimMatcher as I, type KnowledgeMemoryBenchmarkCase as J, type KnowledgeAnswerBenchmarkCase as K, type KnowledgeMemoryBenchmarkTaskKind as L, type KnowledgeMemoryEvent as M, type KnowledgeMemoryFactMatcher as N, type KnowledgeRetrievalBenchmarkCase as O, type KnowledgeRetrievalBenchmarkQrel as P, type KnowledgeRetrievalBenchmarkQuery as Q, type RetrievalConfig as R, type MemoryAdapterBenchmarkCandidate as S, type MemoryAdapterBenchmarkRankingRow as T, type OwnedAgentMemoryRunLease as U, type PartitionRetrievalScenariosOptions as V, type RetrievalEvalRetrieverInput as W, type RetrievalEvalRetrieverResult as X, type RetrievalGoldTarget as Y, type RetrievalHoldoutBypassReason as Z, type RetrievalHoldoutCallContext as _, type RetrievalEvalScenario as a, type RetrievalHoldoutEligibleItem as a0, type RetrievalHoldoutEvent as a1, type RetrievalHoldoutResult as a2, type RetrievalHoldoutSessionState as a3, type RetrievalMetricSummary as a4, type RetrievalRecallJudgeOptions as a5, type RetrievalScenarioPartitions as a6, type RetrievedSourceSpan as a7, type RunKnowledgeBenchmarkSuiteOptions as a8, type RunKnowledgeBenchmarkSuiteResult as a9, type RunMemoryAdapterBenchmarkOptions as aa, type RunMemoryAdapterBenchmarkResult as ab, acquireAgentMemoryRunLease as ac, buildRetrievalEvalDispatch as ad, partitionRetrievalScenarios as ae, retrievalConfigFromSurface as af, retrievalConfigSurface as ag, retrievalRecallJudge as ah, scoreRetrievalArtifact as ai, type RetrievalEvalArtifact as b, type RetrievalEvalRetriever as c, type RetrievalMetricWeights as d, type RetrievedKnowledgeHit as e, type AgentMemoryAdapter as f, type AgentMemoryBranchIsolation as g, type AgentMemoryContext as h, type AgentMemoryControllerMode as i, type AgentMemoryHit as j, type AgentMemoryKind as k, type AgentMemoryRunLease as l, type AgentMemoryScope as m, type AgentMemorySearchOptions as n, type AgentMemoryWriteInput as o, type AgentMemoryWriteResult as p, type BuildRetrievalEvalDispatchOptions as q, type KnowledgeAnswerBenchmarkTaskKind as r, type KnowledgeBenchmarkArtifact as s, type KnowledgeBenchmarkCase as t, type KnowledgeBenchmarkCaseBase as u, type KnowledgeBenchmarkDistribution as v, type KnowledgeBenchmarkEvaluation as w, type KnowledgeBenchmarkFamily as x, type KnowledgeBenchmarkReport as y, type KnowledgeBenchmarkResponder as z };

package/docs/eval/investment-material-facts.md

@@ -1,10 +1,10 @@
-# Held-out investment-research eval set — material facts + provenance
+# Held-out investment-research eval set: material facts + provenance
 
 This is the answer key and the provenance ledger for `tests/eval/investment-thesis-set.ts`.
 
 **What the set measures.** Give a research loop a company + ticker + an as-of
 **cutoff** date and ask it to write an investment thesis. Then grade that thesis
-against the held-out **material facts** below — facts the loop never saw. A high
+against the held-out **material facts** below, facts the loop never saw. A high
 score means the thesis surfaced the buried, material, non-obvious drivers a
 thorough analyst would flag and a single ticker search would miss; it is **not**
 teaching-to-the-test, because the answer key is firewalled from every loop and
@@ -20,14 +20,13 @@ the grader is a `$0`, model-free substring check (`gradeFactAgainstText`).
    guessed (see the drop log).
 3. **Knowable at the cutoff.** Every value was disclosed in, or computable from, a
    filing available on or before the cutoff. The eventual collapse is **not** a
-   checklist item — it is recorded as `knownOutcome`, for the reader only, and is
-   never graded.
+   checklist item. It is recorded as `knownOutcome` for the reader and is never graded.
 
 All five primary documents were fetched live from `https://www.sec.gov/Archives/`
 during curation (a `curl` with a descriptive `User-Agent`, per SEC fair-access
 rules). Every dollar figure below was read directly out of the de-tagged filing
-text. Provenance is verifiable: each `sourceUrl` contains the company's SEC CIK,
-and `tests/eval/investment-thesis-set.test.ts` asserts that invariant.
+text. Each `sourceUrl` contains the company's SEC CIK, and
+`tests/eval/investment-thesis-set.test.ts` checks that invariant.
 
 ---
 
@@ -47,7 +46,7 @@ set was curated (June 2026); `investment-thesis-set.test.ts` asserts this.
 
 ---
 
-## SIVB — SVB Financial Group (cutoff 2023-02-24)
+## SIVB: SVB Financial Group (cutoff 2023-02-24)
 
 Source: [FY2022 10-K (`sivb-20221231.htm`)](https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm)
 **Known outcome (not graded):** FDIC receivership March 10, 2023; holding-company Chapter 11 March 17, 2023.
@@ -59,14 +58,14 @@ Source: [FY2022 10-K (`sivb-20221231.htm`)](https://www.sec.gov/Archives/edgar/d
 | SIVB/f3 | concentration | Run-prone uninsured deposit base | "estimated uninsured deposits in U.S. offices that exceed the FDIC insurance limit were **$151.5 billion**" |
 | SIVB/f4 | margin-trend | Cheap deposits fleeing → funding cost set to rise | "Noninterest-bearing demand deposits to total deposits decreased by **20 percentage points to 47 percent**" |
 | SIVB/f5 | concentration | Single-client-type (innovation-economy) deposit + credit base | 10-K frames the franchise around "the innovation economy" (technology, life-science, venture) |
-| SIVB/f6 | off-balance-sheet | AFS loss in AOCI — the visible, smaller tip | "Available-for-sale securities, at fair value (cost of $ **28,602**) **26,069**" → ~$2.5B AFS loss in AOCI |
+| SIVB/f6 | off-balance-sheet | AFS loss in AOCI; the visible, smaller tip | "Available-for-sale securities, at fair value (cost of $ **28,602**) **26,069**" → ~$2.5B AFS loss in AOCI |
 
 The decisive, non-obvious fact is SIVB/f1+f2: an interest-rate loss roughly equal
 to all of equity, sitting in the footnotes because HTM accounting keeps it out of
 both earnings and book equity. A ticker search shows a profitable bank; the
 filing shows a mark-to-market hole the size of its capital.
 
-## BBBY — Bed Bath & Beyond Inc. (cutoff 2022-04-21)
+## BBBY: Bed Bath & Beyond Inc. (cutoff 2022-04-21)
 
 Source: [FY2021 10-K (`bbby-20220226.htm`)](https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm)
 **Known outcome (not graded):** Chapter 11 on April 23, 2023; equity wiped out.
@@ -80,11 +79,11 @@ Source: [FY2021 10-K (`bbby-20220226.htm`)](https://www.sec.gov/Archives/edgar/d
 | BBBY/f5 | margin-trend | Inventory building into a demand decline | "Merchandise inventories **1,725,410** 1,671,909" ($ thousands) |
 
 The non-obvious fact is BBBY/f1+f2+f4 together: in FY2021 the company **lost $560M,
-generated only $18M of operating cash, and still spent $575M buying back stock** —
+generated only $18M of operating cash, and still spent $575M buying back stock**,
 returning more cash than it had. The buyback, not the income statement alone, is
 why a $1.3B equity base became $174M.
 
-## CVNA — Carvana Co. (cutoff 2023-02-23)
+## CVNA: Carvana Co. (cutoff 2023-02-23)
 
 Source: [FY2022 10-K (`cvna-20221231.htm`)](https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm)
 **Known outcome (not graded):** Stock fell ~98% from its 2021 peak; a 2023 debt exchange cut and extended obligations, narrowly avoiding bankruptcy.
@@ -97,18 +96,18 @@ Source: [FY2022 10-K (`cvna-20221231.htm`)](https://www.sec.gov/Archives/edgar/d
 | CVNA/f4 | governance | Recurring related-party leases with the founder's family | Related-Party note: DriveTime, controlled by "Ernest Garcia II, Ernest Garcia III, and entities controlled by one or both of them" |
 | CVNA/f5 | liquidity | A wide loss showing unit economics had not turned | "Net loss $ (**2,894**)" ($ millions) |
 
-The non-obvious facts are CVNA/f2 (interest expense up 2.8x — the debt was now
-expensive, not just large) and CVNA/f4 (the controlling Garcia family on both
+The non-obvious facts are CVNA/f2 (interest expense up 2.8x, making the debt expensive
+as well as large) and CVNA/f4 (the controlling Garcia family on both
 sides of material leases via DriveTime), neither of which a ticker quote shows.
 
-## PTON — Peloton Interactive, Inc. (cutoff 2022-09-07)
+## PTON: Peloton Interactive, Inc. (cutoff 2022-09-07)
 
 Source: [FY2022 10-K (`pton-20220630.htm`)](https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm)
 **Known outcome (not graded):** Stock fell ~95% from its 2021 peak; founder-CEO departed; mass layoffs and a multi-year turnaround.
 
 | ID | Lens | Material fact | Value read from the filing |
 |---|---|---|---|
-| PTON/f1 | margin-trend | Hardware gross margin turned **negative** | Connected Fitness "Gross Margin decreased to (**11**)" percent — losing money per unit sold |
+| PTON/f1 | margin-trend | Hardware gross margin turned **negative** | Connected Fitness "Gross Margin decreased to (**11**)" percent; losing money per unit sold |
 | PTON/f2 | liquidity | Inventory glut as pandemic demand normalized | "Inventories, net **1,104.5** 937" ($ millions) |
 | PTON/f3 | governance | Dual-class super-voting control | "Class B common stock has **20 votes per share** and our Class A common stock has one vote per share" |
 | PTON/f4 | liquidity | An order-of-magnitude wider loss | "Net loss $ (**2,827**)" ($ millions) |
@@ -116,11 +115,11 @@ Source: [FY2022 10-K (`pton-20220630.htm`)](https://www.sec.gov/Archives/edgar/d
 | PTON/f6 | leverage | Locked-in purchase commitments into falling demand | "purchase commitments related to the manufacture of Peloton products were estimated to be approximately **$334**" million |
 
 The non-obvious fact is PTON/f1: revenue was still large, but the **hardware was
-sold below cost** (−11% gross margin) — the unit economics, not just the growth
-rate, had broken. PTON/f6 compounds it: the company was contractually obliged to
+sold below cost** (−11% gross margin). The unit economics had broken despite the
+top-line growth. PTON/f6 compounds it: the company was contractually obliged to
 buy more inventory it could not sell.
 
-## SI — Silvergate Capital Corporation (cutoff 2022-02-28)
+## SI: Silvergate Capital Corporation (cutoff 2022-02-28)
 
 Source: [FY2021 10-K (`si-20211231.htm`)](https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm)
 **Known outcome (not graded):** After the FTX collapse triggered a deposit run, Silvergate announced a voluntary wind-down and liquidation of Silvergate Bank in March 2023.
@@ -134,7 +133,7 @@ Source: [FY2021 10-K (`si-20211231.htm`)](https://www.sec.gov/Archives/edgar/dat
 | SI/f5 | concentration | The moat AND the funding are the same crypto-only bet | "Silvergate Exchange Network ('SEN'), our proprietary ... payment network for participants in the digital currency industry" |
 
 The non-obvious fact is SI/f1+f2 together: **99.5% noninterest-bearing deposits,
-~58% from crypto exchanges** — a funding base with no contractual term and a
+~58% from crypto exchanges**, a funding base with no contractual term and a
 single correlated counterparty type. A ticker quote shows a fast-growing,
 low-cost-funding bank; the filing shows a bank that could be emptied in days if
 crypto sentiment turned.
@@ -176,8 +175,8 @@ should not be read as a general "investment-research quality" number. The biases
 
 3. **Sector skew toward financials + distressed consumer.** Two of five are banks
    (SIVB, SI). The two banks are deliberately given **different** buried-risk
-   lenses — SIVB is an interest-rate / duration / off-balance-sheet story, SI is a
-   single-industry deposit-concentration story — so they are not redundant, but
+   lenses, SIVB is an interest-rate / duration / off-balance-sheet story, SI is a
+   single-industry deposit-concentration story, so they are not redundant, but
    the set still over-indexes on balance-sheet fragility and under-tests, e.g.,
    technology platform risk, supply-chain concentration, or accounting-policy
    aggressiveness in a healthy grower.
@@ -197,33 +196,33 @@ should not be read as a general "investment-research quality" number. The biases
    required a second independent source to establish was dropped (below) rather
    than sourced to the filing alone.
 
-## Drop log — items considered and NOT included (honesty over coverage)
+## Drop log: excluded items
 
 These were candidate facts I could not independently ground to a primary source
 available at the cutoff, so I **dropped them rather than guess**:
 
-- **First Republic Bank (FRC)** — dropped as a company entirely. First Republic
+- **First Republic Bank (FRC)**: dropped as a company entirely. First Republic
   was a state-chartered bank that filed its annual reports with the **FDIC**, not
   on SEC EDGAR, so its 10-K is not at a `sec.gov/Archives` URL and I could not give
-  it the same clean, CIK-verifiable provenance as the other five. Its widely cited
+  it the same CIK-matched provenance as the other five. Its widely cited
   figures (~$15B HTM-style loss, ~$119.5B uninsured deposits) are real but are best
-  sourced from the FDIC OIG Material Loss Review, a **post**-cutoff document — using
+  sourced from the FDIC OIG Material Loss Review, a **post**-cutoff document, using
   it would violate rule 3. Replaced with Silvergate, whose 10-K is on EDGAR.
 
-- **SIVB exact total unrealized-loss footnote line** — I report the HTM gap as the
+- **SIVB exact total unrealized-loss footnote line**: I report the HTM gap as the
   arithmetic difference of two figures printed on the balance sheet
   ($91,321M − $76,169M), which is exact and on-cutoff. I did **not** include a
   separately-quoted "$15.1B net unrealized loss" sentence because I did not locate
   that exact phrasing in the de-tagged text; quoting a number I could not point to
   verbatim would break rule 2. The computed value is conservative and checkable.
 
-- **CVNA negative gross-profit-per-unit** — a frequently-cited Carvana red flag,
+- **CVNA negative gross-profit-per-unit**: a frequently-cited Carvana red flag,
   but the per-unit figure I could find cleanly was a derived/analyst number, not a
   single line item I could quote verbatim from the 10-K at the cutoff. Dropped in
   favor of the directly-quoted total debt, interest expense, ADESA price, related
   party, and net loss.
 
-- **PTON / SI specific debt-covenant or going-concern language** — I searched for
+- **PTON / SI specific debt-covenant or going-concern language**: I searched for
   explicit "substantial doubt / going concern" wording in both filings and did
   **not** find it at these cutoffs (it came later). I did not invent it. The facts
   included are the ones actually present in the cutoff-date document.
@@ -248,5 +247,5 @@ for (const company of investmentThesisSet) {
 ```
 
 The grader is deterministic and model-free, so the same thesis always scores the
-same and the answer key never reaches a model the loop could observe — the same
+same and the answer key never reaches a model the loop could observe, the same
 firewall the deep-question exam (`tests/loops/held-out-exam.ts`) uses.

package/docs/eval/rag-eval-roadmap.md

@@ -1,6 +1,6 @@
 # RAG Eval Completion Roadmap
 
-Verdict: `runRetrievalImprovementLoop()` is the right first loop, but it is only the retrieval layer.
+Verdict: use retrieval-only optimization when the retriever is the only changing component, and full RAG optimization when retrieval and answer behavior must move together.
 SOTA RAG evaluation requires retrieval quality, context quality, generated-answer quality, abstention behavior, robustness, and operating budgets.
 
 ## Research Basis
@@ -12,30 +12,34 @@ SOTA RAG evaluation requires retrieval quality, context quality, generated-answe
 | [TruLens RAG Triad](https://www.trulens.org/getting_started/core_concepts/rag_triad/) | The minimal end-to-end triad is context relevance, groundedness, and answer relevance. |
 | [RAGChecker](https://papers.nips.cc/paper_files/paper/2024/hash/27245589131d17368cccdfa990cbf16e-Abstract-Datasets_and_Benchmarks_Track.html) | Fine-grained diagnosis should separate retrieval misses, noisy context, and unsupported generated claims. |
 | [BEIR](https://arxiv.org/abs/2104.08663) / TREC-style retrieval | Retrieval still needs classical rank metrics: Recall@k, Precision@k, MRR, MAP, and nDCG. |
-| [CRAG](https://arxiv.org/abs/2406.04744) | Real RAG evals must include long-tail, dynamic, multi-hop, and unanswerable questions, not only easy static facts. |
+| [CRAG](https://arxiv.org/abs/2406.04744) | Real RAG evals must include long-tail, dynamic, multi-hop, and unanswerable questions alongside easy static facts. |
 | [DeepEval RAG metrics](https://deepeval.com/docs/metrics-faithfulness) | Production tools converge on faithfulness, answer relevance, and context relevance as generator/retriever checks. |
 
 ## Current Repo Status
 
 Done:
 
-- `runRetrievalImprovementLoop()` auto-searches retrieval configs through `agent-eval`.
+- `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs.
+- `runRagOptimization()` does the same for a serialized retrieval and answer configuration.
 - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases:
   retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion.
 - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans.
-- The retrieval judge reports recall, MRR, nDCG, precision@k, cost, and held-out promotion.
-- The loop is tested with a real `agent-eval` run where `{ k: 2 }` beats `{ k: 1 }`.
+- The retrieval judge reports recall, MRR, nDCG, and precision@k; `agent-eval` reports cost separately.
+- Selection and final data remain independent, and the optimization method never receives final cases.
+- Every reusable optimization requires an immutable execution reference covering retrieval, index, model, judge, and external-service behavior.
+- The integration is tested with complete methods for retrieval and full RAG configuration.
 - The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`.
 - `ragAnswerQualityJudge()` and `createRagAnswerQualityHook()` score context precision/recall/relevance/sufficiency, faithfulness, answer relevance/correctness, citation support, abstention, and unsupported-answer rate.
 - `normalizeExternalRagScores()` and the row exporters make Ragas, DeepEval, TruLens, RAGChecker, and custom evaluator outputs pluggable instead of hard dependencies.
 - `scoreKnowledgeBaseIndex()` validates generic wiki/KB health: citation coverage, source-backed pages, stale sources, duplicate source hashes, and lint/validation errors.
 - `calibrateRagAnswerJudge()` enforces the strong-vs-weak metric check before trusting a RAG answer metric.
+- `runKnowledgeImprovementJob()` in `@tangle-network/agent-runtime` connects runtime backends and worker factories to candidate KB creation, readiness checks, frozen comparisons, spend measurement, and explicit activation.
+- `agent-knowledge` remains runtime-free; applications with their own agent runner pass an `updateKnowledge` callback directly.
 
 Not done:
 
 - Slice-level reporting for freshness, distractors, multi-hop, and long-tail cases.
-- Packaged runtime adapters for browser/coding/research agents.
-  The lifecycle API accepts those agents as hooks today; it does not hardcode provider-specific workers.
+- A maintained, public benchmark pack with at least 100 labeled scenarios and published baseline results.
 
 ## Completion Criteria
 
@@ -55,9 +59,9 @@ Required slices:
 
 Ship criteria:
 
-- Holdout source-span Recall@5 is at least 0.90.
-- Holdout nDCG@5 is at least 0.80.
-- Train-to-holdout recall gap is at most 0.08.
+- Final source-span Recall@5 is at least 0.90.
+- Final nDCG@5 is at least 0.80.
+- Train-to-final recall gap is at most 0.08.
 - Stale or forbidden source hit rate is at most 0.02.
 - p95 retrieval latency and cost do not regress by more than 10 percent versus baseline.
 
@@ -68,9 +72,9 @@ Score it with deterministic checks first and LLM judges only for semantic qualit
 
 Ship criteria:
 
-- Faithfulness or groundedness is at least 0.95 on holdout.
-- Answer relevance is at least 0.90 on holdout.
-- Answer correctness is at least 0.85 on human-labeled holdout.
+- Faithfulness or groundedness is at least 0.95 on final cases.
+- Answer relevance is at least 0.90 on final cases.
+- Answer correctness is at least 0.85 on human-labeled final cases.
 - Citation support is at least 0.95 for claims that cite sources.
 - Unsupported-answer rate on unanswerable questions is at most 0.05.
 
@@ -95,13 +99,13 @@ Ship criteria:
 
 - Every failed eval has one primary failure class.
 - At least 95 percent of generated claims can be mapped to supporting context, contradicted context, or no context.
-- Reports show metrics by slice and by failure class, not only the aggregate score.
+- Reports show metrics by slice and failure class as well as the aggregate score.
 
 ### Phase 4: Production Loop
 
 Run the same eval pack on every retrieval or prompt change.
-Keep train/dev/holdout isolated.
-Never tune on holdout.
+Keep train, selection, and final data isolated.
+Never tune on final data.
 
 Ship criteria:
 
@@ -115,5 +119,5 @@ Ship criteria:
 
 1. Add slice-level aggregation helpers for freshness, distractors, multi-hop, long-tail, and unanswerable cases.
 2. Add forbidden/stale source targets to retrieval scenarios.
-3. Add packaged adapters that turn `agent-runtime` browser, research, and coding agents into `runRagKnowledgeImprovementLoop()` hooks.
+3. Add a maintained benchmark pack with at least 100 labeled scenarios and a reproducible baseline report.
 4. Add a CLI command that runs the lifecycle loop and writes a reproducible report under `.agent-knowledge/eval/`.