@tangle-network/agent-knowledge 4.1.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,7 +24,7 @@ import {
24
24
  runAgentMemoryExperiment,
25
25
  runAgentMemoryImprovement,
26
26
  toOffPolicyTrajectory
27
- } from "../chunk-CPMLJYA3.js";
27
+ } from "../chunk-LMR53POQ.js";
28
28
  import {
29
29
  AgentMemoryLifecycleTimeoutError,
30
30
  AgentMemoryLifecycleUnsafeError,
@@ -37,7 +37,7 @@ import {
37
37
  resolveMemoryCleanupTimeoutMs,
38
38
  runBoundedMemoryLifecycle,
39
39
  sleepForMemoryRecovery
40
- } from "../chunk-BBCIB4UL.js";
40
+ } from "../chunk-EYIA5PLQ.js";
41
41
  import "../chunk-DQ3PDMDP.js";
42
42
  import "../chunk-YMKHCTS2.js";
43
43
  export {
@@ -1,8 +1,8 @@
1
- import { JsonValue, Scenario, DispatchContext, RunImprovementLoopResult, ParameterCandidate, JudgeConfig, Gate, RunImprovementLoopOptions, MutableSurface, SurfaceProposer, CampaignStorage, CostLedgerHandle, CampaignResult } from '@tangle-network/agent-eval/campaign';
1
+ import { Scenario, DispatchContext, MutableSurface, JudgeConfig, CampaignStorage, CostLedgerHandle, CampaignResult } from '@tangle-network/agent-eval/campaign';
2
2
  import { c as KnowledgeIndex, S as SourceRecord } from './types-6x0OpfW6.js';
3
+ import { AgentCandidateJsonValue } from '@tangle-network/agent-interface';
3
4
 
4
- type RetrievalConfig = Record<string, JsonValue>;
5
- type RetrievalParameterSearchSpace = Record<string, readonly JsonValue[]>;
5
+ type RetrievalConfig = Record<string, AgentCandidateJsonValue>;
6
6
  type RetrievalGoldTarget = {
7
7
  kind: 'page';
8
8
  pageId: string;
@@ -44,7 +44,7 @@ interface RetrievedKnowledgeHit {
44
44
  sourceIds?: readonly string[];
45
45
  sourceSpans?: readonly RetrievedSourceSpan[];
46
46
  snippet?: string;
47
- metadata?: Record<string, JsonValue>;
47
+ metadata?: Record<string, AgentCandidateJsonValue>;
48
48
  }
49
49
  interface RetrievalEvalArtifact {
50
50
  config: RetrievalConfig;
@@ -54,7 +54,7 @@ interface RetrievalEvalArtifact {
54
54
  durationMs: number;
55
55
  /** Informational copy. Billable retrievers account through context.cost.runPaidCall. */
56
56
  costUsd?: number;
57
- metadata?: Record<string, JsonValue>;
57
+ metadata?: Record<string, AgentCandidateJsonValue>;
58
58
  }
59
59
  interface RetrievalMetricSummary {
60
60
  recall: number;
@@ -79,7 +79,7 @@ interface RetrievalEvalRetrieverResult {
79
79
  hits: readonly RetrievedKnowledgeHit[];
80
80
  /** Informational copy. Billable retrievers account through context.cost.runPaidCall. */
81
81
  costUsd?: number;
82
- metadata?: Record<string, JsonValue>;
82
+ metadata?: Record<string, AgentCandidateJsonValue>;
83
83
  }
84
84
  type RetrievalEvalRetriever = (input: RetrievalEvalRetrieverInput) => Promise<readonly RetrievedKnowledgeHit[] | RetrievalEvalRetrieverResult>;
85
85
  interface BuildRetrievalEvalDispatchOptions {
@@ -97,65 +97,22 @@ interface RetrievalRecallJudgeOptions {
97
97
  name?: string;
98
98
  weights?: RetrievalMetricWeights;
99
99
  }
100
- interface BuildRetrievalParameterCandidatesOptions {
101
- baseline?: RetrievalConfig;
102
- }
103
- interface RetrievalParameterSweepProposerOptions {
104
- candidates?: readonly ParameterCandidate[];
105
- searchSpace?: RetrievalParameterSearchSpace;
106
- baseline?: RetrievalConfig;
100
+ interface PartitionRetrievalScenariosOptions {
101
+ selectionFraction?: number;
102
+ finalFraction?: number;
103
+ seed?: number;
107
104
  }
108
- type RetrievalLoopBaseOptions = RunImprovementLoopOptions<RetrievalEvalScenario, RetrievalEvalArtifact>;
109
- interface RunRetrievalImprovementLoopOptions {
110
- baseline: RetrievalConfig;
111
- scenarios: readonly RetrievalEvalScenario[];
112
- holdoutScenarios?: readonly RetrievalEvalScenario[];
113
- index?: KnowledgeIndex;
114
- defaultK?: number;
115
- retrieve?: RetrievalEvalRetriever;
116
- candidates?: readonly ParameterCandidate[];
117
- searchSpace?: RetrievalParameterSearchSpace;
118
- judges?: readonly JudgeConfig<RetrievalEvalArtifact, RetrievalEvalScenario>[];
119
- gate?: Gate<RetrievalEvalArtifact, RetrievalEvalScenario>;
120
- metricWeights?: RetrievalMetricWeights;
121
- targetRecall?: number;
122
- holdoutFraction?: number;
123
- splitSeed?: number;
124
- deltaThreshold?: number;
125
- runDir?: RetrievalLoopBaseOptions['runDir'];
126
- seed?: RetrievalLoopBaseOptions['seed'];
127
- reps?: RetrievalLoopBaseOptions['reps'];
128
- resumable?: RetrievalLoopBaseOptions['resumable'];
129
- costCeiling?: RetrievalLoopBaseOptions['costCeiling'];
130
- maxConcurrency?: RetrievalLoopBaseOptions['maxConcurrency'];
131
- dispatchTimeoutMs?: RetrievalLoopBaseOptions['dispatchTimeoutMs'];
132
- expectUsage?: RetrievalLoopBaseOptions['expectUsage'];
133
- tracing?: RetrievalLoopBaseOptions['tracing'];
134
- storage?: RetrievalLoopBaseOptions['storage'];
135
- populationSize?: RetrievalLoopBaseOptions['populationSize'];
136
- maxGenerations?: RetrievalLoopBaseOptions['maxGenerations'];
137
- promoteTopK?: RetrievalLoopBaseOptions['promoteTopK'];
138
- maxImprovementShots?: RetrievalLoopBaseOptions['maxImprovementShots'];
139
- report?: RetrievalLoopBaseOptions['report'];
140
- findings?: RetrievalLoopBaseOptions['findings'];
141
- now?: RetrievalLoopBaseOptions['now'];
142
- }
143
- interface RunRetrievalImprovementLoopResult extends RunImprovementLoopResult<RetrievalEvalArtifact, RetrievalEvalScenario> {
144
- baselineConfig: RetrievalConfig;
145
- winnerConfig: RetrievalConfig;
146
- trainScenarios: readonly RetrievalEvalScenario[];
147
- holdoutScenarios: readonly RetrievalEvalScenario[];
148
- candidates: readonly ParameterCandidate[];
149
- targetRecall?: number;
105
+ interface RetrievalScenarioPartitions {
106
+ trainScenarios: RetrievalEvalScenario[];
107
+ selectionScenarios: RetrievalEvalScenario[];
108
+ finalScenarios: RetrievalEvalScenario[];
150
109
  }
151
110
  declare function retrievalConfigSurface(config: RetrievalConfig): string;
152
111
  declare function retrievalConfigFromSurface(surface: MutableSurface): RetrievalConfig;
153
112
  declare function buildRetrievalEvalDispatch(options: BuildRetrievalEvalDispatchOptions): (surface: MutableSurface, scenario: RetrievalEvalScenario, context: DispatchContext) => Promise<RetrievalEvalArtifact>;
154
113
  declare function retrievalRecallJudge(options?: RetrievalRecallJudgeOptions): JudgeConfig<RetrievalEvalArtifact, RetrievalEvalScenario>;
155
114
  declare function scoreRetrievalArtifact(artifact: RetrievalEvalArtifact, scenario: RetrievalEvalScenario): RetrievalMetricSummary;
156
- declare function buildRetrievalParameterCandidates(searchSpace: RetrievalParameterSearchSpace, options?: BuildRetrievalParameterCandidatesOptions): ParameterCandidate[];
157
- declare function retrievalParameterSweepProposer(options: RetrievalParameterSweepProposerOptions): SurfaceProposer;
158
- declare function runRetrievalImprovementLoop(options: RunRetrievalImprovementLoopOptions): Promise<RunRetrievalImprovementLoopResult>;
115
+ declare function partitionRetrievalScenarios(scenarios: readonly RetrievalEvalScenario[], options?: PartitionRetrievalScenariosOptions): RetrievalScenarioPartitions;
159
116
 
160
117
  type AgentMemoryKind = 'message' | 'entity' | 'fact' | 'preference' | 'observation' | 'reasoning-trace';
161
118
  interface AgentMemoryScope {
@@ -662,4 +619,4 @@ interface BuildRetrievalBenchmarkCasesFromQrelsOptions {
662
619
  splitOf?: (queryId: string) => KnowledgeBenchmarkSplit;
663
620
  }
664
621
 
665
- export { type RetrievalHoldoutConfig as $, type AgentMemoryAcquireRunLease as A, type BuildRetrievalBenchmarkCasesFromQrelsOptions as B, type KnowledgeBenchmarkSpec as C, type KnowledgeBenchmarkSplit as D, type KnowledgeBenchmarkTaskKind as E, type KnowledgeClaimMatcher as F, type KnowledgeMemoryBenchmarkCase as G, type KnowledgeMemoryBenchmarkTaskKind as H, type KnowledgeMemoryEvent as I, type KnowledgeMemoryFactMatcher as J, type KnowledgeAnswerBenchmarkCase as K, type KnowledgeRetrievalBenchmarkCase as L, type KnowledgeRetrievalBenchmarkQrel as M, type KnowledgeRetrievalBenchmarkQuery as N, type MemoryAdapterBenchmarkCandidate as O, type MemoryAdapterBenchmarkRankingRow as P, type OwnedAgentMemoryRunLease as Q, type RunRetrievalImprovementLoopResult as R, type RetrievalConfig as S, type RetrievalEvalArtifact as T, type RetrievalEvalRetriever as U, type RetrievalEvalRetrieverInput as V, type RetrievalEvalRetrieverResult as W, type RetrievalEvalScenario as X, type RetrievalGoldTarget as Y, type RetrievalHoldoutBypassReason as Z, type RetrievalHoldoutCallContext as _, type RunRetrievalImprovementLoopOptions as a, type RetrievalHoldoutEligibleItem as a0, type RetrievalHoldoutEvent as a1, type RetrievalHoldoutResult as a2, type RetrievalHoldoutSessionState as a3, type RetrievalMetricSummary as a4, type RetrievalMetricWeights as a5, type RetrievalParameterSearchSpace as a6, type RetrievalParameterSweepProposerOptions as a7, type RetrievalRecallJudgeOptions as a8, type RetrievedKnowledgeHit as a9, type RetrievedSourceSpan as aa, type RunKnowledgeBenchmarkSuiteOptions as ab, type RunKnowledgeBenchmarkSuiteResult as ac, type RunMemoryAdapterBenchmarkOptions as ad, type RunMemoryAdapterBenchmarkResult as ae, acquireAgentMemoryRunLease as af, buildRetrievalEvalDispatch as ag, buildRetrievalParameterCandidates as ah, retrievalConfigFromSurface as ai, retrievalConfigSurface as aj, retrievalParameterSweepProposer as ak, retrievalRecallJudge as al, runRetrievalImprovementLoop as am, scoreRetrievalArtifact as an, type AgentMemoryAdapter as b, type AgentMemoryBranchIsolation as c, type AgentMemoryContext as d, type AgentMemoryControllerMode as e, type AgentMemoryHit as f, type AgentMemoryKind as g, type AgentMemoryRunLease as h, type AgentMemoryScope as i, type AgentMemorySearchOptions as j, type AgentMemoryWriteInput as k, type AgentMemoryWriteResult as l, type BuildRetrievalEvalDispatchOptions as m, type BuildRetrievalParameterCandidatesOptions as n, type KnowledgeAnswerBenchmarkTaskKind as o, type KnowledgeBenchmarkArtifact as p, type KnowledgeBenchmarkCase as q, type KnowledgeBenchmarkCaseBase as r, type KnowledgeBenchmarkDistribution as s, type KnowledgeBenchmarkEvaluation as t, type KnowledgeBenchmarkFamily as u, type KnowledgeBenchmarkReport as v, type KnowledgeBenchmarkResponder as w, type KnowledgeBenchmarkScenario as x, type KnowledgeBenchmarkSliceSummary as y, type KnowledgeBenchmarkSource as z };
622
+ export { type RetrievalHoldoutConfig as $, type AgentMemoryAcquireRunLease as A, type BuildRetrievalBenchmarkCasesFromQrelsOptions as B, type KnowledgeBenchmarkScenario as C, type KnowledgeBenchmarkSliceSummary as D, type KnowledgeBenchmarkSource as E, type KnowledgeBenchmarkSpec as F, type KnowledgeBenchmarkSplit as G, type KnowledgeBenchmarkTaskKind as H, type KnowledgeClaimMatcher as I, type KnowledgeMemoryBenchmarkCase as J, type KnowledgeAnswerBenchmarkCase as K, type KnowledgeMemoryBenchmarkTaskKind as L, type KnowledgeMemoryEvent as M, type KnowledgeMemoryFactMatcher as N, type KnowledgeRetrievalBenchmarkCase as O, type KnowledgeRetrievalBenchmarkQrel as P, type KnowledgeRetrievalBenchmarkQuery as Q, type RetrievalConfig as R, type MemoryAdapterBenchmarkCandidate as S, type MemoryAdapterBenchmarkRankingRow as T, type OwnedAgentMemoryRunLease as U, type PartitionRetrievalScenariosOptions as V, type RetrievalEvalRetrieverInput as W, type RetrievalEvalRetrieverResult as X, type RetrievalGoldTarget as Y, type RetrievalHoldoutBypassReason as Z, type RetrievalHoldoutCallContext as _, type RetrievalEvalScenario as a, type RetrievalHoldoutEligibleItem as a0, type RetrievalHoldoutEvent as a1, type RetrievalHoldoutResult as a2, type RetrievalHoldoutSessionState as a3, type RetrievalMetricSummary as a4, type RetrievalRecallJudgeOptions as a5, type RetrievalScenarioPartitions as a6, type RetrievedSourceSpan as a7, type RunKnowledgeBenchmarkSuiteOptions as a8, type RunKnowledgeBenchmarkSuiteResult as a9, type RunMemoryAdapterBenchmarkOptions as aa, type RunMemoryAdapterBenchmarkResult as ab, acquireAgentMemoryRunLease as ac, buildRetrievalEvalDispatch as ad, partitionRetrievalScenarios as ae, retrievalConfigFromSurface as af, retrievalConfigSurface as ag, retrievalRecallJudge as ah, scoreRetrievalArtifact as ai, type RetrievalEvalArtifact as b, type RetrievalEvalRetriever as c, type RetrievalMetricWeights as d, type RetrievedKnowledgeHit as e, type AgentMemoryAdapter as f, type AgentMemoryBranchIsolation as g, type AgentMemoryContext as h, type AgentMemoryControllerMode as i, type AgentMemoryHit as j, type AgentMemoryKind as k, type AgentMemoryRunLease as l, type AgentMemoryScope as m, type AgentMemorySearchOptions as n, type AgentMemoryWriteInput as o, type AgentMemoryWriteResult as p, type BuildRetrievalEvalDispatchOptions as q, type KnowledgeAnswerBenchmarkTaskKind as r, type KnowledgeBenchmarkArtifact as s, type KnowledgeBenchmarkCase as t, type KnowledgeBenchmarkCaseBase as u, type KnowledgeBenchmarkDistribution as v, type KnowledgeBenchmarkEvaluation as w, type KnowledgeBenchmarkFamily as x, type KnowledgeBenchmarkReport as y, type KnowledgeBenchmarkResponder as z };
@@ -1,6 +1,6 @@
1
1
  # RAG Eval Completion Roadmap
2
2
 
3
- Verdict: `runRetrievalImprovementLoop()` is the right first loop, but it is only the retrieval layer.
3
+ Verdict: use retrieval-only optimization when the retriever is the only changing component, and full RAG optimization when retrieval and answer behavior must move together.
4
4
  SOTA RAG evaluation requires retrieval quality, context quality, generated-answer quality, abstention behavior, robustness, and operating budgets.
5
5
 
6
6
  ## Research Basis
@@ -19,12 +19,15 @@ SOTA RAG evaluation requires retrieval quality, context quality, generated-answe
19
19
 
20
20
  Done:
21
21
 
22
- - `runRetrievalImprovementLoop()` auto-searches retrieval configs through `agent-eval`.
22
+ - `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs.
23
+ - `runRagOptimization()` does the same for a serialized retrieval and answer configuration.
23
24
  - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases:
24
25
  retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion.
25
26
  - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans.
26
- - The retrieval judge reports recall, MRR, nDCG, precision@k, cost, and held-out promotion.
27
- - The loop is tested with a real `agent-eval` run where `{ k: 2 }` beats `{ k: 1 }`.
27
+ - The retrieval judge reports recall, MRR, nDCG, and precision@k; `agent-eval` reports cost separately.
28
+ - Selection and final data remain independent, and the optimization method never receives final cases.
29
+ - Every reusable optimization requires an immutable execution reference covering retrieval, index, model, judge, and external-service behavior.
30
+ - The integration is tested with complete methods for retrieval and full RAG configuration.
28
31
  - The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`.
29
32
  - `ragAnswerQualityJudge()` and `createRagAnswerQualityHook()` score context precision/recall/relevance/sufficiency, faithfulness, answer relevance/correctness, citation support, abstention, and unsupported-answer rate.
30
33
  - `normalizeExternalRagScores()` and the row exporters make Ragas, DeepEval, TruLens, RAGChecker, and custom evaluator outputs pluggable instead of hard dependencies.
@@ -56,9 +59,9 @@ Required slices:
56
59
 
57
60
  Ship criteria:
58
61
 
59
- - Holdout source-span Recall@5 is at least 0.90.
60
- - Holdout nDCG@5 is at least 0.80.
61
- - Train-to-holdout recall gap is at most 0.08.
62
+ - Final source-span Recall@5 is at least 0.90.
63
+ - Final nDCG@5 is at least 0.80.
64
+ - Train-to-final recall gap is at most 0.08.
62
65
  - Stale or forbidden source hit rate is at most 0.02.
63
66
  - p95 retrieval latency and cost do not regress by more than 10 percent versus baseline.
64
67
 
@@ -69,9 +72,9 @@ Score it with deterministic checks first and LLM judges only for semantic qualit
69
72
 
70
73
  Ship criteria:
71
74
 
72
- - Faithfulness or groundedness is at least 0.95 on holdout.
73
- - Answer relevance is at least 0.90 on holdout.
74
- - Answer correctness is at least 0.85 on human-labeled holdout.
75
+ - Faithfulness or groundedness is at least 0.95 on final cases.
76
+ - Answer relevance is at least 0.90 on final cases.
77
+ - Answer correctness is at least 0.85 on human-labeled final cases.
75
78
  - Citation support is at least 0.95 for claims that cite sources.
76
79
  - Unsupported-answer rate on unanswerable questions is at most 0.05.
77
80
 
@@ -101,8 +104,8 @@ Ship criteria:
101
104
  ### Phase 4: Production Loop
102
105
 
103
106
  Run the same eval pack on every retrieval or prompt change.
104
- Keep train/dev/holdout isolated.
105
- Never tune on holdout.
107
+ Keep train, selection, and final data isolated.
108
+ Never tune on final data.
106
109
 
107
110
  Ship criteria:
108
111
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-knowledge",
3
- "version": "4.1.0",
4
- "description": "Source-grounded, eval-gated knowledge growth primitives for agents.",
3
+ "version": "5.0.0",
4
+ "description": "Build, search, evaluate, and improve source-backed knowledge bases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-knowledge#readme",
6
6
  "repository": {
7
7
  "type": "git",
@@ -51,6 +51,7 @@
51
51
  "files": [
52
52
  "dist",
53
53
  "docs",
54
+ "skills",
54
55
  "AGENTS.md",
55
56
  "CHANGELOG.md",
56
57
  "README.md"
@@ -67,11 +68,13 @@
67
68
  "typecheck": "tsc --noEmit",
68
69
  "lint": "biome check src tests",
69
70
  "format": "biome format --write src tests",
70
- "verify:package": "node scripts/verify-package.mjs"
71
+ "check:skills": "node scripts/check-skills.mjs",
72
+ "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs",
73
+ "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs"
71
74
  },
72
75
  "dependencies": {
73
- "@tangle-network/agent-eval": "^0.122.8",
74
- "@tangle-network/agent-interface": "^0.31.0",
76
+ "@tangle-network/agent-eval": "0.126.5",
77
+ "@tangle-network/agent-interface": "^0.32.0",
75
78
  "proper-lockfile": "4.1.2",
76
79
  "zod": "^4.4.3"
77
80
  },
@@ -0,0 +1,97 @@
1
+ ---
2
+ name: build-with-agent-knowledge
3
+ description: Build, evaluate, and improve source-backed knowledge, retrieval, and memory systems.
4
+ ---
5
+
6
+ # Build With Agent Knowledge
7
+
8
+ Use this when a product needs a knowledge base, retrieval system, RAG improvement process, or agent memory provider.
9
+ Read the installed package README, exports, types, and nearest tests before choosing an API.
10
+ Do not copy signatures from this skill.
11
+
12
+ ## Choose The Job
13
+
14
+ | Product need | Package capability |
15
+ |---|---|
16
+ | Start a source-backed Markdown knowledge base | File knowledge base and source registry |
17
+ | Search an existing package knowledge base | File search provider or a product search adapter |
18
+ | Improve knowledge without editing live files | Isolated knowledge candidates |
19
+ | Tune retrieval on labeled questions | Retrieval improvement loop |
20
+ | Diagnose and repair retrieval, sources, pages, and answers together | RAG knowledge improvement loop |
21
+ | Compare memory systems | Memory adapter and experiment APIs |
22
+ | Run retrieval, answer, knowledge, or memory cases | Knowledge benchmark APIs |
23
+ | Let agents research or edit candidates | Runtime knowledge integration |
24
+
25
+ Use the narrowest capability that solves the product problem.
26
+ Do not add an agent loop when deterministic ingestion or indexing is enough.
27
+
28
+ ## Define Truth And Success
29
+
30
+ Record:
31
+
32
+ - the user task the knowledge should improve;
33
+ - authoritative and disallowed sources;
34
+ - tenant, user, agent, and sharing scope;
35
+ - freshness and deletion requirements;
36
+ - the current retrieval, answer, or memory baseline;
37
+ - objective checks, semantic checks, cost, and latency limits;
38
+ - who may approve and apply a candidate.
39
+
40
+ Use independent labels and source evidence.
41
+ The agent's current answer is not a gold answer.
42
+
43
+ ## Keep The Boundary Clean
44
+
45
+ `agent-knowledge` owns source records, indexes, retrieval tests, memory contracts, isolated candidates, and exact promotion.
46
+ It does not own model choice, prompts, browsing, agent scheduling, product authorization, or product storage transactions.
47
+
48
+ Supply callbacks for research, retrieval, answer generation, and scoring.
49
+ Use `@tangle-network/agent-runtime/knowledge` when those callbacks should run agents.
50
+ Use existing vector, graph, search, and memory systems through adapters instead of rebuilding their databases here.
51
+
52
+ ## Build The Smallest Complete Path
53
+
54
+ 1. Ingest one real source with provenance and tenant scope.
55
+ 2. Build or connect the index used by the product.
56
+ 3. Run one representative query or memory sequence through the production path.
57
+ 4. Capture retrieved items, final answer or action, citations, errors, tokens, cost, and latency.
58
+ 5. Prove a known good case passes and a realistic unsupported or missed case fails.
59
+ 6. Add only the missing improvement step: retrieval search, source acquisition, page update, answer repair, or memory policy.
60
+ 7. Write changes to an isolated candidate with a stable run identity.
61
+ 8. Compare baseline and candidate on the same development cases, then on unseen cases.
62
+ 9. Apply only an approved candidate whose base identity is still current.
63
+
64
+ Use separate run IDs for parallel branches and the same run ID to resume interrupted work.
65
+ Reject stale promotion rather than replacing newer knowledge.
66
+
67
+ ## Evaluate The Right Layer
68
+
69
+ | Layer | Minimum evidence |
70
+ |---|---|
71
+ | Retrieval | Labeled relevant items, ranking measures, misses, latency, and cost |
72
+ | Answer | Claim support, relevance, citation correctness, abstention, and final text |
73
+ | Knowledge base | Source coverage, provenance, freshness, structure, conflicts, and validation findings |
74
+ | Memory | Multi-turn task outcome, correct recall, harmful recall, isolation, writes, latency, and cost |
75
+
76
+ Report service and measurement failures separately from product failures.
77
+ Keep candidate-generation cases separate from the final decision set.
78
+ Bundled benchmark samples prove adapter wiring only; use complete external datasets for benchmark claims.
79
+
80
+ ## Completion
81
+
82
+ One customer-like path must prove:
83
+
84
+ ```text
85
+ source or memory event -> production retrieval -> observable answer or action
86
+ -> isolated candidate -> baseline comparison -> unseen comparison
87
+ -> approved promotion or correctly rejected change -> reproducible rerun
88
+ ```
89
+
90
+ Report installed versions, exact imports, provider adapters, scope policy, case counts, baseline and candidate results, cost, latency, candidate identity, promotion result, and artifact paths.
91
+
92
+ ## Then consider
93
+
94
+ - `eval-engineering` when new production-derived cases are needed.
95
+ - `build-with-agent-runtime` when agents should research, edit, or compare candidates.
96
+ - `agent-eval-adoption` when the product needs shared comparison and release records.
97
+ - `harden` when changing tenant isolation, source trust, deletion, or promotion authority.