@tangle-network/agent-knowledge 4.1.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -1
- package/CHANGELOG.md +28 -0
- package/README.md +220 -955
- package/dist/benchmarks/index.d.ts +3 -2
- package/dist/benchmarks/index.js +1 -1
- package/dist/{chunk-BBCIB4UL.js → chunk-EYIA5PLQ.js} +27 -195
- package/dist/chunk-EYIA5PLQ.js.map +1 -0
- package/dist/{chunk-CPMLJYA3.js → chunk-LMR53POQ.js} +1227 -574
- package/dist/chunk-LMR53POQ.js.map +1 -0
- package/dist/index-Cf7txrYP.d.ts +790 -0
- package/dist/index.d.ts +208 -24
- package/dist/index.js +632 -80
- package/dist/index.js.map +1 -1
- package/dist/memory/index.d.ts +8 -706
- package/dist/memory/index.js +2 -2
- package/dist/{types-DP38encz.d.ts → types-BY-xLVw-.d.ts} +16 -59
- package/docs/eval/rag-eval-roadmap.md +15 -12
- package/package.json +8 -5
- package/skills/build-with-agent-knowledge/SKILL.md +97 -0
- package/dist/chunk-BBCIB4UL.js.map +0 -1
- package/dist/chunk-CPMLJYA3.js.map +0 -1
package/dist/memory/index.js
CHANGED
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
runAgentMemoryExperiment,
|
|
25
25
|
runAgentMemoryImprovement,
|
|
26
26
|
toOffPolicyTrajectory
|
|
27
|
-
} from "../chunk-
|
|
27
|
+
} from "../chunk-LMR53POQ.js";
|
|
28
28
|
import {
|
|
29
29
|
AgentMemoryLifecycleTimeoutError,
|
|
30
30
|
AgentMemoryLifecycleUnsafeError,
|
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
resolveMemoryCleanupTimeoutMs,
|
|
38
38
|
runBoundedMemoryLifecycle,
|
|
39
39
|
sleepForMemoryRecovery
|
|
40
|
-
} from "../chunk-
|
|
40
|
+
} from "../chunk-EYIA5PLQ.js";
|
|
41
41
|
import "../chunk-DQ3PDMDP.js";
|
|
42
42
|
import "../chunk-YMKHCTS2.js";
|
|
43
43
|
export {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Scenario, DispatchContext, MutableSurface, JudgeConfig, CampaignStorage, CostLedgerHandle, CampaignResult } from '@tangle-network/agent-eval/campaign';
|
|
2
2
|
import { c as KnowledgeIndex, S as SourceRecord } from './types-6x0OpfW6.js';
|
|
3
|
+
import { AgentCandidateJsonValue } from '@tangle-network/agent-interface';
|
|
3
4
|
|
|
4
|
-
type RetrievalConfig = Record<string,
|
|
5
|
-
type RetrievalParameterSearchSpace = Record<string, readonly JsonValue[]>;
|
|
5
|
+
type RetrievalConfig = Record<string, AgentCandidateJsonValue>;
|
|
6
6
|
type RetrievalGoldTarget = {
|
|
7
7
|
kind: 'page';
|
|
8
8
|
pageId: string;
|
|
@@ -44,7 +44,7 @@ interface RetrievedKnowledgeHit {
|
|
|
44
44
|
sourceIds?: readonly string[];
|
|
45
45
|
sourceSpans?: readonly RetrievedSourceSpan[];
|
|
46
46
|
snippet?: string;
|
|
47
|
-
metadata?: Record<string,
|
|
47
|
+
metadata?: Record<string, AgentCandidateJsonValue>;
|
|
48
48
|
}
|
|
49
49
|
interface RetrievalEvalArtifact {
|
|
50
50
|
config: RetrievalConfig;
|
|
@@ -54,7 +54,7 @@ interface RetrievalEvalArtifact {
|
|
|
54
54
|
durationMs: number;
|
|
55
55
|
/** Informational copy. Billable retrievers account through context.cost.runPaidCall. */
|
|
56
56
|
costUsd?: number;
|
|
57
|
-
metadata?: Record<string,
|
|
57
|
+
metadata?: Record<string, AgentCandidateJsonValue>;
|
|
58
58
|
}
|
|
59
59
|
interface RetrievalMetricSummary {
|
|
60
60
|
recall: number;
|
|
@@ -79,7 +79,7 @@ interface RetrievalEvalRetrieverResult {
|
|
|
79
79
|
hits: readonly RetrievedKnowledgeHit[];
|
|
80
80
|
/** Informational copy. Billable retrievers account through context.cost.runPaidCall. */
|
|
81
81
|
costUsd?: number;
|
|
82
|
-
metadata?: Record<string,
|
|
82
|
+
metadata?: Record<string, AgentCandidateJsonValue>;
|
|
83
83
|
}
|
|
84
84
|
type RetrievalEvalRetriever = (input: RetrievalEvalRetrieverInput) => Promise<readonly RetrievedKnowledgeHit[] | RetrievalEvalRetrieverResult>;
|
|
85
85
|
interface BuildRetrievalEvalDispatchOptions {
|
|
@@ -97,65 +97,22 @@ interface RetrievalRecallJudgeOptions {
|
|
|
97
97
|
name?: string;
|
|
98
98
|
weights?: RetrievalMetricWeights;
|
|
99
99
|
}
|
|
100
|
-
interface
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
candidates?: readonly ParameterCandidate[];
|
|
105
|
-
searchSpace?: RetrievalParameterSearchSpace;
|
|
106
|
-
baseline?: RetrievalConfig;
|
|
100
|
+
interface PartitionRetrievalScenariosOptions {
|
|
101
|
+
selectionFraction?: number;
|
|
102
|
+
finalFraction?: number;
|
|
103
|
+
seed?: number;
|
|
107
104
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
holdoutScenarios?: readonly RetrievalEvalScenario[];
|
|
113
|
-
index?: KnowledgeIndex;
|
|
114
|
-
defaultK?: number;
|
|
115
|
-
retrieve?: RetrievalEvalRetriever;
|
|
116
|
-
candidates?: readonly ParameterCandidate[];
|
|
117
|
-
searchSpace?: RetrievalParameterSearchSpace;
|
|
118
|
-
judges?: readonly JudgeConfig<RetrievalEvalArtifact, RetrievalEvalScenario>[];
|
|
119
|
-
gate?: Gate<RetrievalEvalArtifact, RetrievalEvalScenario>;
|
|
120
|
-
metricWeights?: RetrievalMetricWeights;
|
|
121
|
-
targetRecall?: number;
|
|
122
|
-
holdoutFraction?: number;
|
|
123
|
-
splitSeed?: number;
|
|
124
|
-
deltaThreshold?: number;
|
|
125
|
-
runDir?: RetrievalLoopBaseOptions['runDir'];
|
|
126
|
-
seed?: RetrievalLoopBaseOptions['seed'];
|
|
127
|
-
reps?: RetrievalLoopBaseOptions['reps'];
|
|
128
|
-
resumable?: RetrievalLoopBaseOptions['resumable'];
|
|
129
|
-
costCeiling?: RetrievalLoopBaseOptions['costCeiling'];
|
|
130
|
-
maxConcurrency?: RetrievalLoopBaseOptions['maxConcurrency'];
|
|
131
|
-
dispatchTimeoutMs?: RetrievalLoopBaseOptions['dispatchTimeoutMs'];
|
|
132
|
-
expectUsage?: RetrievalLoopBaseOptions['expectUsage'];
|
|
133
|
-
tracing?: RetrievalLoopBaseOptions['tracing'];
|
|
134
|
-
storage?: RetrievalLoopBaseOptions['storage'];
|
|
135
|
-
populationSize?: RetrievalLoopBaseOptions['populationSize'];
|
|
136
|
-
maxGenerations?: RetrievalLoopBaseOptions['maxGenerations'];
|
|
137
|
-
promoteTopK?: RetrievalLoopBaseOptions['promoteTopK'];
|
|
138
|
-
maxImprovementShots?: RetrievalLoopBaseOptions['maxImprovementShots'];
|
|
139
|
-
report?: RetrievalLoopBaseOptions['report'];
|
|
140
|
-
findings?: RetrievalLoopBaseOptions['findings'];
|
|
141
|
-
now?: RetrievalLoopBaseOptions['now'];
|
|
142
|
-
}
|
|
143
|
-
interface RunRetrievalImprovementLoopResult extends RunImprovementLoopResult<RetrievalEvalArtifact, RetrievalEvalScenario> {
|
|
144
|
-
baselineConfig: RetrievalConfig;
|
|
145
|
-
winnerConfig: RetrievalConfig;
|
|
146
|
-
trainScenarios: readonly RetrievalEvalScenario[];
|
|
147
|
-
holdoutScenarios: readonly RetrievalEvalScenario[];
|
|
148
|
-
candidates: readonly ParameterCandidate[];
|
|
149
|
-
targetRecall?: number;
|
|
105
|
+
interface RetrievalScenarioPartitions {
|
|
106
|
+
trainScenarios: RetrievalEvalScenario[];
|
|
107
|
+
selectionScenarios: RetrievalEvalScenario[];
|
|
108
|
+
finalScenarios: RetrievalEvalScenario[];
|
|
150
109
|
}
|
|
151
110
|
declare function retrievalConfigSurface(config: RetrievalConfig): string;
|
|
152
111
|
declare function retrievalConfigFromSurface(surface: MutableSurface): RetrievalConfig;
|
|
153
112
|
declare function buildRetrievalEvalDispatch(options: BuildRetrievalEvalDispatchOptions): (surface: MutableSurface, scenario: RetrievalEvalScenario, context: DispatchContext) => Promise<RetrievalEvalArtifact>;
|
|
154
113
|
declare function retrievalRecallJudge(options?: RetrievalRecallJudgeOptions): JudgeConfig<RetrievalEvalArtifact, RetrievalEvalScenario>;
|
|
155
114
|
declare function scoreRetrievalArtifact(artifact: RetrievalEvalArtifact, scenario: RetrievalEvalScenario): RetrievalMetricSummary;
|
|
156
|
-
declare function
|
|
157
|
-
declare function retrievalParameterSweepProposer(options: RetrievalParameterSweepProposerOptions): SurfaceProposer;
|
|
158
|
-
declare function runRetrievalImprovementLoop(options: RunRetrievalImprovementLoopOptions): Promise<RunRetrievalImprovementLoopResult>;
|
|
115
|
+
declare function partitionRetrievalScenarios(scenarios: readonly RetrievalEvalScenario[], options?: PartitionRetrievalScenariosOptions): RetrievalScenarioPartitions;
|
|
159
116
|
|
|
160
117
|
type AgentMemoryKind = 'message' | 'entity' | 'fact' | 'preference' | 'observation' | 'reasoning-trace';
|
|
161
118
|
interface AgentMemoryScope {
|
|
@@ -662,4 +619,4 @@ interface BuildRetrievalBenchmarkCasesFromQrelsOptions {
|
|
|
662
619
|
splitOf?: (queryId: string) => KnowledgeBenchmarkSplit;
|
|
663
620
|
}
|
|
664
621
|
|
|
665
|
-
export { type RetrievalHoldoutConfig as $, type AgentMemoryAcquireRunLease as A, type BuildRetrievalBenchmarkCasesFromQrelsOptions as B, type
|
|
622
|
+
export { type RetrievalHoldoutConfig as $, type AgentMemoryAcquireRunLease as A, type BuildRetrievalBenchmarkCasesFromQrelsOptions as B, type KnowledgeBenchmarkScenario as C, type KnowledgeBenchmarkSliceSummary as D, type KnowledgeBenchmarkSource as E, type KnowledgeBenchmarkSpec as F, type KnowledgeBenchmarkSplit as G, type KnowledgeBenchmarkTaskKind as H, type KnowledgeClaimMatcher as I, type KnowledgeMemoryBenchmarkCase as J, type KnowledgeAnswerBenchmarkCase as K, type KnowledgeMemoryBenchmarkTaskKind as L, type KnowledgeMemoryEvent as M, type KnowledgeMemoryFactMatcher as N, type KnowledgeRetrievalBenchmarkCase as O, type KnowledgeRetrievalBenchmarkQrel as P, type KnowledgeRetrievalBenchmarkQuery as Q, type RetrievalConfig as R, type MemoryAdapterBenchmarkCandidate as S, type MemoryAdapterBenchmarkRankingRow as T, type OwnedAgentMemoryRunLease as U, type PartitionRetrievalScenariosOptions as V, type RetrievalEvalRetrieverInput as W, type RetrievalEvalRetrieverResult as X, type RetrievalGoldTarget as Y, type RetrievalHoldoutBypassReason as Z, type RetrievalHoldoutCallContext as _, type RetrievalEvalScenario as a, type RetrievalHoldoutEligibleItem as a0, type RetrievalHoldoutEvent as a1, type RetrievalHoldoutResult as a2, type RetrievalHoldoutSessionState as a3, type RetrievalMetricSummary as a4, type RetrievalRecallJudgeOptions as a5, type RetrievalScenarioPartitions as a6, type RetrievedSourceSpan as a7, type RunKnowledgeBenchmarkSuiteOptions as a8, type RunKnowledgeBenchmarkSuiteResult as a9, type RunMemoryAdapterBenchmarkOptions as aa, type RunMemoryAdapterBenchmarkResult as ab, acquireAgentMemoryRunLease as ac, buildRetrievalEvalDispatch as ad, partitionRetrievalScenarios as ae, retrievalConfigFromSurface as af, retrievalConfigSurface as ag, retrievalRecallJudge as ah, scoreRetrievalArtifact as ai, type RetrievalEvalArtifact as b, type RetrievalEvalRetriever as c, type RetrievalMetricWeights as d, type RetrievedKnowledgeHit as e, type AgentMemoryAdapter as f, type AgentMemoryBranchIsolation as g, type AgentMemoryContext as h, type AgentMemoryControllerMode as i, type AgentMemoryHit as j, type AgentMemoryKind as k, type AgentMemoryRunLease as l, type AgentMemoryScope as m, type AgentMemorySearchOptions as n, type AgentMemoryWriteInput as o, type AgentMemoryWriteResult as p, type BuildRetrievalEvalDispatchOptions as q, type KnowledgeAnswerBenchmarkTaskKind as r, type KnowledgeBenchmarkArtifact as s, type KnowledgeBenchmarkCase as t, type KnowledgeBenchmarkCaseBase as u, type KnowledgeBenchmarkDistribution as v, type KnowledgeBenchmarkEvaluation as w, type KnowledgeBenchmarkFamily as x, type KnowledgeBenchmarkReport as y, type KnowledgeBenchmarkResponder as z };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# RAG Eval Completion Roadmap
|
|
2
2
|
|
|
3
|
-
Verdict:
|
|
3
|
+
Verdict: use retrieval-only optimization when the retriever is the only changing component, and full RAG optimization when retrieval and answer behavior must move together.
|
|
4
4
|
SOTA RAG evaluation requires retrieval quality, context quality, generated-answer quality, abstention behavior, robustness, and operating budgets.
|
|
5
5
|
|
|
6
6
|
## Research Basis
|
|
@@ -19,12 +19,15 @@ SOTA RAG evaluation requires retrieval quality, context quality, generated-answe
|
|
|
19
19
|
|
|
20
20
|
Done:
|
|
21
21
|
|
|
22
|
-
- `runRetrievalImprovementLoop()`
|
|
22
|
+
- `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs.
|
|
23
|
+
- `runRagOptimization()` does the same for a serialized retrieval and answer configuration.
|
|
23
24
|
- `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases:
|
|
24
25
|
retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion.
|
|
25
26
|
- Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans.
|
|
26
|
-
- The retrieval judge reports recall, MRR, nDCG, precision@k
|
|
27
|
-
-
|
|
27
|
+
- The retrieval judge reports recall, MRR, nDCG, and precision@k; `agent-eval` reports cost separately.
|
|
28
|
+
- Selection and final data remain independent, and the optimization method never receives final cases.
|
|
29
|
+
- Every reusable optimization requires an immutable execution reference covering retrieval, index, model, judge, and external-service behavior.
|
|
30
|
+
- The integration is tested with complete methods for retrieval and full RAG configuration.
|
|
28
31
|
- The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`.
|
|
29
32
|
- `ragAnswerQualityJudge()` and `createRagAnswerQualityHook()` score context precision/recall/relevance/sufficiency, faithfulness, answer relevance/correctness, citation support, abstention, and unsupported-answer rate.
|
|
30
33
|
- `normalizeExternalRagScores()` and the row exporters make Ragas, DeepEval, TruLens, RAGChecker, and custom evaluator outputs pluggable instead of hard dependencies.
|
|
@@ -56,9 +59,9 @@ Required slices:
|
|
|
56
59
|
|
|
57
60
|
Ship criteria:
|
|
58
61
|
|
|
59
|
-
-
|
|
60
|
-
-
|
|
61
|
-
- Train-to-
|
|
62
|
+
- Final source-span Recall@5 is at least 0.90.
|
|
63
|
+
- Final nDCG@5 is at least 0.80.
|
|
64
|
+
- Train-to-final recall gap is at most 0.08.
|
|
62
65
|
- Stale or forbidden source hit rate is at most 0.02.
|
|
63
66
|
- p95 retrieval latency and cost do not regress by more than 10 percent versus baseline.
|
|
64
67
|
|
|
@@ -69,9 +72,9 @@ Score it with deterministic checks first and LLM judges only for semantic qualit
|
|
|
69
72
|
|
|
70
73
|
Ship criteria:
|
|
71
74
|
|
|
72
|
-
- Faithfulness or groundedness is at least 0.95 on
|
|
73
|
-
- Answer relevance is at least 0.90 on
|
|
74
|
-
- Answer correctness is at least 0.85 on human-labeled
|
|
75
|
+
- Faithfulness or groundedness is at least 0.95 on final cases.
|
|
76
|
+
- Answer relevance is at least 0.90 on final cases.
|
|
77
|
+
- Answer correctness is at least 0.85 on human-labeled final cases.
|
|
75
78
|
- Citation support is at least 0.95 for claims that cite sources.
|
|
76
79
|
- Unsupported-answer rate on unanswerable questions is at most 0.05.
|
|
77
80
|
|
|
@@ -101,8 +104,8 @@ Ship criteria:
|
|
|
101
104
|
### Phase 4: Production Loop
|
|
102
105
|
|
|
103
106
|
Run the same eval pack on every retrieval or prompt change.
|
|
104
|
-
Keep train
|
|
105
|
-
Never tune on
|
|
107
|
+
Keep train, selection, and final data isolated.
|
|
108
|
+
Never tune on final data.
|
|
106
109
|
|
|
107
110
|
Ship criteria:
|
|
108
111
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-knowledge",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "5.0.0",
|
|
4
|
+
"description": "Build, search, evaluate, and improve source-backed knowledge bases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-knowledge#readme",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -51,6 +51,7 @@
|
|
|
51
51
|
"files": [
|
|
52
52
|
"dist",
|
|
53
53
|
"docs",
|
|
54
|
+
"skills",
|
|
54
55
|
"AGENTS.md",
|
|
55
56
|
"CHANGELOG.md",
|
|
56
57
|
"README.md"
|
|
@@ -67,11 +68,13 @@
|
|
|
67
68
|
"typecheck": "tsc --noEmit",
|
|
68
69
|
"lint": "biome check src tests",
|
|
69
70
|
"format": "biome format --write src tests",
|
|
70
|
-
"
|
|
71
|
+
"check:skills": "node scripts/check-skills.mjs",
|
|
72
|
+
"verify:package": "pnpm run check:skills && node scripts/verify-package.mjs",
|
|
73
|
+
"verify:official-optimizers": "node scripts/verify-official-optimizers.mjs"
|
|
71
74
|
},
|
|
72
75
|
"dependencies": {
|
|
73
|
-
"@tangle-network/agent-eval": "
|
|
74
|
-
"@tangle-network/agent-interface": "^0.
|
|
76
|
+
"@tangle-network/agent-eval": "0.126.5",
|
|
77
|
+
"@tangle-network/agent-interface": "^0.32.0",
|
|
75
78
|
"proper-lockfile": "4.1.2",
|
|
76
79
|
"zod": "^4.4.3"
|
|
77
80
|
},
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: build-with-agent-knowledge
|
|
3
|
+
description: Build, evaluate, and improve source-backed knowledge, retrieval, and memory systems.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Build With Agent Knowledge
|
|
7
|
+
|
|
8
|
+
Use this when a product needs a knowledge base, retrieval system, RAG improvement process, or agent memory provider.
|
|
9
|
+
Read the installed package README, exports, types, and nearest tests before choosing an API.
|
|
10
|
+
Do not copy signatures from this skill.
|
|
11
|
+
|
|
12
|
+
## Choose The Job
|
|
13
|
+
|
|
14
|
+
| Product need | Package capability |
|
|
15
|
+
|---|---|
|
|
16
|
+
| Start a source-backed Markdown knowledge base | File knowledge base and source registry |
|
|
17
|
+
| Search an existing package knowledge base | File search provider or a product search adapter |
|
|
18
|
+
| Improve knowledge without editing live files | Isolated knowledge candidates |
|
|
19
|
+
| Tune retrieval on labeled questions | Retrieval improvement loop |
|
|
20
|
+
| Diagnose and repair retrieval, sources, pages, and answers together | RAG knowledge improvement loop |
|
|
21
|
+
| Compare memory systems | Memory adapter and experiment APIs |
|
|
22
|
+
| Run retrieval, answer, knowledge, or memory cases | Knowledge benchmark APIs |
|
|
23
|
+
| Let agents research or edit candidates | Runtime knowledge integration |
|
|
24
|
+
|
|
25
|
+
Use the narrowest capability that solves the product problem.
|
|
26
|
+
Do not add an agent loop when deterministic ingestion or indexing is enough.
|
|
27
|
+
|
|
28
|
+
## Define Truth And Success
|
|
29
|
+
|
|
30
|
+
Record:
|
|
31
|
+
|
|
32
|
+
- the user task the knowledge should improve;
|
|
33
|
+
- authoritative and disallowed sources;
|
|
34
|
+
- tenant, user, agent, and sharing scope;
|
|
35
|
+
- freshness and deletion requirements;
|
|
36
|
+
- the current retrieval, answer, or memory baseline;
|
|
37
|
+
- objective checks, semantic checks, cost, and latency limits;
|
|
38
|
+
- who may approve and apply a candidate.
|
|
39
|
+
|
|
40
|
+
Use independent labels and source evidence.
|
|
41
|
+
The agent's current answer is not a gold answer.
|
|
42
|
+
|
|
43
|
+
## Keep The Boundary Clean
|
|
44
|
+
|
|
45
|
+
`agent-knowledge` owns source records, indexes, retrieval tests, memory contracts, isolated candidates, and exact promotion.
|
|
46
|
+
It does not own model choice, prompts, browsing, agent scheduling, product authorization, or product storage transactions.
|
|
47
|
+
|
|
48
|
+
Supply callbacks for research, retrieval, answer generation, and scoring.
|
|
49
|
+
Use `@tangle-network/agent-runtime/knowledge` when those callbacks should run agents.
|
|
50
|
+
Use existing vector, graph, search, and memory systems through adapters instead of rebuilding their databases here.
|
|
51
|
+
|
|
52
|
+
## Build The Smallest Complete Path
|
|
53
|
+
|
|
54
|
+
1. Ingest one real source with provenance and tenant scope.
|
|
55
|
+
2. Build or connect the index used by the product.
|
|
56
|
+
3. Run one representative query or memory sequence through the production path.
|
|
57
|
+
4. Capture retrieved items, final answer or action, citations, errors, tokens, cost, and latency.
|
|
58
|
+
5. Prove a known good case passes and a realistic unsupported or missed case fails.
|
|
59
|
+
6. Add only the missing improvement step: retrieval search, source acquisition, page update, answer repair, or memory policy.
|
|
60
|
+
7. Write changes to an isolated candidate with a stable run identity.
|
|
61
|
+
8. Compare baseline and candidate on the same development cases, then on unseen cases.
|
|
62
|
+
9. Apply only an approved candidate whose base identity is still current.
|
|
63
|
+
|
|
64
|
+
Use separate run IDs for parallel branches and the same run ID to resume interrupted work.
|
|
65
|
+
Reject stale promotion rather than replacing newer knowledge.
|
|
66
|
+
|
|
67
|
+
## Evaluate The Right Layer
|
|
68
|
+
|
|
69
|
+
| Layer | Minimum evidence |
|
|
70
|
+
|---|---|
|
|
71
|
+
| Retrieval | Labeled relevant items, ranking measures, misses, latency, and cost |
|
|
72
|
+
| Answer | Claim support, relevance, citation correctness, abstention, and final text |
|
|
73
|
+
| Knowledge base | Source coverage, provenance, freshness, structure, conflicts, and validation findings |
|
|
74
|
+
| Memory | Multi-turn task outcome, correct recall, harmful recall, isolation, writes, latency, and cost |
|
|
75
|
+
|
|
76
|
+
Report service and measurement failures separately from product failures.
|
|
77
|
+
Keep candidate-generation cases separate from the final decision set.
|
|
78
|
+
Bundled benchmark samples prove adapter wiring only; use complete external datasets for benchmark claims.
|
|
79
|
+
|
|
80
|
+
## Completion
|
|
81
|
+
|
|
82
|
+
One customer-like path must prove:
|
|
83
|
+
|
|
84
|
+
```text
|
|
85
|
+
source or memory event -> production retrieval -> observable answer or action
|
|
86
|
+
-> isolated candidate -> baseline comparison -> unseen comparison
|
|
87
|
+
-> approved promotion or correctly rejected change -> reproducible rerun
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Report installed versions, exact imports, provider adapters, scope policy, case counts, baseline and candidate results, cost, latency, candidate identity, promotion result, and artifact paths.
|
|
91
|
+
|
|
92
|
+
## Then consider
|
|
93
|
+
|
|
94
|
+
- `eval-engineering` when new production-derived cases are needed.
|
|
95
|
+
- `build-with-agent-runtime` when agents should research, edit, or compare candidates.
|
|
96
|
+
- `agent-eval-adoption` when the product needs shared comparison and release records.
|
|
97
|
+
- `harden` when changing tenant isolation, source trust, deletion, or promotion authority.
|