agent-doc-system 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +134 -1
- package/README.md +1 -1
- package/adapters/cache.mjs +5 -4
- package/adapters/database.mjs +3 -2
- package/adapters/deploy-config.mjs +3 -2
- package/adapters/dockerfile.mjs +3 -2
- package/adapters/generic.mjs +2 -1
- package/adapters/github-actions.mjs +2 -1
- package/adapters/go.mjs +4 -3
- package/adapters/node.mjs +3 -2
- package/adapters/rust.mjs +3 -0
- package/adapters/turborepo.mjs +3 -2
- package/adapters/typescript.mjs +7 -5
- package/adapters/wrangler.mjs +3 -2
- package/bin/agentdoc.mjs +9 -2
- package/bin/usage.txt +2 -0
- package/core/audit.mjs +7 -5
- package/core/compile.mjs +7 -5
- package/core/derive.mjs +4 -3
- package/core/descriptors.mjs +3 -2
- package/core/discover.mjs +3 -2
- package/core/fsx.mjs +174 -42
- package/core/graph.mjs +1 -1
- package/core/impact.mjs +11 -4
- package/core/paths.mjs +71 -0
- package/core/query.mjs +58 -23
- package/core/scaffold.mjs +130 -23
- package/core/sourcescan.mjs +4 -1
- package/docs/ADAPTERS.md +1 -1
- package/docs/BENCHMARK.md +83 -21
- package/docs/CI.md +7 -1
- package/docs/EVALS.md +209 -17
- package/docs/MIGRATION.md +22 -4
- package/docs/PROVENANCE.md +6 -0
- package/docs/QUERY.md +25 -3
- package/docs/SPEC.md +47 -2
- package/docs/ci/github-actions.yml +1 -1
- package/evals/baseline-search.mjs +272 -0
- package/evals/facts.mjs +28 -0
- package/evals/harness.mjs +120 -116
- package/package.json +4 -2
- package/skills/agent-doc-system/SKILL.md +2 -1
- package/skills/agent-doc-system/references/cli.md +6 -1
- package/skills/agent-doc-system/references/create-migrate.md +8 -2
- package/templates/ci.yml +1 -1
package/evals/harness.mjs
CHANGED
|
@@ -4,14 +4,8 @@
|
|
|
4
4
|
// an agent through it retrieve the right context, and less noise than unassisted
|
|
5
5
|
// repository exploration".
|
|
6
6
|
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
// baseline what an agent gets by exploring the repository the obvious way:
|
|
10
|
-
// every documentation file, every descriptor, every source file
|
|
11
|
-
// under the touched component. This is the honest shape of "just
|
|
12
|
-
// read the repo", and it is what the catalog has to beat.
|
|
13
|
-
//
|
|
14
|
-
// catalog what `agentdoc query` returns for the same scenario.
|
|
7
|
+
// Compare catalog routing with a graph-assisted targeted reference and a
|
|
8
|
+
// working-tree search baseline. The full graph is only a size yardstick.
|
|
15
9
|
//
|
|
16
10
|
// Retrieval-set overlap against a per-scenario ground truth gives precision,
|
|
17
11
|
// recall, critical-constraint recall, verification-command recall and noise.
|
|
@@ -28,32 +22,35 @@ import { AgentDocError, CODES } from "../core/codes.mjs";
|
|
|
28
22
|
import { queryContext, ContextIndex } from "../core/query.mjs";
|
|
29
23
|
import { impact } from "../core/impact.mjs";
|
|
30
24
|
|
|
25
|
+
import { fact, addRelationFacts, addGraphRelations } from "./facts.mjs";
|
|
26
|
+
import { searchContext } from "./baseline-search.mjs";
|
|
27
|
+
|
|
31
28
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
32
29
|
|
|
33
30
|
// A retrieved item is a "fact reference" the agent would have to understand:
|
|
34
31
|
// an entity ref, a constraint path, a verification command, a contract ref.
|
|
35
32
|
export function factsFromContext(ctx) {
|
|
36
33
|
const out = new Set();
|
|
37
|
-
out.add(ctx.entity.ref);
|
|
38
|
-
for (const n of ctx.neighbours) out.add(n.ref);
|
|
39
|
-
for (const r of ctx.relations.items) out
|
|
40
|
-
for (const p of ctx.journeyPeers || []) out.add(p);
|
|
41
|
-
for (const s2 of ctx.placement.systems) out.add(s2);
|
|
42
|
-
for (const d2 of ctx.placement.domains) out.add(d2);
|
|
43
|
-
for (const a of ctx.apis.provided) out.add("provides
|
|
44
|
-
for (const a of ctx.apis.consumed) out.add("consumes
|
|
45
|
-
for (const c of ctx.apis.consumers || []) out.add("consumer
|
|
46
|
-
if (ctx.apis.provider) out.add("provider
|
|
47
|
-
for (const r of ctx.resources.used) out.add("uses
|
|
48
|
-
for (const r of ctx.resources.usedBy) out.add("usedBy
|
|
49
|
-
for (const e of ctx.events.items) out.add("event
|
|
50
|
-
for (const e of ctx.externalDependencies.items) out.add("external
|
|
51
|
-
for (const v of ctx.verification.items) out.add("verify
|
|
52
|
-
for (const p of ctx.constraints.constraints) out.add("constraint
|
|
53
|
-
for (const d of ctx.constraints.docs) out.add("doc
|
|
54
|
-
for (const r of ctx.constraints.runbooks) out.add("runbook
|
|
55
|
-
for (const c of ctx.authority.conflicts.items) out.add("conflict
|
|
56
|
-
for (const o of ctx.authority.observations) out.add("observation
|
|
34
|
+
out.add(fact("ref", ctx.entity.ref));
|
|
35
|
+
for (const n of ctx.neighbours) out.add(fact("ref", n.ref));
|
|
36
|
+
for (const r of ctx.relations.items) addRelationFacts(out, r.type, r.other, r.direction);
|
|
37
|
+
for (const p of ctx.journeyPeers || []) out.add(fact("ref", p));
|
|
38
|
+
for (const s2 of ctx.placement.systems) out.add(fact("ref", s2));
|
|
39
|
+
for (const d2 of ctx.placement.domains) out.add(fact("ref", d2));
|
|
40
|
+
for (const a of ctx.apis.provided) out.add(fact("provides", a.api));
|
|
41
|
+
for (const a of ctx.apis.consumed) out.add(fact("consumes", a.api));
|
|
42
|
+
for (const c of ctx.apis.consumers || []) out.add(fact("consumer", c));
|
|
43
|
+
if (ctx.apis.provider) out.add(fact("provider", ctx.apis.provider));
|
|
44
|
+
for (const r of ctx.resources.used) out.add(fact("uses", r));
|
|
45
|
+
for (const r of ctx.resources.usedBy) out.add(fact("usedBy", r));
|
|
46
|
+
for (const e of ctx.events.items) out.add(fact("event", e.pattern));
|
|
47
|
+
for (const e of ctx.externalDependencies.items) out.add(fact("external", e.name));
|
|
48
|
+
for (const v of ctx.verification.items) out.add(fact("verify", v.command));
|
|
49
|
+
for (const p of ctx.constraints.constraints) out.add(fact("constraint", p));
|
|
50
|
+
for (const d of ctx.constraints.docs) out.add(fact("doc", d));
|
|
51
|
+
for (const r of ctx.constraints.runbooks) out.add(fact("runbook", r));
|
|
52
|
+
for (const c of ctx.authority.conflicts.items) out.add(fact("conflict", c.key));
|
|
53
|
+
for (const o of ctx.authority.observations) out.add(fact("observation", o.id));
|
|
57
54
|
return out;
|
|
58
55
|
}
|
|
59
56
|
|
|
@@ -62,17 +59,17 @@ export function factsFromContext(ctx) {
|
|
|
62
59
|
// resource, an API boundary, or an event subject it publishes or consumes.
|
|
63
60
|
export function criticalFromContext(ctx) {
|
|
64
61
|
const out = new Set();
|
|
65
|
-
for (const e of ctx.events.items) out.add("event
|
|
66
|
-
for (const p of ctx.constraints.constraints) out.add("constraint
|
|
67
|
-
for (const c of ctx.authority.conflicts.items) out.add("conflict
|
|
68
|
-
for (const o of ctx.authority.observations) out.add("observation
|
|
69
|
-
for (const r of ctx.resources.used) out.add("uses
|
|
70
|
-
for (const r of ctx.resources.usedBy) out.add("usedBy
|
|
71
|
-
for (const a of ctx.apis.consumed) out.add("consumes
|
|
72
|
-
for (const s2 of ctx.placement.systems) out.add(s2);
|
|
73
|
-
for (const d2 of ctx.placement.domains) out.add(d2);
|
|
74
|
-
for (const a of ctx.apis.provided) out.add("provides
|
|
75
|
-
for (const c of ctx.apis.consumers || []) out.add("consumer
|
|
62
|
+
for (const e of ctx.events.items) out.add(fact("event", e.pattern));
|
|
63
|
+
for (const p of ctx.constraints.constraints) out.add(fact("constraint", p));
|
|
64
|
+
for (const c of ctx.authority.conflicts.items) out.add(fact("conflict", c.key));
|
|
65
|
+
for (const o of ctx.authority.observations) out.add(fact("observation", o.id));
|
|
66
|
+
for (const r of ctx.resources.used) out.add(fact("uses", r));
|
|
67
|
+
for (const r of ctx.resources.usedBy) out.add(fact("usedBy", r));
|
|
68
|
+
for (const a of ctx.apis.consumed) out.add(fact("consumes", a.api));
|
|
69
|
+
for (const s2 of ctx.placement.systems) out.add(fact("ref", s2));
|
|
70
|
+
for (const d2 of ctx.placement.domains) out.add(fact("ref", d2));
|
|
71
|
+
for (const a of ctx.apis.provided) out.add(fact("provides", a.api));
|
|
72
|
+
for (const c of ctx.apis.consumers || []) out.add(fact("consumer", c));
|
|
76
73
|
return out;
|
|
77
74
|
}
|
|
78
75
|
|
|
@@ -80,10 +77,10 @@ export function verificationFromContext(ctx) {
|
|
|
80
77
|
// Commands such as `go test ./...` recur in multiple modules. Scenarios can
|
|
81
78
|
// require a scoped check ID as well as a command to measure dependent checks
|
|
82
79
|
// without letting the library's identical command satisfy the expectation.
|
|
83
|
-
return new Set(ctx.verification.items.flatMap((v) => ["verify
|
|
80
|
+
return new Set(ctx.verification.items.flatMap((v) => [fact("verify", v.command), fact("verify-id", v.id)]));
|
|
84
81
|
}
|
|
85
82
|
|
|
86
|
-
// ── baseline:
|
|
83
|
+
// ── baseline-targeted: graph-assisted reference ────────────────────────
|
|
87
84
|
export function baselineContext(graph, touch) {
|
|
88
85
|
const idx = new ContextIndex(graph);
|
|
89
86
|
const seeds = touch.map((t) => idx.resolve(t)).filter(Boolean);
|
|
@@ -94,71 +91,60 @@ export function baselineContext(graph, touch) {
|
|
|
94
91
|
const inSeed = seeds.includes(e);
|
|
95
92
|
const underRoot = roots.some((r) => e.ref && r && d.sourcePath && (d.sourcePath === r || d.sourcePath.startsWith(r + "/")));
|
|
96
93
|
if (!inSeed && !underRoot) continue;
|
|
97
|
-
out.add(e.ref);
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
if (r.targetRef === e.ref) out.add(r.type + ":" + r.sourceRef);
|
|
101
|
-
}
|
|
94
|
+
out.add(fact("ref", e.ref));
|
|
95
|
+
if (e.kind === "API") out.add(fact("provides", e.ref));
|
|
96
|
+
addGraphRelations(out, graph, e.ref);
|
|
102
97
|
for (const ev of graph.interfaces.events) {
|
|
103
|
-
if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref)) out.add("event
|
|
104
|
-
}
|
|
105
|
-
for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add("external:" + x.name);
|
|
106
|
-
for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add("verify:" + v.command);
|
|
107
|
-
for (const c of graph.conflicts) if (c.subject === e.ref) out.add("conflict:" + c.key);
|
|
108
|
-
for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add("observation:" + o.id);
|
|
109
|
-
for (const p of d.docs || []) out.add("doc:" + p);
|
|
110
|
-
for (const p of d.constraints || []) out.add("constraint:" + p);
|
|
111
|
-
for (const p of d.runbooks || []) out.add("runbook:" + p);
|
|
112
|
-
for (const r of graph.relations) {
|
|
113
|
-
if (r.sourceRef === e.ref || r.targetRef === e.ref) out.add(r.type + ":" + (r.sourceRef === e.ref ? r.targetRef : r.sourceRef));
|
|
98
|
+
if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref)) out.add(fact("event", ev.pattern));
|
|
114
99
|
}
|
|
100
|
+
for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add(fact("external", x.name));
|
|
101
|
+
for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add(fact("verify", v.command));
|
|
102
|
+
for (const c of graph.conflicts) if (c.subject === e.ref) out.add(fact("conflict", c.key));
|
|
103
|
+
for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add(fact("observation", o.id));
|
|
104
|
+
for (const p of d.docs || []) out.add(fact("doc", p));
|
|
105
|
+
for (const p of d.constraints || []) out.add(fact("constraint", p));
|
|
106
|
+
for (const p of d.runbooks || []) out.add(fact("runbook", p));
|
|
115
107
|
}
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
for (const f of new Set([...repoDocPaths(graph)])) out.add("
|
|
108
|
+
// Preserve the original targeted reference's repository-wide doc sweep,
|
|
109
|
+
// now using the same labels as query rather than unmatched repodoc labels.
|
|
110
|
+
for (const f of new Set([...repoDocPaths(graph)])) out.add(fact("doc", f));
|
|
119
111
|
return out;
|
|
120
112
|
}
|
|
121
113
|
|
|
122
114
|
function repoDocPaths(graph) {
|
|
123
115
|
const out = new Set();
|
|
124
116
|
for (const e of graph.entities) {
|
|
125
|
-
for (const p of (e.derived || {}).docs || []) out.add(p);
|
|
117
|
+
for (const p of (e.derived || {}).docs || []) out.add(fact("ref", p));
|
|
126
118
|
}
|
|
127
|
-
for (const cfgp of graph.verification) for (const p of cfgp.configPaths || []) out.add(p);
|
|
119
|
+
for (const cfgp of graph.verification) for (const p of cfgp.configPaths || []) out.add(fact("ref", p));
|
|
128
120
|
return [...out].sort();
|
|
129
121
|
}
|
|
130
122
|
|
|
131
|
-
//
|
|
132
|
-
// every check in the repository (or nothing). The realistic reading of
|
|
133
|
-
// "unassisted" is the union, since no routing exists to narrow it.
|
|
134
|
-
// The whole-repository baseline: what an agent that has no routing ends up
|
|
135
|
-
// holding. Every entity, every relation, every subject, every check, every
|
|
136
|
-
// document. This is the honest cost of "just read the repository".
|
|
123
|
+
// Whole-graph fact inventory: a size yardstick, not unaided exploration.
|
|
137
124
|
export function fullRepositoryContext(graph) {
|
|
138
125
|
const out = new Set();
|
|
139
126
|
for (const e of graph.entities) {
|
|
140
|
-
out.add(e.ref);
|
|
127
|
+
out.add(fact("ref", e.ref));
|
|
128
|
+
if (e.kind === "API") out.add(fact("provides", e.ref));
|
|
141
129
|
const d = e.derived || {};
|
|
142
|
-
for (const p of d.docs || []) out.add("doc
|
|
143
|
-
for (const p of d.constraints || []) out.add("constraint
|
|
144
|
-
for (const p of d.runbooks || []) out.add("runbook
|
|
130
|
+
for (const p of d.docs || []) out.add(fact("doc", p));
|
|
131
|
+
for (const p of d.constraints || []) out.add(fact("constraint", p));
|
|
132
|
+
for (const p of d.runbooks || []) out.add(fact("runbook", p));
|
|
145
133
|
}
|
|
146
|
-
for (const
|
|
147
|
-
for (const
|
|
148
|
-
for (const
|
|
149
|
-
for (const x of graph.
|
|
150
|
-
for (const
|
|
151
|
-
for (const
|
|
152
|
-
for (const
|
|
153
|
-
for (const
|
|
154
|
-
for (const p of repoDocPaths(graph)) out.add("repodoc:" + p);
|
|
134
|
+
for (const e of graph.entities) addGraphRelations(out, graph, e.ref);
|
|
135
|
+
for (const ev of graph.interfaces.events) out.add(fact("event", ev.pattern));
|
|
136
|
+
for (const x of graph.externalDependencies) out.add(fact("external", x.name));
|
|
137
|
+
for (const x of graph.capabilities) out.add(fact("capability", x.name));
|
|
138
|
+
for (const v of graph.verification) out.add(fact("verify", v.command));
|
|
139
|
+
for (const c of graph.conflicts) out.add(fact("conflict", c.key));
|
|
140
|
+
for (const o of graph.observations) out.add(fact("observation", o.id));
|
|
141
|
+
for (const p of repoDocPaths(graph)) out.add(fact("doc", p));
|
|
155
142
|
return out;
|
|
156
143
|
}
|
|
157
144
|
|
|
158
|
-
//
|
|
159
|
-
// check in the repository, because nothing narrows it.
|
|
145
|
+
// The historical targeted reference runs every registered check and gate.
|
|
160
146
|
function baselineVerification(graph) {
|
|
161
|
-
return new Set([...graph.verification, ...graph.gates].flatMap((v) => ["verify
|
|
147
|
+
return new Set([...graph.verification, ...graph.gates].flatMap((v) => [fact("verify", v.command), fact("verify-id", v.id)]));
|
|
162
148
|
}
|
|
163
149
|
|
|
164
150
|
export function verificationRecall(retrieved, expected) {
|
|
@@ -184,13 +170,6 @@ function score(retrieved, truth) {
|
|
|
184
170
|
return { hit, total: truth.size, missed, precision: truth.size ? hit / truth.size : 1 };
|
|
185
171
|
}
|
|
186
172
|
|
|
187
|
-
function noiseRatio(retrieved, truth) {
|
|
188
|
-
if (!retrieved.size) return 0;
|
|
189
|
-
let n = 0;
|
|
190
|
-
for (const r of retrieved) if (!truth.has(r)) n++;
|
|
191
|
-
return n / retrieved.size;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
173
|
// ── Irrelevance, measured against reachability rather than against the
|
|
195
174
|
// ground truth.
|
|
196
175
|
//
|
|
@@ -223,35 +202,25 @@ export function relevantClosure(graph, touch) {
|
|
|
223
202
|
if (seeds2.has(r.targetRef)) refs.add(r.sourceRef);
|
|
224
203
|
}
|
|
225
204
|
const out = new Set();
|
|
226
|
-
for (const ref of refs) out.add(ref);
|
|
227
|
-
for (const
|
|
228
|
-
if (refs.has(r.sourceRef)) out.add(r.type + ":" + r.targetRef);
|
|
229
|
-
if (refs.has(r.targetRef)) out.add(r.type + ":" + r.sourceRef);
|
|
230
|
-
}
|
|
205
|
+
for (const ref of refs) out.add(fact("ref", ref));
|
|
206
|
+
for (const ref of refs) addGraphRelations(out, graph, ref);
|
|
231
207
|
for (const e of graph.entities) {
|
|
232
208
|
if (!refs.has(e.ref)) continue;
|
|
209
|
+
if (e.kind === "API") out.add(fact("provides", e.ref));
|
|
233
210
|
const d = e.derived || {};
|
|
234
|
-
for (const p of d.docs || []) out.add("doc
|
|
235
|
-
for (const p of d.constraints || []) out.add("constraint
|
|
236
|
-
for (const p of d.runbooks || []) out.add("runbook
|
|
211
|
+
for (const p of d.docs || []) out.add(fact("doc", p));
|
|
212
|
+
for (const p of d.constraints || []) out.add(fact("constraint", p));
|
|
213
|
+
for (const p of d.runbooks || []) out.add(fact("runbook", p));
|
|
237
214
|
for (const ev of graph.interfaces.events) {
|
|
238
215
|
if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref) || (d.sourcePath && ev.contract.ref.startsWith(d.sourcePath + "/"))) {
|
|
239
|
-
out.add("event
|
|
216
|
+
out.add(fact("event", ev.pattern));
|
|
240
217
|
}
|
|
241
218
|
}
|
|
242
|
-
for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add("external
|
|
243
|
-
for (const x of graph.capabilities) if (x.componentRef === e.ref) out.add("capability
|
|
244
|
-
for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add("verify
|
|
245
|
-
for (const c of graph.conflicts) if (c.subject === e.ref) out.add("conflict
|
|
246
|
-
for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add("observation
|
|
247
|
-
for (const r of graph.relations) {
|
|
248
|
-
if (r.type === "usesResource" && r.sourceRef === e.ref) out.add("uses:" + r.targetRef);
|
|
249
|
-
if (r.type === "usesResource" && r.targetRef === e.ref) out.add("usedBy:" + r.sourceRef);
|
|
250
|
-
if (r.type === "consumesApi" && r.sourceRef === e.ref) out.add("consumes:" + r.targetRef);
|
|
251
|
-
if (r.type === "consumesApi" && r.targetRef === e.ref) out.add("consumer:" + r.sourceRef);
|
|
252
|
-
if (r.type === "providesApi" && r.sourceRef === e.ref) out.add("provides:" + r.targetRef);
|
|
253
|
-
if (r.type === "providesApi" && r.targetRef === e.ref) out.add("provider:" + r.sourceRef);
|
|
254
|
-
}
|
|
219
|
+
for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add(fact("external", x.name));
|
|
220
|
+
for (const x of graph.capabilities) if (x.componentRef === e.ref) out.add(fact("capability", x.name));
|
|
221
|
+
for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add(fact("verify", v.command));
|
|
222
|
+
for (const c of graph.conflicts) if (c.subject === e.ref) out.add(fact("conflict", c.key));
|
|
223
|
+
for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add(fact("observation", o.id));
|
|
255
224
|
}
|
|
256
225
|
return out;
|
|
257
226
|
}
|
|
@@ -321,14 +290,24 @@ export function runRoutingEval(root, { json = false } = {}) {
|
|
|
321
290
|
const base = baselineContext(graph, s.touch);
|
|
322
291
|
const baseV = baselineVerification(graph);
|
|
323
292
|
const full = fullRepositoryContext(graph);
|
|
293
|
+
const search = searchContext(root, s.touch);
|
|
294
|
+
const searchAll = score(search.facts, new Set(s.truth));
|
|
295
|
+
// Retrieval has no graph access. Only scoring maps discovered scoped
|
|
296
|
+
// commands to the registered check IDs for a common selectivity denominator.
|
|
297
|
+
const searchIds = new Set([...graph.verification, ...graph.gates]
|
|
298
|
+
.filter((v) => search.checks.some((c) => c.command === v.command &&
|
|
299
|
+
(c.componentRef === null ? !(v.componentRefs || []).length : (v.componentRefs || []).includes(c.componentRef))))
|
|
300
|
+
.map((v) => v.id));
|
|
301
|
+
const searchV = new Set(search.checks.map((c) => fact("verify", c.command)));
|
|
302
|
+
for (const id of searchIds) searchV.add(fact("verify-id", id));
|
|
324
303
|
|
|
325
304
|
const closure = relevantClosure(graph, s.touch);
|
|
326
|
-
|
|
305
|
+
|
|
327
306
|
const all = score(retrieved, new Set(s.truth));
|
|
328
307
|
const baseAll = score(base, new Set(s.truth));
|
|
329
308
|
const crit = score(critical, new Set(s.critical));
|
|
330
309
|
const noise = irrelevanceRatio(retrieved, closure);
|
|
331
|
-
const baseNoise = irrelevanceRatio(base,
|
|
310
|
+
const baseNoise = irrelevanceRatio(base, closure);
|
|
332
311
|
|
|
333
312
|
perScenario.push({
|
|
334
313
|
id: s.id,
|
|
@@ -345,6 +324,18 @@ export function runRoutingEval(root, { json = false } = {}) {
|
|
|
345
324
|
verificationRecall: verificationRecall(baseV, new Set(s.verification)),
|
|
346
325
|
checkSelectivity: 1, // The baseline runs every verification entry and gate.
|
|
347
326
|
},
|
|
327
|
+
searchBaseline: {
|
|
328
|
+
recall: searchAll.hit / (searchAll.total || 1),
|
|
329
|
+
noise: irrelevanceRatio(search.facts, closure),
|
|
330
|
+
verificationRecall: verificationRecall(searchV, new Set(s.verification)),
|
|
331
|
+
checkSelectivity: checkSelectivity(graph, searchIds),
|
|
332
|
+
missed: searchAll.missed,
|
|
333
|
+
},
|
|
334
|
+
truthCount: all.total,
|
|
335
|
+
recalledCount: all.hit,
|
|
336
|
+
baselineRecalledCount: baseAll.hit,
|
|
337
|
+
searchRecalledCount: searchAll.hit,
|
|
338
|
+
searchCount: search.facts.size,
|
|
348
339
|
missed: all.missed.slice(0, 8),
|
|
349
340
|
offClosure: [...retrieved].filter((r) => !closure.has(r)).slice(0, 8),
|
|
350
341
|
unresolvedTouches: unresolved,
|
|
@@ -357,18 +348,31 @@ export function runRoutingEval(root, { json = false } = {}) {
|
|
|
357
348
|
|
|
358
349
|
const mean = (k) => (perScenario.length ? perScenario.reduce((a, b) => a + b[k], 0) / perScenario.length : 0);
|
|
359
350
|
const meanB = (k) => (perScenario.length ? perScenario.reduce((a, b) => a + b.baseline[k], 0) / perScenario.length : 0);
|
|
360
|
-
|
|
351
|
+
const meanS = (k) => perScenario.length ? perScenario.reduce((a, b) => a + b.searchBaseline[k], 0) / perScenario.length : 0;
|
|
352
|
+
const truthCount = perScenario.reduce((sum, s) => sum + s.truthCount, 0);
|
|
353
|
+
const micro = (key) => truthCount ? round(perScenario.reduce((sum, s) => sum + s[key], 0) / truthCount) : 0;
|
|
361
354
|
const metrics = {
|
|
362
355
|
scenarios: perScenario.length,
|
|
363
356
|
recall: round(mean("recall")),
|
|
357
|
+
microRecall: micro("recalledCount"),
|
|
364
358
|
precision: round(mean("precision")),
|
|
365
359
|
criticalRecall: round(mean("criticalRecall")),
|
|
366
360
|
verificationRecall: round(mean("verificationRecall")),
|
|
367
361
|
checkSelectivity: round(mean("checkSelectivity")),
|
|
368
362
|
noise: round(mean("noise")),
|
|
369
363
|
retrievalReduction: 0,
|
|
364
|
+
searchBaseline: {
|
|
365
|
+
recall: round(meanS("recall")),
|
|
366
|
+
microRecall: micro("searchRecalledCount"),
|
|
367
|
+
size: round(mean("searchCount")),
|
|
368
|
+
noise: round(meanS("noise")),
|
|
369
|
+
verificationRecall: round(meanS("verificationRecall")),
|
|
370
|
+
checkSelectivity: round(meanS("checkSelectivity")),
|
|
371
|
+
},
|
|
370
372
|
baseline: {
|
|
371
373
|
recall: round(meanB("recall")),
|
|
374
|
+
microRecall: micro("baselineRecalledCount"),
|
|
375
|
+
size: round(mean("baselineCount")),
|
|
372
376
|
noise: round(meanB("noise")),
|
|
373
377
|
verificationRecall: round(meanB("verificationRecall")),
|
|
374
378
|
checkSelectivity: round(meanB("checkSelectivity")),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-doc-system",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Portable documentation and software-catalog system with an explicit authority model, for AI coding agents",
|
|
6
6
|
"keywords": [
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"homepage": "https://github.com/lbframe/agent-doc-system",
|
|
19
19
|
"bugs": "https://github.com/lbframe/agent-doc-system/issues",
|
|
20
20
|
"bin": {
|
|
21
|
-
"agentdoc": "
|
|
21
|
+
"agentdoc": "bin/agentdoc.mjs"
|
|
22
22
|
},
|
|
23
23
|
"engines": {
|
|
24
24
|
"node": ">=22"
|
|
@@ -37,6 +37,8 @@
|
|
|
37
37
|
"core",
|
|
38
38
|
"docs",
|
|
39
39
|
"evals/harness.mjs",
|
|
40
|
+
"evals/facts.mjs",
|
|
41
|
+
"evals/baseline-search.mjs",
|
|
40
42
|
"evals/scenarios",
|
|
41
43
|
"evals/thresholds.json",
|
|
42
44
|
"schemas",
|
|
@@ -83,7 +83,8 @@ not work around it: run `agentdoc compile`, review the diff, commit the graph.
|
|
|
83
83
|
## Rules that are never optional
|
|
84
84
|
|
|
85
85
|
- **Never vendor the CLI.** A project gets `agentdoc/` project data (config,
|
|
86
|
-
descriptors, observations),
|
|
86
|
+
descriptors, observations), documentation skeletons at the configured
|
|
87
|
+
`docs.*` paths (`agentdoc/docs/` by default), `.agentdoc/graph.json` and a
|
|
87
88
|
CI workflow — never a copy of the agentdoc source tree (`core/`, `bin/`,
|
|
88
89
|
`adapters/`, `schemas/`, `templates/`, `evals/`, `tests/`).
|
|
89
90
|
- **Never promote uncertainty.** Heuristic guesses do not become authored
|
|
@@ -38,7 +38,12 @@ agentdoc scaffold [--write] # generate descriptors for what can be proven
|
|
|
38
38
|
subdirectory of a cataloged repository it writes at the catalog root, not the
|
|
39
39
|
current directory. `scaffold` is a dry run without `--write`; with `--write` it
|
|
40
40
|
is a superset of `init` — it also installs anything missing from the config,
|
|
41
|
-
doc skeleton and CI workflow, not only component descriptors.
|
|
41
|
+
doc skeleton and CI workflow, not only component descriptors. Both write the
|
|
42
|
+
doc skeleton to the configured `docs.*` paths when a configuration exists, and
|
|
43
|
+
to `agentdoc/docs/` when none does; an existing file is reported as skipped.
|
|
44
|
+
Authored AgentDoc material lives in `agentdoc/`, generated output in
|
|
45
|
+
`.agentdoc/`; `docs/` is left to the product's own user documentation unless a
|
|
46
|
+
configuration declares `docs.*` paths there. `audit` works
|
|
42
47
|
even when the repository cannot compile yet; it reports the compilation
|
|
43
48
|
failures as part of its output.
|
|
44
49
|
|
|
@@ -33,8 +33,14 @@ agentdoc validate && agentdoc compile && agentdoc check
|
|
|
33
33
|
|
|
34
34
|
`init` writes `agentdoc/agentdoc.config.yaml`, the central descriptor files
|
|
35
35
|
(`domains.yaml`, `systems.yaml`, `apis.yaml`, `resources.yaml`),
|
|
36
|
-
`agentdoc/journeys.yaml`, the documentation hierarchy
|
|
37
|
-
|
|
36
|
+
`agentdoc/journeys.yaml`, the documentation hierarchy, the observations
|
|
37
|
+
README, and a CI workflow. The documentation hierarchy goes to the configured
|
|
38
|
+
`docs.product`, `docs.architecture`, `docs.constraints` and `docs.adrDir` when
|
|
39
|
+
a configuration already exists, and to `agentdoc/docs/` when none does
|
|
40
|
+
(authored AgentDoc material lives in `agentdoc/`, generated output in
|
|
41
|
+
`.agentdoc/`; `docs/` stays the product's own documentation). To keep the
|
|
42
|
+
skeleton in `docs/`, declare `docs.*` in `agentdoc/agentdoc.config.yaml`
|
|
43
|
+
first. It writes **project data only** — it
|
|
38
44
|
never copies the agentdoc implementation into the repository.
|
|
39
45
|
|
|
40
46
|
In `agentdoc.config.yaml`:
|
package/templates/ci.yml
CHANGED
|
@@ -17,7 +17,7 @@ jobs:
|
|
|
17
17
|
with: { node-version: '24' }
|
|
18
18
|
- name: Install the agentdoc CLI
|
|
19
19
|
# Unpinned tracks main. For CI reproducibility pin a tag or commit:
|
|
20
|
-
# github:lbframe/agent-doc-system#v1.
|
|
20
|
+
# github:lbframe/agent-doc-system#v1.3.0
|
|
21
21
|
# A compiler upgrade intentionally invalidates committed graphs.
|
|
22
22
|
run: npm install -g github:lbframe/agent-doc-system
|
|
23
23
|
- name: Validate and compile
|