agent-doc-system 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +134 -1
  2. package/README.md +1 -1
  3. package/adapters/cache.mjs +5 -4
  4. package/adapters/database.mjs +3 -2
  5. package/adapters/deploy-config.mjs +3 -2
  6. package/adapters/dockerfile.mjs +3 -2
  7. package/adapters/generic.mjs +2 -1
  8. package/adapters/github-actions.mjs +2 -1
  9. package/adapters/go.mjs +4 -3
  10. package/adapters/node.mjs +3 -2
  11. package/adapters/rust.mjs +3 -0
  12. package/adapters/turborepo.mjs +3 -2
  13. package/adapters/typescript.mjs +7 -5
  14. package/adapters/wrangler.mjs +3 -2
  15. package/bin/agentdoc.mjs +9 -2
  16. package/bin/usage.txt +2 -0
  17. package/core/audit.mjs +7 -5
  18. package/core/compile.mjs +7 -5
  19. package/core/derive.mjs +4 -3
  20. package/core/descriptors.mjs +3 -2
  21. package/core/discover.mjs +3 -2
  22. package/core/fsx.mjs +174 -42
  23. package/core/graph.mjs +1 -1
  24. package/core/impact.mjs +11 -4
  25. package/core/paths.mjs +71 -0
  26. package/core/query.mjs +58 -23
  27. package/core/scaffold.mjs +130 -23
  28. package/core/sourcescan.mjs +4 -1
  29. package/docs/ADAPTERS.md +1 -1
  30. package/docs/BENCHMARK.md +83 -21
  31. package/docs/CI.md +7 -1
  32. package/docs/EVALS.md +209 -17
  33. package/docs/MIGRATION.md +22 -4
  34. package/docs/PROVENANCE.md +6 -0
  35. package/docs/QUERY.md +25 -3
  36. package/docs/SPEC.md +47 -2
  37. package/docs/ci/github-actions.yml +1 -1
  38. package/evals/baseline-search.mjs +272 -0
  39. package/evals/facts.mjs +28 -0
  40. package/evals/harness.mjs +120 -116
  41. package/package.json +4 -2
  42. package/skills/agent-doc-system/SKILL.md +2 -1
  43. package/skills/agent-doc-system/references/cli.md +6 -1
  44. package/skills/agent-doc-system/references/create-migrate.md +8 -2
  45. package/templates/ci.yml +1 -1
package/evals/harness.mjs CHANGED
@@ -4,14 +4,8 @@
4
4
  // an agent through it retrieve the right context, and less noise than unassisted
5
5
  // repository exploration".
6
6
  //
7
- // Two retrieval systems are measured on the same scenarios:
8
- //
9
- // baseline what an agent gets by exploring the repository the obvious way:
10
- // every documentation file, every descriptor, every source file
11
- // under the touched component. This is the honest shape of "just
12
- // read the repo", and it is what the catalog has to beat.
13
- //
14
- // catalog what `agentdoc query` returns for the same scenario.
7
+ // Compare catalog routing with a graph-assisted targeted reference and a
8
+ // working-tree search baseline. The full graph is only a size yardstick.
15
9
  //
16
10
  // Retrieval-set overlap against a per-scenario ground truth gives precision,
17
11
  // recall, critical-constraint recall, verification-command recall and noise.
@@ -28,32 +22,35 @@ import { AgentDocError, CODES } from "../core/codes.mjs";
28
22
  import { queryContext, ContextIndex } from "../core/query.mjs";
29
23
  import { impact } from "../core/impact.mjs";
30
24
 
25
+ import { fact, addRelationFacts, addGraphRelations } from "./facts.mjs";
26
+ import { searchContext } from "./baseline-search.mjs";
27
+
31
28
  const HERE = path.dirname(fileURLToPath(import.meta.url));
32
29
 
33
30
  // A retrieved item is a "fact reference" the agent would have to understand:
34
31
  // an entity ref, a constraint path, a verification command, a contract ref.
35
32
  export function factsFromContext(ctx) {
36
33
  const out = new Set();
37
- out.add(ctx.entity.ref);
38
- for (const n of ctx.neighbours) out.add(n.ref);
39
- for (const r of ctx.relations.items) out.add(r.type + ":" + r.other);
40
- for (const p of ctx.journeyPeers || []) out.add(p);
41
- for (const s2 of ctx.placement.systems) out.add(s2);
42
- for (const d2 of ctx.placement.domains) out.add(d2);
43
- for (const a of ctx.apis.provided) out.add("provides:" + a.api);
44
- for (const a of ctx.apis.consumed) out.add("consumes:" + a.api);
45
- for (const c of ctx.apis.consumers || []) out.add("consumer:" + c);
46
- if (ctx.apis.provider) out.add("provider:" + ctx.apis.provider);
47
- for (const r of ctx.resources.used) out.add("uses:" + r);
48
- for (const r of ctx.resources.usedBy) out.add("usedBy:" + r);
49
- for (const e of ctx.events.items) out.add("event:" + e.pattern);
50
- for (const e of ctx.externalDependencies.items) out.add("external:" + e.name);
51
- for (const v of ctx.verification.items) out.add("verify:" + v.command);
52
- for (const p of ctx.constraints.constraints) out.add("constraint:" + p);
53
- for (const d of ctx.constraints.docs) out.add("doc:" + d);
54
- for (const r of ctx.constraints.runbooks) out.add("runbook:" + r);
55
- for (const c of ctx.authority.conflicts.items) out.add("conflict:" + c.key);
56
- for (const o of ctx.authority.observations) out.add("observation:" + o.id);
34
+ out.add(fact("ref", ctx.entity.ref));
35
+ for (const n of ctx.neighbours) out.add(fact("ref", n.ref));
36
+ for (const r of ctx.relations.items) addRelationFacts(out, r.type, r.other, r.direction);
37
+ for (const p of ctx.journeyPeers || []) out.add(fact("ref", p));
38
+ for (const s2 of ctx.placement.systems) out.add(fact("ref", s2));
39
+ for (const d2 of ctx.placement.domains) out.add(fact("ref", d2));
40
+ for (const a of ctx.apis.provided) out.add(fact("provides", a.api));
41
+ for (const a of ctx.apis.consumed) out.add(fact("consumes", a.api));
42
+ for (const c of ctx.apis.consumers || []) out.add(fact("consumer", c));
43
+ if (ctx.apis.provider) out.add(fact("provider", ctx.apis.provider));
44
+ for (const r of ctx.resources.used) out.add(fact("uses", r));
45
+ for (const r of ctx.resources.usedBy) out.add(fact("usedBy", r));
46
+ for (const e of ctx.events.items) out.add(fact("event", e.pattern));
47
+ for (const e of ctx.externalDependencies.items) out.add(fact("external", e.name));
48
+ for (const v of ctx.verification.items) out.add(fact("verify", v.command));
49
+ for (const p of ctx.constraints.constraints) out.add(fact("constraint", p));
50
+ for (const d of ctx.constraints.docs) out.add(fact("doc", d));
51
+ for (const r of ctx.constraints.runbooks) out.add(fact("runbook", r));
52
+ for (const c of ctx.authority.conflicts.items) out.add(fact("conflict", c.key));
53
+ for (const o of ctx.authority.observations) out.add(fact("observation", o.id));
57
54
  return out;
58
55
  }
59
56
 
@@ -62,17 +59,17 @@ export function factsFromContext(ctx) {
62
59
  // resource, an API boundary, or an event subject it publishes or consumes.
63
60
  export function criticalFromContext(ctx) {
64
61
  const out = new Set();
65
- for (const e of ctx.events.items) out.add("event:" + e.pattern);
66
- for (const p of ctx.constraints.constraints) out.add("constraint:" + p);
67
- for (const c of ctx.authority.conflicts.items) out.add("conflict:" + c.key);
68
- for (const o of ctx.authority.observations) out.add("observation:" + o.id);
69
- for (const r of ctx.resources.used) out.add("uses:" + r);
70
- for (const r of ctx.resources.usedBy) out.add("usedBy:" + r);
71
- for (const a of ctx.apis.consumed) out.add("consumes:" + a.api);
72
- for (const s2 of ctx.placement.systems) out.add(s2);
73
- for (const d2 of ctx.placement.domains) out.add(d2);
74
- for (const a of ctx.apis.provided) out.add("provides:" + a.api);
75
- for (const c of ctx.apis.consumers || []) out.add("consumer:" + c);
62
+ for (const e of ctx.events.items) out.add(fact("event", e.pattern));
63
+ for (const p of ctx.constraints.constraints) out.add(fact("constraint", p));
64
+ for (const c of ctx.authority.conflicts.items) out.add(fact("conflict", c.key));
65
+ for (const o of ctx.authority.observations) out.add(fact("observation", o.id));
66
+ for (const r of ctx.resources.used) out.add(fact("uses", r));
67
+ for (const r of ctx.resources.usedBy) out.add(fact("usedBy", r));
68
+ for (const a of ctx.apis.consumed) out.add(fact("consumes", a.api));
69
+ for (const s2 of ctx.placement.systems) out.add(fact("ref", s2));
70
+ for (const d2 of ctx.placement.domains) out.add(fact("ref", d2));
71
+ for (const a of ctx.apis.provided) out.add(fact("provides", a.api));
72
+ for (const c of ctx.apis.consumers || []) out.add(fact("consumer", c));
76
73
  return out;
77
74
  }
78
75
 
@@ -80,10 +77,10 @@ export function verificationFromContext(ctx) {
80
77
  // Commands such as `go test ./...` recur in multiple modules. Scenarios can
81
78
  // require a scoped check ID as well as a command to measure dependent checks
82
79
  // without letting the library's identical command satisfy the expectation.
83
- return new Set(ctx.verification.items.flatMap((v) => ["verify:" + v.command, "verify-id:" + v.id]));
80
+ return new Set(ctx.verification.items.flatMap((v) => [fact("verify", v.command), fact("verify-id", v.id)]));
84
81
  }
85
82
 
86
- // ── baseline: unassisted repository exploration ────────────────────────
83
+ // ── baseline-targeted: graph-assisted reference ────────────────────────
87
84
  export function baselineContext(graph, touch) {
88
85
  const idx = new ContextIndex(graph);
89
86
  const seeds = touch.map((t) => idx.resolve(t)).filter(Boolean);
@@ -94,71 +91,60 @@ export function baselineContext(graph, touch) {
94
91
  const inSeed = seeds.includes(e);
95
92
  const underRoot = roots.some((r) => e.ref && r && d.sourcePath && (d.sourcePath === r || d.sourcePath.startsWith(r + "/")));
96
93
  if (!inSeed && !underRoot) continue;
97
- out.add(e.ref);
98
- for (const r of graph.relations) {
99
- if (r.sourceRef === e.ref) out.add(r.type + ":" + r.targetRef);
100
- if (r.targetRef === e.ref) out.add(r.type + ":" + r.sourceRef);
101
- }
94
+ out.add(fact("ref", e.ref));
95
+ if (e.kind === "API") out.add(fact("provides", e.ref));
96
+ addGraphRelations(out, graph, e.ref);
102
97
  for (const ev of graph.interfaces.events) {
103
- if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref)) out.add("event:" + ev.pattern);
104
- }
105
- for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add("external:" + x.name);
106
- for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add("verify:" + v.command);
107
- for (const c of graph.conflicts) if (c.subject === e.ref) out.add("conflict:" + c.key);
108
- for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add("observation:" + o.id);
109
- for (const p of d.docs || []) out.add("doc:" + p);
110
- for (const p of d.constraints || []) out.add("constraint:" + p);
111
- for (const p of d.runbooks || []) out.add("runbook:" + p);
112
- for (const r of graph.relations) {
113
- if (r.sourceRef === e.ref || r.targetRef === e.ref) out.add(r.type + ":" + (r.sourceRef === e.ref ? r.targetRef : r.sourceRef));
98
+ if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref)) out.add(fact("event", ev.pattern));
114
99
  }
100
+ for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add(fact("external", x.name));
101
+ for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add(fact("verify", v.command));
102
+ for (const c of graph.conflicts) if (c.subject === e.ref) out.add(fact("conflict", c.key));
103
+ for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add(fact("observation", o.id));
104
+ for (const p of d.docs || []) out.add(fact("doc", p));
105
+ for (const p of d.constraints || []) out.add(fact("constraint", p));
106
+ for (const p of d.runbooks || []) out.add(fact("runbook", p));
115
107
  }
116
- // A repository-wide documentation sweep is what an agent does when the
117
- // routing is not available: read everything that looks like documentation.
118
- for (const f of new Set([...repoDocPaths(graph)])) out.add("repodoc:" + f);
108
+ // Preserve the original targeted reference's repository-wide doc sweep,
109
+ // now using the same labels as query rather than unmatched repodoc labels.
110
+ for (const f of new Set([...repoDocPaths(graph)])) out.add(fact("doc", f));
119
111
  return out;
120
112
  }
121
113
 
122
114
  function repoDocPaths(graph) {
123
115
  const out = new Set();
124
116
  for (const e of graph.entities) {
125
- for (const p of (e.derived || {}).docs || []) out.add(p);
117
+ for (const p of (e.derived || {}).docs || []) out.add(fact("ref", p));
126
118
  }
127
- for (const cfgp of graph.verification) for (const p of cfgp.configPaths || []) out.add(p);
119
+ for (const cfgp of graph.verification) for (const p of cfgp.configPaths || []) out.add(fact("ref", p));
128
120
  return [...out].sort();
129
121
  }
130
122
 
131
- // Baseline verification recall: without routing, an agent runs the union of
132
- // every check in the repository (or nothing). The realistic reading of
133
- // "unassisted" is the union, since no routing exists to narrow it.
134
- // The whole-repository baseline: what an agent that has no routing ends up
135
- // holding. Every entity, every relation, every subject, every check, every
136
- // document. This is the honest cost of "just read the repository".
123
+ // Whole-graph fact inventory: a size yardstick, not unaided exploration.
137
124
  export function fullRepositoryContext(graph) {
138
125
  const out = new Set();
139
126
  for (const e of graph.entities) {
140
- out.add(e.ref);
127
+ out.add(fact("ref", e.ref));
128
+ if (e.kind === "API") out.add(fact("provides", e.ref));
141
129
  const d = e.derived || {};
142
- for (const p of d.docs || []) out.add("doc:" + p);
143
- for (const p of d.constraints || []) out.add("constraint:" + p);
144
- for (const p of d.runbooks || []) out.add("runbook:" + p);
130
+ for (const p of d.docs || []) out.add(fact("doc", p));
131
+ for (const p of d.constraints || []) out.add(fact("constraint", p));
132
+ for (const p of d.runbooks || []) out.add(fact("runbook", p));
145
133
  }
146
- for (const r of graph.relations) out.add(r.type + ":" + (r.sourceRef === undefined ? "" : r.targetRef));
147
- for (const r of graph.relations) out.add(r.type + ":" + r.sourceRef);
148
- for (const ev of graph.interfaces.events) out.add("event:" + ev.pattern);
149
- for (const x of graph.externalDependencies) out.add("external:" + x.name);
150
- for (const x of graph.capabilities) out.add("capability:" + x.name);
151
- for (const v of graph.verification) out.add("verify:" + v.command);
152
- for (const c of graph.conflicts) out.add("conflict:" + c.key);
153
- for (const o of graph.observations) out.add("observation:" + o.id);
154
- for (const p of repoDocPaths(graph)) out.add("repodoc:" + p);
134
+ for (const e of graph.entities) addGraphRelations(out, graph, e.ref);
135
+ for (const ev of graph.interfaces.events) out.add(fact("event", ev.pattern));
136
+ for (const x of graph.externalDependencies) out.add(fact("external", x.name));
137
+ for (const x of graph.capabilities) out.add(fact("capability", x.name));
138
+ for (const v of graph.verification) out.add(fact("verify", v.command));
139
+ for (const c of graph.conflicts) out.add(fact("conflict", c.key));
140
+ for (const o of graph.observations) out.add(fact("observation", o.id));
141
+ for (const p of repoDocPaths(graph)) out.add(fact("doc", p));
155
142
  return out;
156
143
  }
157
144
 
158
- // Baseline verification: without routing, an agent runs the union of every
159
- // check in the repository, because nothing narrows it.
145
+ // The historical targeted reference runs every registered check and gate.
160
146
  function baselineVerification(graph) {
161
- return new Set([...graph.verification, ...graph.gates].flatMap((v) => ["verify:" + v.command, "verify-id:" + v.id]));
147
+ return new Set([...graph.verification, ...graph.gates].flatMap((v) => [fact("verify", v.command), fact("verify-id", v.id)]));
162
148
  }
163
149
 
164
150
  export function verificationRecall(retrieved, expected) {
@@ -184,13 +170,6 @@ function score(retrieved, truth) {
184
170
  return { hit, total: truth.size, missed, precision: truth.size ? hit / truth.size : 1 };
185
171
  }
186
172
 
187
- function noiseRatio(retrieved, truth) {
188
- if (!retrieved.size) return 0;
189
- let n = 0;
190
- for (const r of retrieved) if (!truth.has(r)) n++;
191
- return n / retrieved.size;
192
- }
193
-
194
173
  // ── Irrelevance, measured against reachability rather than against the
195
174
  // ground truth.
196
175
  //
@@ -223,35 +202,25 @@ export function relevantClosure(graph, touch) {
223
202
  if (seeds2.has(r.targetRef)) refs.add(r.sourceRef);
224
203
  }
225
204
  const out = new Set();
226
- for (const ref of refs) out.add(ref);
227
- for (const r of graph.relations) {
228
- if (refs.has(r.sourceRef)) out.add(r.type + ":" + r.targetRef);
229
- if (refs.has(r.targetRef)) out.add(r.type + ":" + r.sourceRef);
230
- }
205
+ for (const ref of refs) out.add(fact("ref", ref));
206
+ for (const ref of refs) addGraphRelations(out, graph, ref);
231
207
  for (const e of graph.entities) {
232
208
  if (!refs.has(e.ref)) continue;
209
+ if (e.kind === "API") out.add(fact("provides", e.ref));
233
210
  const d = e.derived || {};
234
- for (const p of d.docs || []) out.add("doc:" + p);
235
- for (const p of d.constraints || []) out.add("constraint:" + p);
236
- for (const p of d.runbooks || []) out.add("runbook:" + p);
211
+ for (const p of d.docs || []) out.add(fact("doc", p));
212
+ for (const p of d.constraints || []) out.add(fact("constraint", p));
213
+ for (const p of d.runbooks || []) out.add(fact("runbook", p));
237
214
  for (const ev of graph.interfaces.events) {
238
215
  if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref) || (d.sourcePath && ev.contract.ref.startsWith(d.sourcePath + "/"))) {
239
- out.add("event:" + ev.pattern);
216
+ out.add(fact("event", ev.pattern));
240
217
  }
241
218
  }
242
- for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add("external:" + x.name);
243
- for (const x of graph.capabilities) if (x.componentRef === e.ref) out.add("capability:" + x.name);
244
- for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add("verify:" + v.command);
245
- for (const c of graph.conflicts) if (c.subject === e.ref) out.add("conflict:" + c.key);
246
- for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add("observation:" + o.id);
247
- for (const r of graph.relations) {
248
- if (r.type === "usesResource" && r.sourceRef === e.ref) out.add("uses:" + r.targetRef);
249
- if (r.type === "usesResource" && r.targetRef === e.ref) out.add("usedBy:" + r.sourceRef);
250
- if (r.type === "consumesApi" && r.sourceRef === e.ref) out.add("consumes:" + r.targetRef);
251
- if (r.type === "consumesApi" && r.targetRef === e.ref) out.add("consumer:" + r.sourceRef);
252
- if (r.type === "providesApi" && r.sourceRef === e.ref) out.add("provides:" + r.targetRef);
253
- if (r.type === "providesApi" && r.targetRef === e.ref) out.add("provider:" + r.sourceRef);
254
- }
219
+ for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add(fact("external", x.name));
220
+ for (const x of graph.capabilities) if (x.componentRef === e.ref) out.add(fact("capability", x.name));
221
+ for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add(fact("verify", v.command));
222
+ for (const c of graph.conflicts) if (c.subject === e.ref) out.add(fact("conflict", c.key));
223
+ for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add(fact("observation", o.id));
255
224
  }
256
225
  return out;
257
226
  }
@@ -321,14 +290,24 @@ export function runRoutingEval(root, { json = false } = {}) {
321
290
  const base = baselineContext(graph, s.touch);
322
291
  const baseV = baselineVerification(graph);
323
292
  const full = fullRepositoryContext(graph);
293
+ const search = searchContext(root, s.touch);
294
+ const searchAll = score(search.facts, new Set(s.truth));
295
+ // Retrieval has no graph access. Only scoring maps discovered scoped
296
+ // commands to the registered check IDs for a common selectivity denominator.
297
+ const searchIds = new Set([...graph.verification, ...graph.gates]
298
+ .filter((v) => search.checks.some((c) => c.command === v.command &&
299
+ (c.componentRef === null ? !(v.componentRefs || []).length : (v.componentRefs || []).includes(c.componentRef))))
300
+ .map((v) => v.id));
301
+ const searchV = new Set(search.checks.map((c) => fact("verify", c.command)));
302
+ for (const id of searchIds) searchV.add(fact("verify-id", id));
324
303
 
325
304
  const closure = relevantClosure(graph, s.touch);
326
- const baseClosure = relevantClosure(graph, s.touch);
305
+
327
306
  const all = score(retrieved, new Set(s.truth));
328
307
  const baseAll = score(base, new Set(s.truth));
329
308
  const crit = score(critical, new Set(s.critical));
330
309
  const noise = irrelevanceRatio(retrieved, closure);
331
- const baseNoise = irrelevanceRatio(base, baseClosure);
310
+ const baseNoise = irrelevanceRatio(base, closure);
332
311
 
333
312
  perScenario.push({
334
313
  id: s.id,
@@ -345,6 +324,18 @@ export function runRoutingEval(root, { json = false } = {}) {
345
324
  verificationRecall: verificationRecall(baseV, new Set(s.verification)),
346
325
  checkSelectivity: 1, // The baseline runs every verification entry and gate.
347
326
  },
327
+ searchBaseline: {
328
+ recall: searchAll.hit / (searchAll.total || 1),
329
+ noise: irrelevanceRatio(search.facts, closure),
330
+ verificationRecall: verificationRecall(searchV, new Set(s.verification)),
331
+ checkSelectivity: checkSelectivity(graph, searchIds),
332
+ missed: searchAll.missed,
333
+ },
334
+ truthCount: all.total,
335
+ recalledCount: all.hit,
336
+ baselineRecalledCount: baseAll.hit,
337
+ searchRecalledCount: searchAll.hit,
338
+ searchCount: search.facts.size,
348
339
  missed: all.missed.slice(0, 8),
349
340
  offClosure: [...retrieved].filter((r) => !closure.has(r)).slice(0, 8),
350
341
  unresolvedTouches: unresolved,
@@ -357,18 +348,31 @@ export function runRoutingEval(root, { json = false } = {}) {
357
348
 
358
349
  const mean = (k) => (perScenario.length ? perScenario.reduce((a, b) => a + b[k], 0) / perScenario.length : 0);
359
350
  const meanB = (k) => (perScenario.length ? perScenario.reduce((a, b) => a + b.baseline[k], 0) / perScenario.length : 0);
360
- void meanB;
351
+ const meanS = (k) => perScenario.length ? perScenario.reduce((a, b) => a + b.searchBaseline[k], 0) / perScenario.length : 0;
352
+ const truthCount = perScenario.reduce((sum, s) => sum + s.truthCount, 0);
353
+ const micro = (key) => truthCount ? round(perScenario.reduce((sum, s) => sum + s[key], 0) / truthCount) : 0;
361
354
  const metrics = {
362
355
  scenarios: perScenario.length,
363
356
  recall: round(mean("recall")),
357
+ microRecall: micro("recalledCount"),
364
358
  precision: round(mean("precision")),
365
359
  criticalRecall: round(mean("criticalRecall")),
366
360
  verificationRecall: round(mean("verificationRecall")),
367
361
  checkSelectivity: round(mean("checkSelectivity")),
368
362
  noise: round(mean("noise")),
369
363
  retrievalReduction: 0,
364
+ searchBaseline: {
365
+ recall: round(meanS("recall")),
366
+ microRecall: micro("searchRecalledCount"),
367
+ size: round(mean("searchCount")),
368
+ noise: round(meanS("noise")),
369
+ verificationRecall: round(meanS("verificationRecall")),
370
+ checkSelectivity: round(meanS("checkSelectivity")),
371
+ },
370
372
  baseline: {
371
373
  recall: round(meanB("recall")),
374
+ microRecall: micro("baselineRecalledCount"),
375
+ size: round(mean("baselineCount")),
372
376
  noise: round(meanB("noise")),
373
377
  verificationRecall: round(meanB("verificationRecall")),
374
378
  checkSelectivity: round(meanB("checkSelectivity")),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-doc-system",
3
- "version": "1.2.0",
3
+ "version": "1.3.0",
4
4
  "type": "module",
5
5
  "description": "Portable documentation and software-catalog system with an explicit authority model, for AI coding agents",
6
6
  "keywords": [
@@ -18,7 +18,7 @@
18
18
  "homepage": "https://github.com/lbframe/agent-doc-system",
19
19
  "bugs": "https://github.com/lbframe/agent-doc-system/issues",
20
20
  "bin": {
21
- "agentdoc": "./bin/agentdoc.mjs"
21
+ "agentdoc": "bin/agentdoc.mjs"
22
22
  },
23
23
  "engines": {
24
24
  "node": ">=22"
@@ -37,6 +37,8 @@
37
37
  "core",
38
38
  "docs",
39
39
  "evals/harness.mjs",
40
+ "evals/facts.mjs",
41
+ "evals/baseline-search.mjs",
40
42
  "evals/scenarios",
41
43
  "evals/thresholds.json",
42
44
  "schemas",
@@ -83,7 +83,8 @@ not work around it: run `agentdoc compile`, review the diff, commit the graph.
83
83
  ## Rules that are never optional
84
84
 
85
85
  - **Never vendor the CLI.** A project gets `agentdoc/` project data (config,
86
- descriptors, observations), `docs/` skeletons, `.agentdoc/graph.json` and a
86
+ descriptors, observations), documentation skeletons at the configured
87
+ `docs.*` paths (`agentdoc/docs/` by default), `.agentdoc/graph.json` and a
87
88
  CI workflow — never a copy of the agentdoc source tree (`core/`, `bin/`,
88
89
  `adapters/`, `schemas/`, `templates/`, `evals/`, `tests/`).
89
90
  - **Never promote uncertainty.** Heuristic guesses do not become authored
@@ -38,7 +38,12 @@ agentdoc scaffold [--write] # generate descriptors for what can be proven
38
38
  subdirectory of a cataloged repository it writes at the catalog root, not the
39
39
  current directory. `scaffold` is a dry run without `--write`; with `--write` it
40
40
  is a superset of `init` — it also installs anything missing from the config,
41
- doc skeleton and CI workflow, not only component descriptors. `audit` works
41
+ doc skeleton and CI workflow, not only component descriptors. Both write the
42
+ doc skeleton to the configured `docs.*` paths when a configuration exists, and
43
+ to `agentdoc/docs/` when none does; an existing file is reported as skipped.
44
+ Authored AgentDoc material lives in `agentdoc/`, generated output in
45
+ `.agentdoc/`; `docs/` is left to the product's own user documentation unless a
46
+ configuration declares `docs.*` paths there. `audit` works
42
47
  even when the repository cannot compile yet; it reports the compilation
43
48
  failures as part of its output.
44
49
 
@@ -33,8 +33,14 @@ agentdoc validate && agentdoc compile && agentdoc check
33
33
 
34
34
  `init` writes `agentdoc/agentdoc.config.yaml`, the central descriptor files
35
35
  (`domains.yaml`, `systems.yaml`, `apis.yaml`, `resources.yaml`),
36
- `agentdoc/journeys.yaml`, the documentation hierarchy under `docs/`, the
37
- observations README, and a CI workflow. It writes **project data only** — it
36
+ `agentdoc/journeys.yaml`, the documentation hierarchy, the observations
37
+ README, and a CI workflow. The documentation hierarchy goes to the configured
38
+ `docs.product`, `docs.architecture`, `docs.constraints` and `docs.adrDir` when
39
+ a configuration already exists, and to `agentdoc/docs/` when none does
40
+ (authored AgentDoc material lives in `agentdoc/`, generated output in
41
+ `.agentdoc/`; `docs/` stays the product's own documentation). To keep the
42
+ skeleton in `docs/`, declare `docs.*` in `agentdoc/agentdoc.config.yaml`
43
+ first. It writes **project data only** — it
38
44
  never copies the agentdoc implementation into the repository.
39
45
 
40
46
  In `agentdoc.config.yaml`:
package/templates/ci.yml CHANGED
@@ -17,7 +17,7 @@ jobs:
17
17
  with: { node-version: '24' }
18
18
  - name: Install the agentdoc CLI
19
19
  # Unpinned tracks main. For CI reproducibility pin a tag or commit:
20
- # github:lbframe/agent-doc-system#v1.2.0
20
+ # github:lbframe/agent-doc-system#v1.3.0
21
21
  # A compiler upgrade intentionally invalidates committed graphs.
22
22
  run: npm install -g github:lbframe/agent-doc-system
23
23
  - name: Validate and compile