agent-doc-system 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +269 -0
  2. package/README.md +38 -0
  3. package/adapters/ansible.mjs +1 -1
  4. package/adapters/cache.mjs +8 -2
  5. package/adapters/contract-consumers.mjs +1 -1
  6. package/adapters/database.mjs +18 -4
  7. package/adapters/events.mjs +1 -1
  8. package/adapters/github-actions.mjs +244 -85
  9. package/adapters/go.mjs +1 -1
  10. package/adapters/index.mjs +3 -1
  11. package/adapters/object-store.mjs +2 -2
  12. package/adapters/python.mjs +1142 -0
  13. package/adapters/resource-client.mjs +1 -1
  14. package/adapters/rust.mjs +987 -0
  15. package/adapters/typescript.mjs +1 -1
  16. package/bin/agentdoc.mjs +2 -2
  17. package/core/audit.mjs +5 -3
  18. package/core/codes.mjs +4 -0
  19. package/core/compile.mjs +124 -11
  20. package/core/dependents.mjs +42 -0
  21. package/core/discover.mjs +60 -9
  22. package/core/fsx.mjs +17 -0
  23. package/core/graph.mjs +1 -1
  24. package/core/impact.mjs +55 -44
  25. package/core/query.mjs +18 -8
  26. package/core/scaffold.mjs +87 -22
  27. package/core/sourcescan.mjs +11 -2
  28. package/core/toml.mjs +463 -140
  29. package/docs/ADAPTERS.md +196 -6
  30. package/docs/BENCHMARK.md +68 -0
  31. package/docs/CI.md +1 -1
  32. package/docs/EVALS.md +33 -2
  33. package/docs/MIGRATION.md +4 -2
  34. package/docs/QUERY.md +53 -4
  35. package/docs/SPEC.md +22 -0
  36. package/docs/ci/github-actions.yml +1 -1
  37. package/evals/harness.mjs +32 -6
  38. package/evals/scenarios/fixture-b-03-change-shared-lib.json +8 -1
  39. package/evals/scenarios/fixture-c-03-shared-vocabulary.json +7 -1
  40. package/evals/scenarios/fixture-e-01-change-shared-lib.json +14 -0
  41. package/evals/scenarios/fixture-e-02-change-manifest.json +9 -0
  42. package/evals/scenarios/fixture-e-03-local-module.json +9 -0
  43. package/evals/scenarios/fixture-e-04-ambiguous-crate.json +14 -0
  44. package/evals/scenarios/fixture-f-01-change-shared-lib.json +9 -0
  45. package/evals/scenarios/fixture-f-02-change-manifest.json +9 -0
  46. package/evals/scenarios/fixture-f-03-shadowed-import.json +9 -0
  47. package/evals/scenarios/fixture-f-04-nested-workspace.json +9 -0
  48. package/evals/scenarios/fixture-f-05-alembic-database.json +9 -0
  49. package/evals/scenarios/fixture-f-06-unittest-job.json +9 -0
  50. package/package.json +2 -1
  51. package/skills/agent-doc-system/references/create-migrate.md +13 -3
  52. package/templates/ci.yml +1 -1
package/docs/ADAPTERS.md CHANGED
@@ -44,9 +44,17 @@ Available on `ctx`:
44
44
  | `productionDependencies(ref)` | production dependency names, for client-of-resource rules |
45
45
  | `deferSecondPass(fn)` | run after every unit is known |
46
46
  | `requirements` | declared normative requirements (`subject`, `key`, `value`, `source`, `satisfiedBy`, `expects`) an adapter may derive the implementation side of |
47
- | `unitByPkgName`, `unitByGoModule` | cross-unit resolution built before extraction |
47
+ | `unitByPkgName`, `unitByGoModule`, `unitByCrateName` | cross-unit resolution built before extraction |
48
+ | `unitByPythonDist`, `unitByPythonImport` | Python distribution names (PEP 503) and import names, built before extraction |
48
49
  | `errors.push({ code, message })` | a hard error, collected rather than thrown |
49
50
 
51
+ For per-unit source inventories, use `sourceFiles(repo, unit.root)` or
52
+ `repo.walkUnit(unit.root, directory = unit.root)`. Discovery computes ownership
53
+ once from eligible units: these scans prune nested unit roots, so a file belongs
54
+ only to its deepest enclosing unit. `repo.ownsUnitPath(unit.root, path)` applies
55
+ the same boundary to custom traversals. Keep `repo.walk` for repository-wide
56
+ discovery and authored inputs; explicit cross-unit references can still be read.
57
+
50
58
  ## The three rules
51
59
 
52
60
  ### 1. An adapter may not know the repository
@@ -90,6 +98,8 @@ cannot name the file a fact came from, you cannot add the fact.
90
98
  | `turborepo` | `turbo.json` (nearest ancestor) | per-package task commands |
91
99
  | `typescript` | `tsconfig.json` | language, project references, file-router routes |
92
100
  | `go` | `go.mod`, `go.work` | module roots, module graph, `go test`/`go vet` |
101
+ | `rust` | `Cargo.toml` (`[package]`), workspace `members` (including a workspace below the repository root) | crate roots, path, workspace, and `[patch]`/`[replace]` dependency edges, `cargo test`/`cargo clippy` |
102
+ | `python` | `pyproject.toml`, `setup.py`, `setup.cfg`, `requirements*.txt` | distribution graph, import edges, evidence-backed test and static commands |
93
103
  | `openapi` | `*.openapi.y[a]ml` | operation inventory, deterministic consumers |
94
104
  | `protobuf` / `graphql` | contract files | method/type inventory, consumers |
95
105
  | `oidc` | discovery-path literals | OIDC client consumers |
@@ -106,6 +116,53 @@ cannot name the file a fact came from, you cannot add the fact.
106
116
  `generic` is always on. Everything else is opt-in through `adapters: [...]`, and
107
117
  an unknown name is `AGENTDOC_CONFIG`, not a silent no-op.
108
118
 
119
+ ### `github-actions`
120
+
121
+ Recognizable test, lint and build invocations in `run` steps contribute
122
+ checks. The executable and task determine the tier; a shell `test`, a step
123
+ name or text printed by `echo` does not. Turbo (including package-manager
124
+ wrappers), Nx (`run`, `run-many`, `affected`), Make and Just tasks are resolved
125
+ by task name. Multi-task commands contribute a check for each distinct tier.
126
+ Repository scripts invoked directly (`./scripts/test.sh`) or through `bash`,
127
+ `sh`, `node`, `python` or `python3` also contribute checks when their basename,
128
+ without its extension, matches the task vocabulary (`test`, `lint`, `check`,
129
+ `typecheck`, `build`, `e2e`, `integration`, `contract`, etc.). The script must
130
+ resolve to a literal file inside the repository, relative to the current
131
+ step directory. Its path and arguments use the same scope rules as other
132
+ invocations; without component scope it is a repository gate. Scripts with
133
+ other names, such as `deploy.sh`, are ignored. Script contents are not executed
134
+ or interpreted.
135
+ Simple scripts are split into individual commands, never copied wholesale or
136
+ truncated. A literal `cd` updates the directory for following commands in the
137
+ same step, including across lines. Shell control flow, pipelines,
138
+ substitutions and heredocs are not interpreted; a non-literal `cd`, directory
139
+ stack operation, `source`, `eval` or `export` stops extraction for that step.
140
+
141
+ `uses` actions whose names contain lint, test, check, scan or audit also
142
+ contribute checks. Setup, checkout, cache, upload, download and artifact
143
+ actions are excluded. Action scope comes from `with.working-directory`,
144
+ `with.working_directory` or `with.path`; run-step defaults do not apply. The
145
+ action reference is recorded as its command, with the workflow as its evidence.
146
+
147
+ Scope comes from the step's `working-directory` (including job and workflow
148
+ defaults), a component path argument, an explicit package selector, or a
149
+ workflow trigger's `paths` filter with a literal directory prefix. Paths match
150
+ whole segments; a mention in another step cannot scope a check. Turbo filters
151
+ match literal package names or directory paths; Nx project selectors match a
152
+ package name, `project.json` name or component root; Make's `-C` resolves a
153
+ directory relative to the step's working directory. Unscoped checks are
154
+ repository gates. A working directory is retained in the recorded
155
+ command so the check can be run from the repository root.
156
+
157
+ `impact` on a workflow follows its recorded check configuration paths to the
158
+ covered components, which seed the same reverse build/test dependency closure
159
+ as directly changed components. Their own checks precede their dependents'
160
+ checks, with test-only routes contributing only unit checks. Repository gates
161
+ from that file appear in `affectedGates` and are appended after component
162
+ checks in `recommendedVerification`. The combined list is capped at 25;
163
+ `recommendedVerificationTotal` and `recommendedVerificationTruncated` include
164
+ gates. A workflow matched by a check or gate is removed from `unmatchedFiles`.
165
+
109
166
  ### `ansible`
110
167
 
111
168
  Reads playbooks under a unit root and follows what they reach — plays,
@@ -139,6 +196,122 @@ candidate, which is never elected — the requirement stays
139
196
  `AGENTDOC_REQUIREMENT_UNVERIFIED` and the diagnostic names what was opaque.
140
197
  Inventory files, `group_vars`/`host_vars` and variables are not read.
141
198
 
199
+ ### `rust`
200
+
201
+ Reads `Cargo.toml`. A `[package]` is a unit. A virtual workspace manifest (only
202
+ `[workspace]`) is not. Nothing is executed, and `Cargo.lock` is not read.
203
+
204
+ | it derives | from |
205
+ |---|---|
206
+ | `languages: rust`, `runtimes: rust` | the package manifest |
207
+ | deployable | `[[bin]]`, `src/main.rs`, or a `src/bin/` directory |
208
+ | importable | `[lib]` or `src/lib.rs`; both shapes when both are present |
209
+ | `buildDependsOn` | `[dependencies]`, `[build-dependencies]`, and `[target.<cfg>.dependencies]` / `[target.<cfg>.build-dependencies]`, when the entry is a `path`, `workspace = true` resolved through the nearest `[workspace.dependencies]`, or a `[patch.<registry>]` / `[replace]` path redirect to a sibling inside the repository. A production `use`, `extern crate`, or `<crate>::` path (`-` and `_` are the same ident) adds provenance to that edge; it never creates one |
210
+ | `testDependsOn` | `[dev-dependencies]` and target-specific dev-dependencies, resolved the same way |
211
+ | `hasTests` | a `tests/` directory, or `#[test]` / `#[cfg(test)]` in a `.rs` file |
212
+ | verification | `cargo test -p <name>` (unit) and `cargo clippy -p <name>` (static) |
213
+
214
+ `-p <name>` is the whole scope of the recorded command. Cargo, run at a virtual
215
+ workspace root, has no current package; `-p` is what makes the command mean
216
+ this crate rather than every member. `configPaths` is that crate's
217
+ `Cargo.toml`.
218
+
219
+ Workspace roots come from every `Cargo.toml` that has `[workspace]`, not only
220
+ the one at the repository root. The search is bounded to five directories
221
+ down, stays inside the repository, and does not enter `discovery.notUnitRoots`
222
+ or their descendants (the authored-descriptor exception is described below),
223
+ `discovery.generatedDirs`, dot-directories, or `test` / `tests` / `examples` /
224
+ `example`. `members` minus `exclude` are still relative to the manifest that
225
+ declares them; `*` matches one path segment, and a member that leaves the
226
+ repository is ignored. `workspace = true` resolves against the nearest
227
+ enclosing workspace. A virtual manifest (no `[package]`) is not a unit and is
228
+ not scaffolded. A manifest that has both `[package]` and `[workspace]` is a
229
+ unit, the same as any other package. Scaffold types a crate from its shape: deployable-only
230
+ (`[[bin]]`, `src/main.rs`, `src/bin/`) is `service`, importable-only is
231
+ `library`, and a crate that is both follows the usual rule (deployable wins,
232
+ so `service`).
233
+
234
+ The manifest route and a corroborating import share the target crate's
235
+ package name as their edge key, so one dependency is one edge. A rename
236
+ (`package = "…"`) is the crates.io name; the `use` path is the table key.
237
+ `[patch.<registry>]` and `[replace]` contribute only when the value is a
238
+ `path` that stays inside the repository and the package is already a
239
+ dependency. A path and a patch that name two different siblings is
240
+ `AGENTDOC_BINDING_AMBIGUOUS` and no edge is created.
241
+
242
+ **What it cannot see, and therefore never invents:** a dependency that exists
243
+ only as a `use`, `extern crate`, or `<crate>::` path. Rust will not link a
244
+ crate the manifest does not declare, and a local module (`mod engine;
245
+ engine::run()`) is the same spelling as a crate path, so an import that the
246
+ manifest did not already resolve creates no edge. An import whose ident
247
+ matches two sibling crates (`foo-bar` and `foo_bar` are both `foo_bar`) is
248
+ `AGENTDOC_BINDING_AMBIGUOUS` and still creates no edge. Also invisible: a
249
+ `Cargo.lock` resolution or any version requirement that merely shares a name
250
+ with a sibling; a workspace member outside the repository; a workspace
251
+ manifest deeper than the search bound; a `#[cfg(test)]` or `#[test]` item,
252
+ which is not a production import; a duplicate package name, which is not
253
+ elected either. A `Cargo.toml` this TOML reader rejects yields
254
+ `AGENTDOC_YAML_PARSE` and no dependency edges. The unit is still recognised when a `[package]` header is present, so
255
+ a parse failure is not an empty unit and not a crash. Char literals and
256
+ lifetimes are not strings: a lifetime does not hide the imports after it, and
257
+ a `"` string that spans lines does not leak the idents inside it.
258
+
259
+ ### `python`
260
+
261
+ Reads a unit's Python manifest and its `import` / `from` statements. It does
262
+ not run Python, resolve a lockfile, or inspect a virtualenv.
263
+
264
+ | it derives | from |
265
+ |---|---|
266
+ | deployable | `[project.scripts]`, `[project.gui-scripts]`, `[tool.poetry.scripts]`, a `console_scripts` entry point, `__main__.py`, `manage.py`, or an ASGI/WSGI app object (`app` / `application` assigned from `FastAPI`, `Flask`, `Starlette`, `Litestar`, `Sanic` or `Quart`, or a call to `get_asgi_application` / `get_wsgi_application`) |
267
+ | importable | a build backend (`[build-system] build-backend`, or `setup.py` / `setup.cfg`) and a package |
268
+ | the import name | an explicit package list (`[tool.setuptools] packages`, `[tool.poetry] packages`, `[tool.hatch.build.targets.wheel] packages`); otherwise the distribution name with `-` turned into `_` when that directory exists; otherwise the only package directory |
269
+ | `buildDependsOn` | `[project] dependencies`, `[tool.uv.sources]` (`path`, or `workspace = true` resolved against the nearest enclosing uv workspace), Poetry `path`, `-e <dir>` in a production `requirements*.txt`, or a production `import` / `from` of a sibling's import name |
270
+ | `testDependsOn` | `[dependency-groups]`, `[project.optional-dependencies]` groups named dev/test/lint/type, `[tool.poetry.group.*]`, or a requirements file whose name contains `dev` or `test` |
271
+ | `languages` / `runtimes` | the manifest. `requiresPython` only when `requires-python`, `python_requires`, or Poetry's `python` dependency is written down |
272
+ | `pytest` | a pytest dependency or group, `[tool.pytest.ini_options]`, `pytest.ini`, or `conftest.py` |
273
+ | `python -m unittest discover` | test modules that import `unittest`, and only when nothing above proves pytest |
274
+ | `ruff check` / `mypy` | the tool declared as a dependency or configured (`[tool.ruff]`, `ruff.toml`, `[tool.mypy]`, `mypy.ini`) |
275
+
276
+ `uv run` or `poetry run` is prefixed only when that lockfile contains a
277
+ `name = "<tool>"` entry. Both lockfiles proving the same tool is ambiguous:
278
+ the command is left unprefixed. PEP 508 extras, versions and markers are
279
+ parsed and not evaluated. Two import packages and no explicit list, or two
280
+ units publishing the same distribution or import name, produce
281
+ `AGENTDOC_BINDING_AMBIGUOUS` and no cross-unit link.
282
+
283
+ An import under `if TYPE_CHECKING:` is still an import. The adapter does not
284
+ evaluate control flow, so the name is classified like any other import:
285
+ `buildDependsOn` when it resolves to a sibling, or `testDependsOn` when the
286
+ manifest puts that distribution in a dev group. It is not a separate edge.
287
+
288
+ A uv workspace manifest is read at the repository root and below it, to a
289
+ fixed depth, in path order, staying inside the repository. `notUnitRoots`, their
290
+ descendants, and generated directories are not searched; the authored-descriptor
291
+ exception is described below. `workspace = true` names the member of
292
+ the nearest enclosing workspace whose distribution name matches; another unit
293
+ with the same name is not substituted. Every declared member that has a Python
294
+ manifest is a unit, including one with no build backend and no entrypoint.
295
+ Its shape is still only the evidence, so scaffold writes `other` when the
296
+ manifest is neither deployable nor importable. A manifest that only declares
297
+ the workspace, with no distribution of its own, is not a unit.
298
+
299
+ A `.py` file uses the shared source inventory and is indexed for the deepest
300
+ eligible unit that contains it. Package, entrypoint and test discovery use the
301
+ same ownership boundaries; workspace-member discovery remains repository-wide. A
302
+ parent unit does not gain an import edge from a file that lives in a nested
303
+ member.
304
+
305
+ An import is not evidence of a sibling when the importing unit itself provides
306
+ that top-level module or package — at the unit root, under `src/`, or in the
307
+ importing file's own directory. If a sibling publishes the same import name,
308
+ the result is `AGENTDOC_BINDING_AMBIGUOUS` and no edge.
309
+
310
+ **What it cannot see:** a namespace package with no `__init__.py`; a
311
+ `setup.py` dependency that is not a list of string literals; a `requirements`
312
+ file pulled in with `-r` / `-c`; a lockfile's resolved versions; a PDM
313
+ workspace; a workspace manifest deeper than the search bound.
314
+
142
315
  `asyncapi` is a contract **type** (declared on an API descriptor, validated by
143
316
  the contract layer), not an adapter — there is no `asyncapi` entry in
144
317
  `adapters: [...]`.
@@ -186,16 +359,33 @@ vocabulary. It does, however, know the shape of a manifest filename in three
186
359
  places, and those are the honest exceptions worth naming:
187
360
 
188
361
  - `core/compile.mjs` builds a manifest pre-pass over `package.json`, `go.mod`
189
- and `go.work`, so `unitByPkgName`, `unitByGoModule` and
362
+ and `go.work`, and, when the `rust` adapter is enabled, over `Cargo.toml`.
363
+ `unitByPkgName`, `unitByGoModule`, `unitByCrateName` and
190
364
  `transitiveDependencies` are available to every adapter regardless of
191
- execution order.
365
+ execution order. Cargo is probed only when `rust` is enabled: an existence
366
+ probe is part of the input hash, and probing `Cargo.toml` in every unit
367
+ would change every graph that does not ask for it.
368
+ - The same pre-pass reads a Python manifest (`pyproject.toml`, otherwise
369
+ `setup.cfg` / `setup.py` / `requirements*.txt`) into `unitByPythonDist` and
370
+ `unitByPythonImport`, and feeds those siblings into `dependencyMap` and
371
+ `productionDependencies`, only when the `python` adapter is enabled. Each
372
+ unit root is then listed so `requirements*.txt` is visible; that listing is
373
+ part of the input hash. A repository that does not enable `python` is not
374
+ probed, and its graph does not change.
192
375
  - `core/impact.mjs` classifies a changed file by consulting the compiled graph —
193
376
  its contracts, its provenance paths, its extractor identities — rather than a
194
377
  hand-written list of technology filenames. A list would silently under-report
195
378
  for every repository that is not Node-and-Go.
196
- - `core/discover.mjs` has a default list of directories that are never units
197
- (`agentdoc`, `docs`, `scripts`, `.github`), overridable through
198
- `discovery.notUnitRoots`.
379
+ - `core/discover.mjs` excludes `agentdoc`, `docs`, `scripts`, `node_modules`, and
380
+ `.github` by default. `discovery.notUnitRoots` replaces that list (`[]`
381
+ disables it). These roots and their descendants cannot be inferred from
382
+ adapter markers, adapter `roots()`, or supplemental roots, and scaffold never
383
+ proposes or writes descriptors there in dry run or `--write`. An existing
384
+ authored descriptor strictly below an excluded root still claims a unit and
385
+ is preserved; the adapter must corroborate its artifact type at that unit's
386
+ own root as usual. A descriptor exactly at an excluded root is refused with
387
+ `AGENTDOC_COMPONENT_UNCORROBORATED` naming `notUnitRoots` and asking the author
388
+ to remove the exclusion or move the descriptor.
199
389
 
200
390
  Each is a place a future change should push outward, not a licence to add more.
201
391
 
@@ -0,0 +1,68 @@
1
+ # BENCHMARK
2
+
3
+ Two measurements back the numbers in the README. Both are reproducible from a
4
+ checkout. Neither involves a model, so phrasing cannot move them.
5
+
6
+ ## Routing quality: `node evals/run-all.mjs`
7
+
8
+ The routing evaluation is described in [EVALS.md](EVALS.md). It has 30 scenarios
9
+ on six repositories (Node, TypeScript, Go, Rust and Python), with thresholds
10
+ registered on 2026-09-28 and not moved since. For each scenario, it compares
11
+ the context `agentdoc query` + `agentdoc impact` returns with what an agent
12
+ gets by reading only the component it is changing.
13
+
14
+ | | reading the touched component | AgentDoc |
15
+ |---|---|---|
16
+ | facts the change needs, retrieved | 58% | **100%** |
17
+ | retrieved facts that are unrelated (noise) | 40% | **8.7%** |
18
+ | share of the repository's checks returned (selectivity) | 100% | **46%** (29% on the golden corpus) |
19
+ | context size vs. reading the whole catalog | — | **−70%** |
20
+
21
+ Every check a scenario requires is in AgentDoc's answer (verification recall
22
+ 1.0). The baseline never misses one either, because it runs every check in the
23
+ repository. Selectivity is therefore the meaningful comparison: the share of
24
+ the repository's checks the answer contains, counting everything the agent is
25
+ shown. It is reported, not gated; small corpora inflate it, and a
26
+ one-component corpus scores 100% because there is nothing to narrow.
27
+
28
+ The size reduction is measured on the golden corpus (`examples/koda`). The
29
+ smaller fixtures hold fewer than 100 facts, so the harness reports their
30
+ reduction but does not aggregate it.
31
+
32
+ ## Fresh setup on public repositories: `node scripts/bench/run.mjs`
33
+
34
+ [`scripts/bench/run.mjs`](../scripts/bench/run.mjs) clones each repository in
35
+ [`repos.json`](../scripts/bench/repos.json) at a pinned commit, then does the
36
+ following:
37
+
38
+ 1. Runs `agentdoc init` and `agentdoc scaffold --write`.
39
+ 2. Fills in the one line AgentDoc refuses to decide: each component's placement.
40
+ 3. Counts every other edit a human would need before the catalog validates. That
41
+ number is the "setup ok" column, and the target is zero.
42
+ 4. Compiles three times, and checks that the graph is byte-identical.
43
+ 5. For each component with a matching file, samples the first tracked source
44
+ file whose path does not contain `test` (benchmarks and examples may be
45
+ included). It measures the `agentdoc query` answer, at about 4 characters per
46
+ token, and the `agentdoc impact` result.
47
+
48
+ Measured with AgentDoc 1.2.0 on Node 24, on a 2019 Intel Core i9 laptop (load average about 4):
49
+
50
+ | repository | files | setup ok | components | dependency edges | checks | init + scaffold + median compile | median compile | deterministic | `query` answer, median / max |
51
+ |---|---|---|---|---|---|---|---|---|---|
52
+ | [tokio-rs/axum](https://github.com/tokio-rs/axum) @ 86aa615 (Rust) | 505 | yes | 4 | 11 | 9 | 1.0 s | 0.5 s | yes | ~151 / 153 tokens |
53
+ | [pydantic/pydantic-ai](https://github.com/pydantic/pydantic-ai) @ 5a4d576 (Python, TS) | 5,142 | yes | 8 | 9 | 5 | 4.6 s | 1.9 s | yes | ~112 / 144 tokens |
54
+ | [fastapi/full-stack-fastapi-template](https://github.com/fastapi/full-stack-fastapi-template) @ f4bcfab (Python, TS) | 252 | yes | 3 | 0 | 9 | 1.0 s | 0.4 s | yes | ~100 / 126 tokens |
55
+ | [t3-oss/create-t3-turbo](https://github.com/t3-oss/create-t3-turbo) @ 8f945b7 (TypeScript) | 141 | yes | 13 | 47 | 25 | 1.0 s | 0.4 s | yes | ~158 / 263 tokens |
56
+
57
+ Timings depend on the machine and its load, so run the script yourself. The
58
+ other columns are deterministic for a given AgentDoc version and commit.
59
+
60
+ ### What these numbers do not say
61
+
62
+ - A `query` answer is a map of what to read, what the change reaches, and which
63
+ checks to run. It does not replace the code, so its size is not a "tokens
64
+ saved" figure.
65
+ - "Checks" counts the commands AgentDoc can prove from the repository. For
66
+ example, pydantic-ai's tests live outside the package roots, so AgentDoc
67
+ records none for those packages rather than guess one.
68
+ - Four repositories are a smoke test of the setup path, not a survey.
package/docs/CI.md CHANGED
@@ -20,7 +20,7 @@ jobs:
20
20
  with: { fetch-depth: 0 }
21
21
  - uses: actions/setup-node@v4
22
22
  with: { node-version: '24' }
23
- # Pin a tag for reproducibility, e.g. github:lbframe/agent-doc-system#v1.1.0 —
23
+ # Pin a tag for reproducibility, e.g. github:lbframe/agent-doc-system#v1.2.0 —
24
24
  # a compiler upgrade intentionally invalidates every committed graph.
25
25
  - run: npm install -g github:lbframe/agent-doc-system
26
26
  - run: agentdoc validate
package/docs/EVALS.md CHANGED
@@ -30,11 +30,35 @@ is deterministic and cannot be moved by phrasing.
30
30
  |---|---|
31
31
  | recall | fraction of the scenario's ground-truth facts the catalog retrieved |
32
32
  | critical recall | the same, restricted to critical context: constraints, conflicts, observations, resources, API boundaries, event subjects |
33
- | verification recall | fraction of the checks that must be run, that were returned |
33
+ | verification recall | fraction of the scenario's verification expectations returned; 1 when there are no expectations, for both catalog and baseline |
34
+ | check selectivity | distinct recommended check IDs / distinct check IDs in the repository graph (`verification` + `gates`); reported only, not gated; lower means fewer checks recommended |
34
35
  | precision | `1 - irrelevance` |
35
36
  | irrelevance (noise) | fraction of retrieved facts outside the graph's one-hop closure of the touched entities |
36
37
  | retrieval reduction | `1 - catalog facts / baseline-full facts` |
37
38
 
39
+ Scenario verification expectations accept `verify:<command>` and
40
+ `verify-id:<check-id>`. Use check IDs when multiple components have the same
41
+ command (for example `go test ./...`), so the library's own check cannot stand in
42
+ for a dependent's. Retrieved context includes the entity refs already returned
43
+ in query's neighbour summaries, even when those neighbours are dependencies and
44
+ therefore absent from impact's affected components.
45
+
46
+ Check selectivity counts the union of checks returned by queries for touched
47
+ and impact-affected entities (`verification.items` and `globalGates`), plus
48
+ impact's `recommendedVerification`. It measures the returned lists, respecting
49
+ their output caps. IDs deduplicate repeated recommendations while preserving
50
+ separate scoped checks that happen to share a command. With no checks in the
51
+ graph, selectivity is 1 (there is nothing to narrow). Corpus values are scenario
52
+ means; the aggregate is scenario-weighted across corpora. Both CLI reports
53
+ include per-scenario values in their `--json` output.
54
+
55
+ Baseline verification recall is 1 by construction: its verification strategy
56
+ runs every graph verification entry and repository gate, so it cannot miss a
57
+ registered expectation. Empty expectations also score 1 on both sides. Its
58
+ check selectivity is therefore 1. Selectivity is the meaningful comparison of
59
+ how much routing narrows the check set, read alongside catalog verification
60
+ recall so that omitting a required check is not mistaken for an improvement.
61
+
38
62
  ### Why noise is measured against reachability, not against the truth set
39
63
 
40
64
  The first definition of noise was the complement of truth recall. It is
@@ -101,6 +125,8 @@ Two additional gates use the same numbers:
101
125
  | `fixtures/a-node-ts-postgres` | 3 | single Node/TypeScript service, REST, Postgres, GitHub Actions |
102
126
  | `fixtures/b-go-multi-service` | 4 | multi-service Go, protobuf contracts, shared library, resources |
103
127
  | `fixtures/c-alt-monorepo` | 4 | a layout that is deliberately not `apps`+`packages`+`workers`, mixed runtimes, inconsistent pre-existing documentation |
128
+ | `fixtures/e-rust-workspace` | 4 | Cargo workspace, shared crate and its dependent, member manifest, local module, ambiguous crate names |
129
+ | `fixtures/f-python-workspace` | 6 | uv workspace, shared library and its dependent, member manifest, shadowed import, nested workspace, Alembic database, unittest |
104
130
 
105
131
  Scenarios are repository-scoped: a scenario only runs against the corpus it was
106
132
  written for, declared by a `repository` tag in a `.agentdoc-fixture.json` at the
@@ -117,7 +143,12 @@ cron; debug a production mismatch.
117
143
  The synthetic fixtures add: change an API contract; change a write path; change
118
144
  the only-writer rule; break an end-to-end journey; change an edge worker's flush
119
145
  schedule; follow stale documentation while debugging; change a subject vocabulary
120
- shared across runtimes; add an operation to a contract.
146
+ shared across runtimes; add an operation to a contract. The Rust workspace adds
147
+ a shared crate, the manifest that declares its dependents, a local module that
148
+ shares a name with another crate, and a hyphenated crate beside its underscored
149
+ twin. The Python workspace adds a shared library, a service manifest, an import
150
+ shadowed by a local module, a nested workspace member, an Alembic migration
151
+ bound to its database, and a job whose check is `python -m unittest discover`.
121
152
 
122
153
  ## Running it
123
154
 
package/docs/MIGRATION.md CHANGED
@@ -204,8 +204,10 @@ Then, in order:
204
204
  documentation, fix the code, or record an authority rule saying which class
205
205
  wins for that key.
206
206
  3. **Orphan documentation.** `documentation.orphanDocs`. For each: link it to a
207
- component via `context.docs`, or delete it. An orphan is a document nothing
208
- will ever route to.
207
+ Component, Resource or System via `context.docs`, `context.constraints` or
208
+ `context.runbooks`, or delete it. Conventional component documents and
209
+ configured repository-wide roles also count as routed. An orphan is a
210
+ document nothing will ever route to.
209
211
  4. **Missing documentation.** `documentation.undocumentedComponents` and
210
212
  `documentation.unverifiedComponents`. These are the components an agent will
211
213
  get wrong.
package/docs/QUERY.md CHANGED
@@ -84,6 +84,17 @@ withheld.
84
84
  endpoint is proved by running their suites, so `verification` for an API subject
85
85
  is the union over the provider and its consumers.
86
86
 
87
+ **A component change carries its dependents' checks.** Verification follows the
88
+ reverse `buildDependsOn` / `testDependsOn` closure (including their inverse
89
+ `buildDependencyOf` / `testDependencyOf` spellings). Build dependents contribute
90
+ all tiers; a route containing a test dependency contributes only `unit` checks.
91
+ If an all-build route also reaches that dependent, all tiers apply. Cycles are
92
+ visited safely. The subject's own checks come first, then the dependents', with
93
+ each group sorted by check ID. A check shared by multiple components appears
94
+ once by ID; identical command text in different component checks remains scoped
95
+ by its ID and `configPaths`. The usual verification budget and truncation fields
96
+ apply to the combined list.
97
+
87
98
  **Journey peers are named.** If a component appears in a journey, the other
88
99
  components in that journey are in scope and are listed explicitly rather than
89
100
  left for the agent to rediscover.
@@ -127,9 +138,25 @@ the slice is never mistaken for the whole.
127
138
  | assertions | 25 |
128
139
  | provenance | 20 |
129
140
  | contracts | 10 |
141
+ | docs (markdown) | 12 |
142
+ | constraints (markdown) | uncapped |
143
+ | runbooks (markdown) | 12 |
130
144
 
131
145
  ## Markdown rendering
132
146
 
147
+ Component, Resource and System descriptors may carry `spec.context`. Their
148
+ documentation, constraints and runbooks appear under `docs`, `CONSTRAINTS` and
149
+ `runbooks`, respectively. Components also include conventional files discovered
150
+ in their source root. API and Domain schemas do not allow `context`.
151
+
152
+ Markdown docs and runbooks default to 12 paths each. Constraints are uncapped by
153
+ default so critical routing context is preserved. Set an explicit cap with, for
154
+ example, `--budget docs=1,constraints=5,runbooks=1`. A shortened line ends with
155
+ `[truncated: 1 of 2]`, reporting the number shown and the total. JSON retains the
156
+ complete arrays under `constraints.docs`, `constraints.constraints` and
157
+ `constraints.runbooks`; these markdown-only caps do not shorten the JSON arrays
158
+ or `entity.spec.context`.
159
+
133
160
  `--md` (the default) produces a compact block suitable for a prompt or a human:
134
161
 
135
162
  ```
@@ -164,10 +191,32 @@ contracts, documentation, resources, verification and catalog sources; whether
164
191
  the graph will be invalidated; the recommended verification commands; any open
165
192
  conflicts on the affected subjects; and related journeys.
166
193
 
167
- Reachability rules: a contract change reaches its provider and every consumer; a
168
- component change reaches its transitive dependents two hops out, and everything
169
- connected to it by a runtime call or a schedule. Two hops is deliberate — a
170
- third hop is where the graph stops being a routing aid and becomes a dump.
194
+ Reachability rules: a contract change reaches its provider and every consumer;
195
+ a Resource change reaches components bound to that exact Resource. A migration
196
+ change (including Alembic `alembic/versions`) reports its changed path and the
197
+ logical databases bound to its enclosing component through existing
198
+ `usesResource` / `resourceUsedBy` relations. Migration authority alone never
199
+ establishes ownership or invents a Resource binding.
200
+
201
+ A component change reaches its reverse build/test dependency closure, plus
202
+ direct runtime-call and schedule peers. Dependencies are not affected just
203
+ because the changed component uses them; they remain explicitly typed context
204
+ in query's `relations`. The dependent closure has no hop cutoff, so a long chain
205
+ cannot silently omit an affected consumer.
206
+
207
+ Changing a workflow file follows recorded verification `configPaths` to the
208
+ components its checks cover. Those components seed the reverse build/test
209
+ dependency closure just like directly changed components. Matching repository
210
+ gates appear in `affectedGates`; a workflow matched by any check or gate is
211
+ removed from `unmatchedFiles`.
212
+
213
+ Impact uses the same verification selection and ordering as query, including
214
+ the test-only tier restriction. Directly affected components' checks precede
215
+ their dependents' checks, followed by the matching repository gates. The
216
+ combined `recommendedVerification` array is capped at 25;
217
+ `recommendedVerificationTotal` gives the uncapped count including gates and
218
+ `recommendedVerificationTruncated` reports overflow. Query's default cap is 12
219
+ and can be adjusted with `--budget verification=...`.
171
220
 
172
221
  ## Using it in a prompt
173
222
 
package/docs/SPEC.md CHANGED
@@ -237,6 +237,28 @@ The core has no knowledge of folder names. A directory is a unit when it holds
237
237
  a component descriptor, when an adapter recognises it from a workspace file, or
238
238
  when it is listed in `discovery.supplementalRoots`.
239
239
 
240
+ `init` configures the recursive component descriptor glob `*/**/agentdoc.yaml`,
241
+ so descriptors for adapter-discovered units and units added later are loaded
242
+ without regenerating a path list. The scaffold layout scan supplements its
243
+ shallow marker heuristics with adapter discovery: declared workspace members
244
+ under `examples`, `example`, `test`, or `tests`, or beyond the scan depth, remain
245
+ units. Those names alone do not create units. Scaffold selects artifact types
246
+ from adapter shape: deployable (including both shapes) → `service`, importable
247
+ only → `library`, neither → `other`. Placement remains a human choice.
248
+
249
+ `discovery.notUnitRoots` excludes each listed root and its whole subtree from
250
+ inferred units (adapter markers, adapter `roots()`, and supplemental roots) and
251
+ from scaffold layout scans and descriptor proposals, in both dry run and
252
+ `--write`. The defaults are `agentdoc`, `docs`, `scripts`, `node_modules`, and
253
+ `.github`; a configured list replaces them, and `[]` disables these exclusions.
254
+ An existing authored component descriptor strictly below an excluded root still
255
+ claims that directory as a unit. Its own root must supply the usual adapter
256
+ evidence for its artifact type; the exclusion does not waive corroboration.
257
+ Scaffold preserves that descriptor and never proposes or writes one there.
258
+ A descriptor exactly at an excluded root is refused with
259
+ `AGENTDOC_COMPONENT_UNCORROBORATED` naming the exclusion: remove that root from
260
+ `notUnitRoots` or move the descriptor.
261
+
240
262
  - Every discovered unit must have exactly one descriptor at its root, or
241
263
  `AGENTDOC_COVERAGE_ZERO`.
242
264
  - Two descriptors at one root, or a descriptor no adapter can corroborate, is an
@@ -21,7 +21,7 @@ jobs:
21
21
  with: { node-version: '24' }
22
22
  - name: Install the agentdoc CLI
23
23
  # Unpinned tracks main. For CI reproducibility pin a tag or commit:
24
- # github:lbframe/agent-doc-system#v1.1.0
24
+ # github:lbframe/agent-doc-system#v1.2.0
25
25
  # A compiler upgrade intentionally invalidates committed graphs.
26
26
  run: npm install -g github:lbframe/agent-doc-system
27
27
  - name: Validate and compile
package/evals/harness.mjs CHANGED
@@ -35,6 +35,7 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
35
35
  export function factsFromContext(ctx) {
36
36
  const out = new Set();
37
37
  out.add(ctx.entity.ref);
38
+ for (const n of ctx.neighbours) out.add(n.ref);
38
39
  for (const r of ctx.relations.items) out.add(r.type + ":" + r.other);
39
40
  for (const p of ctx.journeyPeers || []) out.add(p);
40
41
  for (const s2 of ctx.placement.systems) out.add(s2);
@@ -76,7 +77,10 @@ export function criticalFromContext(ctx) {
76
77
  }
77
78
 
78
79
  export function verificationFromContext(ctx) {
79
- return new Set(ctx.verification.items.map((v) => "verify:" + v.command));
80
+ // Commands such as `go test ./...` recur in multiple modules. Scenarios can
81
+ // require a scoped check ID as well as a command to measure dependent checks
82
+ // without letting the library's identical command satisfy the expectation.
83
+ return new Set(ctx.verification.items.flatMap((v) => ["verify:" + v.command, "verify-id:" + v.id]));
80
84
  }
81
85
 
82
86
  // ── baseline: unassisted repository exploration ────────────────────────
@@ -154,7 +158,20 @@ export function fullRepositoryContext(graph) {
154
158
  // Baseline verification: without routing, an agent runs the union of every
155
159
  // check in the repository, because nothing narrows it.
156
160
  function baselineVerification(graph) {
157
- return new Set(graph.verification.map((v) => "verify:" + v.command));
161
+ return new Set([...graph.verification, ...graph.gates].flatMap((v) => ["verify:" + v.command, "verify-id:" + v.id]));
162
+ }
163
+
164
+ export function verificationRecall(retrieved, expected) {
165
+ const { hit, total } = score(retrieved, expected);
166
+ return total ? hit / total : 1;
167
+ }
168
+
169
+ // Count scoped check IDs, not commands: identical commands in different
170
+ // components are separate checks. Gates are repository-scoped checks too.
171
+ export function checkSelectivity(graph, recommendedIds) {
172
+ const all = new Set([...graph.verification, ...graph.gates].map((v) => v.id));
173
+ const recommended = [...all].filter((id) => recommendedIds.has(id)).length;
174
+ return all.size ? recommended / all.size : 1;
158
175
  }
159
176
 
160
177
  function score(retrieved, truth) {
@@ -271,6 +288,10 @@ export function runRoutingEval(root, { json = false } = {}) {
271
288
  const retrieved = new Set();
272
289
  const critical = new Set();
273
290
  const verif = new Set();
291
+ const recommendedIds = new Set();
292
+ const collectChecks = (ctx) => {
293
+ for (const v of [...ctx.verification.items, ...ctx.globalGates]) recommendedIds.add(v.id);
294
+ };
274
295
  let unresolved = 0;
275
296
  for (const t of s.touch) {
276
297
  let ctx;
@@ -285,13 +306,16 @@ export function runRoutingEval(root, { json = false } = {}) {
285
306
  for (const x of factsFromContext(ctx)) retrieved.add(x);
286
307
  for (const x of criticalFromContext(ctx)) critical.add(x);
287
308
  for (const x of verificationFromContext(ctx)) verif.add(x);
309
+ collectChecks(ctx);
288
310
  }
289
311
  // Change impact is part of routing: it is what tells the agent which other
290
312
  // components a change reaches. Its output is legitimately part of context.
291
313
  const im = impact(graph, s.touch);
314
+ for (const v of im.recommendedVerification) recommendedIds.add(v.id);
292
315
  for (const c of im.affectedComponents) {
293
316
  const ctx = queryContext(graph, c.ref, {});
294
317
  for (const x of factsFromContext(ctx)) retrieved.add(x);
318
+ collectChecks(ctx);
295
319
  }
296
320
 
297
321
  const base = baselineContext(graph, s.touch);
@@ -303,8 +327,6 @@ export function runRoutingEval(root, { json = false } = {}) {
303
327
  const all = score(retrieved, new Set(s.truth));
304
328
  const baseAll = score(base, new Set(s.truth));
305
329
  const crit = score(critical, new Set(s.critical));
306
- const vf = score(verif, new Set(s.verification));
307
- const baseVf = score(baseV, new Set(s.verification));
308
330
  const noise = irrelevanceRatio(retrieved, closure);
309
331
  const baseNoise = irrelevanceRatio(base, baseClosure);
310
332
 
@@ -314,12 +336,14 @@ export function runRoutingEval(root, { json = false } = {}) {
314
336
  recall: all.hit / (all.total || 1),
315
337
  precision: 1 - noise,
316
338
  criticalRecall: crit.total ? crit.hit / crit.total : 1,
317
- verificationRecall: vf.total ? vf.hit / vf.total : 1,
339
+ verificationRecall: verificationRecall(verif, new Set(s.verification)),
340
+ checkSelectivity: checkSelectivity(graph, recommendedIds),
318
341
  noise,
319
342
  baseline: {
320
343
  recall: baseAll.hit / (baseAll.total || 1),
321
344
  noise: baseNoise,
322
- verificationRecall: baseVf.hit / (baseVf.total || 1),
345
+ verificationRecall: verificationRecall(baseV, new Set(s.verification)),
346
+ checkSelectivity: 1, // The baseline runs every verification entry and gate.
323
347
  },
324
348
  missed: all.missed.slice(0, 8),
325
349
  offClosure: [...retrieved].filter((r) => !closure.has(r)).slice(0, 8),
@@ -340,12 +364,14 @@ export function runRoutingEval(root, { json = false } = {}) {
340
364
  precision: round(mean("precision")),
341
365
  criticalRecall: round(mean("criticalRecall")),
342
366
  verificationRecall: round(mean("verificationRecall")),
367
+ checkSelectivity: round(mean("checkSelectivity")),
343
368
  noise: round(mean("noise")),
344
369
  retrievalReduction: 0,
345
370
  baseline: {
346
371
  recall: round(meanB("recall")),
347
372
  noise: round(meanB("noise")),
348
373
  verificationRecall: round(meanB("verificationRecall")),
374
+ checkSelectivity: round(meanB("checkSelectivity")),
349
375
  },
350
376
  };
351
377
  // Size proxy: retrieved fact count versus the whole-repository baseline.
@@ -5,5 +5,12 @@
5
5
  "touch": ["libs/protocol/ledger.pb.go"],
6
6
  "truth": ["component:default/protocol", "component:default/ingest", "component:default/gateway", "doc:libs/protocol/README.md"],
7
7
  "critical": [],
8
- "verification": []
8
+ "verification": [
9
+ "verify:go test ./...",
10
+ "verify:go vet ./...",
11
+ "verify-id:component:default/gateway:go-test",
12
+ "verify-id:component:default/gateway:go-vet",
13
+ "verify-id:component:default/ingest:go-test",
14
+ "verify-id:component:default/ingest:go-vet"
15
+ ]
9
16
  }