@agentskit/doc-bridge 1.7.45 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +465 -0
- package/CONTRIBUTING.md +23 -0
- package/README.md +87 -40
- package/SECURITY.md +7 -0
- package/action.yml +1 -1
- package/bin/ak-docs.js +2 -2
- package/bin/ak-verify.js +13 -7
- package/dist/cli/program.d.ts +3 -1
- package/dist/cli/program.js +15861 -6223
- package/dist/cli/program.js.map +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +86 -6
- package/dist/config/index.js.map +1 -1
- package/dist/{index-BUL0q7s8.d.ts → index-Beor6Yhi.d.ts} +133 -1
- package/dist/index.d.ts +9871 -1062
- package/dist/index.js +16126 -6125
- package/dist/index.js.map +1 -1
- package/docs/MARKETPLACE.md +1 -1
- package/docs/PRD-documentation-efficiency-study.md +406 -0
- package/docs/PRD-knowledge-retrieval-and-enrichment.md +466 -0
- package/docs/adr/0002-documentation-audit-boundary.md +22 -0
- package/docs/adr/0003-study-protocol-and-historical-evidence.md +40 -0
- package/docs/adr/0004-controlled-study-runner.md +25 -0
- package/docs/adr/0005-documentation-quality-and-criticality.md +20 -0
- package/docs/adr/0006-registry-semantic-grounding.md +20 -0
- package/docs/adr/0007-longitudinal-study-metrics.md +21 -0
- package/docs/adr/0008-study-verification-boundary.md +21 -0
- package/docs/adr/0009-study-provider-cli-adapter.md +25 -0
- package/docs/agent-corpus/INDEX.md +12 -1
- package/docs/agent-corpus/OVERVIEW.md +25 -0
- package/docs/agent-corpus/chat.md +5 -1
- package/docs/agent-corpus/cli.md +16 -0
- package/docs/agent-corpus/conformance.md +12 -0
- package/docs/agent-corpus/doc-bridge.md +47 -0
- package/docs/agent-corpus/doctor.md +8 -0
- package/docs/agent-corpus/gates.md +4 -0
- package/docs/agent-corpus/mcp.md +13 -0
- package/docs/agent-corpus/memory.md +4 -0
- package/docs/agent-corpus/query.md +33 -0
- package/docs/bench/README.md +122 -0
- package/docs/bench/retrieval-baseline-v1.json +28 -0
- package/docs/bench/retrieval-suite-v1.json +1033 -0
- package/docs/chat-and-rag.md +3 -2
- package/docs/for-agents.md +9 -1
- package/docs/getting-started.md +4 -11
- package/docs/guides/gate-ci.md +11 -1
- package/docs/guides/install-and-run.md +9 -65
- package/docs/index.md +22 -1
- package/docs/knowledge-engine-runbook.md +38 -3
- package/docs/landing/assets/context-payload-reduction.svg +21 -0
- package/docs/landing/assets/controlled-ab-comparison.svg +30 -0
- package/docs/landing/index.html +119 -5
- package/docs/loop-workflow.md +117 -0
- package/docs/mcp.md +6 -1
- package/docs/parity/public-claims-v1.json +145 -0
- package/docs/playbook/doc-bridge-pattern.md +1 -1
- package/docs/query.md +90 -2
- package/docs/recipes/index-pipeline.md +1 -1
- package/docs/schemas/agent-handoff-v1.md +15 -0
- package/docs/schemas/doc-bridge-index-v1.md +65 -0
- package/docs/spec/benchmark-v1.md +33 -1
- package/docs/spec/cli.md +30 -10
- package/docs/spec/config-v1.md +147 -8
- package/docs/spec/documentation-audit-v1.md +61 -0
- package/docs/spec/enrichment-overlay-v1.md +241 -0
- package/docs/spec/graph-signals-v1.md +92 -0
- package/docs/spec/incremental-scan-v1.md +102 -0
- package/docs/spec/markdown-analyzer-v1.md +73 -0
- package/docs/spec/mcp-knowledge-tools-v1.md +147 -0
- package/docs/spec/measured-enrichment-v1.md +229 -0
- package/docs/spec/public-parity-v1.md +119 -0
- package/docs/spec/registry-agents.md +6 -0
- package/docs/spec/render-v1.md +122 -0
- package/docs/spec/retrieval-index-v1.md +164 -0
- package/docs/spec/study-metrics-v1.md +58 -0
- package/docs/spec/study-protocol-v1.md +46 -0
- package/docs/spec/study-provider-cli-v1.md +116 -0
- package/docs/spec/study-runner-v1.md +35 -0
- package/docs/spec/study-task-suite-v1.md +41 -0
- package/docs/spec/study-verification-v1.md +40 -0
- package/docs/study/README.md +84 -0
- package/docs/study/ab-adjudicated-cost-analysis-v1.md +29 -0
- package/docs/study/ab-adjudicated-cost-plan-v1.json +33 -0
- package/docs/study/ab-adjudicated-cost-plan-v2-v1.json +33 -0
- package/docs/study/ab-adjudicated-cost-result-v1.json +80 -0
- package/docs/study/ab-baseline-analysis-v1.md +21 -0
- package/docs/study/ab-baseline-plan-v1.json +33 -0
- package/docs/study/ab-baseline-recovery-plan-v1.json +33 -0
- package/docs/study/ab-baseline-result-v1.json +79 -0
- package/docs/study/documentation-audit-round-2026-08-31.json +183 -0
- package/docs/study/historical-evidence-v1.json +252 -0
- package/docs/study/observation-ledger-v1.json +30632 -0
- package/docs/study/phase3-task-coverage-v1.json +34 -0
- package/docs/study/phase4-public-pilot-ledger-v1.json +1344 -0
- package/docs/study/phase4-public-pilot-result-v1.json +52 -0
- package/docs/study/phase4-public-pilot-run-plan-v1.json +26 -0
- package/docs/study/phase4-public-pilot-task-suite-v1.json +71 -0
- package/docs/study/pilot-round-2026-08-31.json +46 -0
- package/docs/study/protocol-v1.json +90 -0
- package/docs/study/publication-gate-v1.md +45 -0
- package/docs/study/quality-scorecard-cycle-plan.md +545 -0
- package/docs/study/quality-scorecard-v1.json +38 -0
- package/docs/study/round-1-adjudicated-smoke-v1.json +30642 -0
- package/docs/study/round-1-instrumentation-plan-v1.md +39 -0
- package/docs/study/round-2-expanded-adjudication-v1.json +91 -0
- package/docs/study/round-2-expanded-validation-v1.md +58 -0
- package/docs/study/round-3-evidence-contract-v1.json +75 -0
- package/docs/study/round-3-evidence-contract-v1.md +57 -0
- package/docs/study/round-4-confirmation-v1.json +75 -0
- package/docs/study/round-4-confirmation-v1.md +55 -0
- package/docs/study/run-plan-v1.json +33 -0
- package/docs/study/semantic-adjudication-cycle-8.md +20 -0
- package/docs/study/task-suite-v1.json +96 -0
- package/docs/study/token-efficiency-plan-v1.md +337 -0
- package/docs/study/token-efficiency-protocol-v2.json +62 -0
- package/docs/study/verification-binding-v1.json +27 -0
- package/docs/validation-cycle-plan.md +14 -0
- package/docs/verification-harness.md +11 -6
- package/ecosystem-claims.json +2 -2
- package/ecosystem-upstream.json +2 -2
- package/ecosystem.json +4 -4
- package/mcpb/manifest.json +9 -1
- package/package.json +25 -6
- package/scripts/check-ecosystem-upstream.mjs +34 -6
- package/skills/doc-bridge-handoff/fixtures/synthetic-repo/docs/for-agents/packages/payments.md +7 -0
- package/skills/doc-bridge-handoff/scripts/resolve-handoff.mjs +1 -1
- package/src/agents/registry-adapter.ts +192 -24
- package/src/audit/documentation.ts +513 -0
- package/src/bench/baseline.ts +198 -0
- package/src/bench/overlay-delta.ts +139 -0
- package/src/bench/retrieval.ts +319 -0
- package/src/budget/compile.ts +91 -0
- package/src/budget/sections.ts +70 -0
- package/src/cli/program.ts +684 -74
- package/src/cli/usage.ts +71 -0
- package/src/config/defaults.ts +1 -0
- package/src/config/index.ts +4 -0
- package/src/config/schema.ts +117 -2
- package/src/conformance/documentation-standard-v1.ts +8 -6
- package/src/discovery/areas.ts +182 -0
- package/src/discovery/documentation.ts +211 -5
- package/src/discovery/identity.ts +24 -0
- package/src/discovery/incremental.ts +314 -0
- package/src/discovery/inputs.ts +110 -0
- package/src/discovery/markdown.ts +481 -0
- package/src/discovery/repository.ts +484 -101
- package/src/doctor/run-doctor.ts +244 -12
- package/src/enrich/approvals.ts +190 -0
- package/src/enrich/cache.ts +93 -0
- package/src/enrich/context-pack.ts +272 -0
- package/src/enrich/overlay.ts +255 -0
- package/src/enrich/review.ts +106 -0
- package/src/enrich/stage.ts +374 -0
- package/src/enrich/stats.ts +100 -0
- package/src/enrich/validate.ts +410 -0
- package/src/federation/llms.ts +2 -4
- package/src/findings/report.ts +103 -0
- package/src/graph/build.ts +356 -0
- package/src/graph/memory.ts +208 -0
- package/src/index-builder/build-handoffs.ts +22 -11
- package/src/index-builder/build-index.ts +132 -3
- package/src/index-builder/llms-txt.ts +48 -8
- package/src/index-builder/project-corpus.ts +111 -0
- package/src/index.ts +630 -2
- package/src/lib/fuzzy-match.ts +235 -0
- package/src/mcp/knowledge.ts +554 -0
- package/src/mcp/server.ts +113 -18
- package/src/metrics/benchmark.ts +21 -0
- package/src/parity/check.ts +309 -0
- package/src/parity/claims.ts +259 -0
- package/src/parity/resolve.ts +160 -0
- package/src/query/handoff.ts +326 -0
- package/src/query/load-index.ts +53 -1
- package/src/query/query.ts +92 -59
- package/src/query/search.ts +289 -92
- package/src/query/text.ts +155 -0
- package/src/reconciliation/reconcile.ts +101 -10
- package/src/render/data.ts +356 -0
- package/src/render/engine.ts +398 -0
- package/src/render/generated.ts +77 -0
- package/src/render/render.ts +209 -0
- package/src/render/template-source.ts +52 -0
- package/src/render/templates.ts +289 -0
- package/src/report/html.ts +2 -2
- package/src/retrieval/bm25.ts +161 -0
- package/src/retrieval/project.ts +495 -0
- package/src/retrieval/rank.ts +383 -0
- package/src/retrieval/weights.ts +39 -0
- package/src/retriever/doc-bridge-retriever.ts +100 -15
- package/src/rules/engine.ts +30 -10
- package/src/schemas/agent-handoff.ts +56 -0
- package/src/schemas/budget.ts +37 -0
- package/src/schemas/doc-bridge-index.ts +53 -2
- package/src/schemas/enrichment.ts +369 -0
- package/src/schemas/json-schemas.ts +39 -2
- package/src/schemas/knowledge.ts +14 -1
- package/src/schemas/retrieval-index.ts +152 -0
- package/src/shims/graphology.d.ts +91 -0
- package/src/study/adjudication.ts +196 -0
- package/src/study/execution.ts +350 -0
- package/src/study/expectations.ts +219 -0
- package/src/study/metrics.ts +467 -0
- package/src/study/protocol.ts +271 -0
- package/src/study/provider-cli.ts +115 -0
- package/src/study/provider-telemetry.ts +47 -0
- package/src/study/quality-scorecard.ts +164 -0
- package/src/study/runner.ts +461 -0
- package/src/study/task-suite.ts +321 -0
- package/src/study/verification.ts +134 -0
- package/src/validate.ts +1 -4
- package/src/version.ts +1 -1
- package/src/workflow/engine.ts +36 -11
- package/scripts/verification-harness.mjs +0 -482
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Registry semantic review grounding boundary
|
|
3
|
+
status: accepted
|
|
4
|
+
date: 2026-08-30
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Registry semantic review grounding boundary
|
|
8
|
+
|
|
9
|
+
## Decision
|
|
10
|
+
|
|
11
|
+
Registry agents may classify and explain deterministic documentation findings, but their proposals must be grounded in the exact snapshot and reconciliation report supplied to the adapter. The adapter requires a known diagnostic, at least one matching evidence reference, and the installed Registry agent identity and version. It accepts either the default configured agent or an alternate installed agent through configuration without changing the common engine.
|
|
12
|
+
|
|
13
|
+
Registry output remains advisory. Human approval is required before a proposal is converted into a fix or considered resolved, and applying a fix requires post-apply verification against the new source revision.
|
|
14
|
+
|
|
15
|
+
## Consequences
|
|
16
|
+
|
|
17
|
+
- Unsupported claims and invented evidence fail closed at the adapter boundary.
|
|
18
|
+
- Deterministic facts remain separate from semantic agent judgment.
|
|
19
|
+
- Agent swaps remain configurable and attributable.
|
|
20
|
+
- The existing fix-proposal approval and verification flow remains the single mutation path.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Deterministic longitudinal study metrics
|
|
3
|
+
status: accepted
|
|
4
|
+
date: 2026-08-30
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Deterministic longitudinal study metrics
|
|
8
|
+
|
|
9
|
+
## Decision
|
|
10
|
+
|
|
11
|
+
Doc Bridge calculates longitudinal study metrics from content-addressed observation ledgers. It exposes subgroup metrics and comparisons for the baseline and current approved round, preserves provider-versus-estimated token provenance, keeps missing measurements explicit, and emits uncertainty for rates. A bounded numeric measurement envelope allows task, documentation, cost, and future operational metrics to be added without changing the observation identity contract.
|
|
12
|
+
|
|
13
|
+
Efficiency improvements are subordinate to correctness, evidence, acceptance, safety, and rework. A round with cheaper or faster execution but a quality regression is reported as `regressed`.
|
|
14
|
+
|
|
15
|
+
## Consequences
|
|
16
|
+
|
|
17
|
+
- The same ledger produces JSON and text reports with the same metric values and content hash.
|
|
18
|
+
- Baseline replacement is outside the calculator and remains an explicit audited operation.
|
|
19
|
+
- Historical data can remain incomplete without being converted into false precision.
|
|
20
|
+
- New metric names can be collected without a schema migration, but interpretation requires a versioned metric contract and documentation.
|
|
21
|
+
- Small samples remain `inconclusive`, so the report supports decisions without overstating evidence.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Controlled study verification boundary
|
|
3
|
+
status: accepted
|
|
4
|
+
date: 2026-08-31
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Controlled study verification boundary
|
|
8
|
+
|
|
9
|
+
## Decision
|
|
10
|
+
|
|
11
|
+
Controlled study results require a separate content-addressed verification binding. The binding records provenance, artifact hashes, anonymized privacy review, and consumed token/runtime budgets. The result envelope is the only artifact eligible for publication or an enterprise study claim.
|
|
12
|
+
|
|
13
|
+
Recovery remains delegated to the existing workflow engine and observation-ledger idempotency. The harness is the authority for state, human approval, tracking authorization, and explicit baseline replacement.
|
|
14
|
+
|
|
15
|
+
## Consequences
|
|
16
|
+
|
|
17
|
+
- A locally calculated report is distinguishable from a verified result.
|
|
18
|
+
- Privacy and budget failures fail closed without exposing matched values.
|
|
19
|
+
- Interrupted or stale work can resume from valid evidence without duplicating observations.
|
|
20
|
+
- Baselines remain immutable during normal runs and have an auditable replacement path.
|
|
21
|
+
- Publication still requires human review; deterministic scanning cannot approve its own output.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Generic CLI adapter for controlled study providers
|
|
3
|
+
status: accepted
|
|
4
|
+
date: 2026-08-31
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Generic CLI adapter for controlled study providers
|
|
8
|
+
|
|
9
|
+
## Decision
|
|
10
|
+
|
|
11
|
+
The controlled study invokes explicitly configured model or agent CLIs as fresh child processes. The adapter accepts an executable plus an argument array, never a shell command string, and allowlists environment variables. Each invocation receives bounded input, runtime, output, and retry limits and must return one JSON object on stdout. Logs belong on stderr.
|
|
12
|
+
|
|
13
|
+
Provider configuration is content-addressed and maps every `(model, scenario)` pair to exactly one command. The adapter does not bundle or require a local model runtime: hosted providers, vendor CLIs, and AgentsKit Registry agents are configured by the study owner. Provider network access is declared per mapping so the run can be audited; repository task network policy remains governed by the study protocol.
|
|
14
|
+
|
|
15
|
+
## Alternatives considered
|
|
16
|
+
|
|
17
|
+
- Ollama as the default: rejected because a heavyweight local runtime would create friction and hardware-dependent results.
|
|
18
|
+
- Direct SDK integrations: deferred because they multiply provider-specific dependencies and credential handling in the core package.
|
|
19
|
+
- Shell command strings: rejected because shell interpretation makes argument provenance and isolation weaker.
|
|
20
|
+
|
|
21
|
+
## Consequences
|
|
22
|
+
|
|
23
|
+
- A real provider can be selected without changing the runner or ledger schema.
|
|
24
|
+
- A missing executable, malformed JSON, non-zero exit, timeout, or output limit fails closed and is recorded as unavailable, invalid, failed, timed-out, or budget-exceeded evidence.
|
|
25
|
+
- Provider credentials remain outside committed study artifacts; only explicitly allowlisted environment variables are passed to the child.
|
|
@@ -3,8 +3,19 @@ type: module
|
|
|
3
3
|
id: doc-bridge-index
|
|
4
4
|
editRoot: src/index-builder
|
|
5
5
|
humanDoc: /docs/recipes/index-pipeline
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/index-builder
|
|
9
|
+
validationPath: pnpm test && pnpm typecheck && node bin/ak-docs.js index
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# Index builder
|
|
9
13
|
|
|
10
|
-
Owns corpus scanning, handoffs, hashes, `llms.txt`, and watch mode. Generated output must remain deterministic.
|
|
14
|
+
Owns corpus scanning, handoffs, hashes, `llms.txt`, and watch mode. Generated output must remain deterministic. The retrieval projection inside the index is a function of the discovery snapshot (`src/retrieval/project.ts`); the builder scans nothing for it beyond the snapshot itself, and `knowledge[]` stays in step with it without body text.
|
|
15
|
+
|
|
16
|
+
`project-corpus.ts` projects every repository document and source module into `index.knowledge`
|
|
17
|
+
so retrieval can see what the discovery snapshot sees; it must walk the same files under the same
|
|
18
|
+
safety limits, which is why both import from `src/discovery/inputs.ts`. One walk produces both the
|
|
19
|
+
entries and the `inputs` fingerprint, so the hash can never describe a different file set than the
|
|
20
|
+
one projected. `llms.txt` stays curated — the renderer filters projected entries out, and the
|
|
21
|
+
conformance profile re-renders to check freshness, so the filter belongs there and nowhere else.
|
|
@@ -2,8 +2,33 @@
|
|
|
2
2
|
type: index
|
|
3
3
|
purpose: Route coding agents to Doc Bridge ownership sidecars.
|
|
4
4
|
owner: maintainers
|
|
5
|
+
lifecycle: active
|
|
6
|
+
sourceOfTruth: doc-bridge.config.json
|
|
7
|
+
validationPath: node bin/ak-docs.js query ownership doc-bridge --agent
|
|
5
8
|
---
|
|
6
9
|
|
|
7
10
|
# Doc Bridge agent corpus
|
|
8
11
|
|
|
9
12
|
Resolve the requested ownership ID with `ak-docs query ownership <id> --agent`. Read the returned sidecar and its linked human guide before editing.
|
|
13
|
+
|
|
14
|
+
## Example
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
ak-docs query ownership doc-bridge --agent
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The command returns a bounded handoff similar to:
|
|
21
|
+
|
|
22
|
+
```json
|
|
23
|
+
{
|
|
24
|
+
"target": { "type": "package", "id": "doc-bridge", "path": "src" },
|
|
25
|
+
"startHere": "docs/agent-corpus/doc-bridge.md",
|
|
26
|
+
"readBeforeEditing": ["docs/agent-corpus/doc-bridge.md", "AGENTS.md"],
|
|
27
|
+
"editRoots": ["src"],
|
|
28
|
+
"checks": ["pnpm test", "pnpm typecheck"],
|
|
29
|
+
"humanDoc": "/docs/POSITIONING"
|
|
30
|
+
}
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Use `startHere` and `readBeforeEditing` first, edit only within `editRoots`,
|
|
34
|
+
then run the declared `checks`.
|
|
@@ -3,8 +3,12 @@ type: module
|
|
|
3
3
|
id: doc-bridge-chat
|
|
4
4
|
editRoot: src/intelligence
|
|
5
5
|
humanDoc: /docs/chat-and-rag
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/intelligence
|
|
9
|
+
validationPath: pnpm test && pnpm typecheck && pnpm docs:typecheck
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# Chat and RAG
|
|
9
13
|
|
|
10
|
-
Owns optional AgentsKit retrieval and chat after deterministic lookup. Conversational UI belongs to
|
|
14
|
+
Owns optional AgentsKit retrieval and chat after deterministic lookup. Conversational UI belongs to the chat surface; enterprise orchestration belongs to the consuming platform.
|
package/docs/agent-corpus/cli.md
CHANGED
|
@@ -3,8 +3,24 @@ type: module
|
|
|
3
3
|
id: doc-bridge-cli
|
|
4
4
|
editRoot: src/cli
|
|
5
5
|
humanDoc: /docs/spec/cli
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/cli
|
|
9
|
+
validationPath: pnpm test && pnpm typecheck
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# CLI
|
|
9
13
|
|
|
10
14
|
Owns public `ak-docs` commands and output modes. Keep JSON output versioned and text output readable.
|
|
15
|
+
|
|
16
|
+
The documentation audit is a deterministic post-discovery check:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
ak-docs audit documentation --json
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
It reports measurable quality, coverage, stale, contradiction, redundancy, and structure-gap signals. `not-analyzed` means semantic evidence is missing; it is not a pass.
|
|
23
|
+
|
|
24
|
+
`ak-docs enrich` and `check --enrich` are the only commands that call a Registry agent for
|
|
25
|
+
enrichment. A failed, timed-out or malformed enrichment is reported and never changes a check
|
|
26
|
+
result or its exit code; `enrich` exits 2 and writes no overlay.
|
|
@@ -3,8 +3,20 @@ type: module
|
|
|
3
3
|
id: doc-bridge-conformance
|
|
4
4
|
editRoot: src/conformance
|
|
5
5
|
humanDoc: /docs/spec/documentation-standard-v1
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/conformance
|
|
9
|
+
validationPath: pnpm test && node bin/ak-docs.js conformance run documentation-standard-v1 --text
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# Conformance
|
|
9
13
|
|
|
10
14
|
Owns versioned documentation-standard rules and evidence. Do not turn missing evidence into a passing score.
|
|
15
|
+
|
|
16
|
+
Documentation audits are a separate, configurable check after discovery and reconciliation:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
ak-docs audit documentation --json
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Use the audit metrics and evidence to review quality, coverage, stale declarations, and structure/documentation gaps. Generated documents are freshness-boundary checks only; semantic findings marked `not-analyzed` require the configured Registry agent or a human review.
|
|
@@ -3,8 +3,55 @@ type: package
|
|
|
3
3
|
package: '@agentskit/doc-bridge'
|
|
4
4
|
editRoot: src
|
|
5
5
|
humanDoc: /docs/POSITIONING
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src
|
|
9
|
+
validationPath: pnpm test && pnpm typecheck
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# Doc Bridge core
|
|
9
13
|
|
|
10
14
|
Owns the public CLI, index, MCP, gates, and doctor contracts. Preserve deterministic behavior and run `pnpm test && pnpm typecheck`.
|
|
15
|
+
|
|
16
|
+
Analyzers emit `observed` facts with file and line evidence, and say what they could not resolve
|
|
17
|
+
as coverage rather than guessing: an ambiguous reference produces a note, not an edge. A near-miss
|
|
18
|
+
resolves only at the Jaro-Winkler threshold with a single candidate. Adding an analyzer means
|
|
19
|
+
adding its version to `analyzerVersions` and bumping `pipelineVersion`, never changing the
|
|
20
|
+
`DiscoverySnapshotV1` envelope.
|
|
21
|
+
|
|
22
|
+
Areas (`area:<dir>`) are the unit between a package and a file. Each module belongs to exactly
|
|
23
|
+
one — the most specific — so containment is a tree and area-scope aggregation has one answer per
|
|
24
|
+
module. An ownership path is an area by declaration even when convention would not derive it.
|
|
25
|
+
|
|
26
|
+
Graph signals live in `src/graph/` and are computed on demand, never stored: sort node and edge
|
|
27
|
+
insertion before any metric, round every score, and keep graphology's format out of every artifact.
|
|
28
|
+
A clustering result is a suggestion in `coverage`, never an entity. A static signal must not be
|
|
29
|
+
worded as a runtime claim.
|
|
30
|
+
|
|
31
|
+
A file-backed entity carries its file's hash, and a second scan may reuse an entity whose hash is
|
|
32
|
+
unchanged — but only while the universe its references resolve against is identical, and only from
|
|
33
|
+
a snapshot produced by this pipeline, these analyzers and this configuration. Reuse either
|
|
34
|
+
reproduces a cold scan exactly or it is refused, and the run reports what it reused as `coverage`.
|
|
35
|
+
|
|
36
|
+
Markdown for people is rendered from templates in `src/render/`, never concatenated: a template
|
|
37
|
+
sees only the variables `src/render/data.ts` computes, sorted and without timestamps, and the
|
|
38
|
+
synchronous evaluator in `src/render/engine.ts` must keep rendering what knap renders. Every
|
|
39
|
+
generated region carries a `<!-- doc-bridge:generated hash=… -->` marker so the analyzer skips it
|
|
40
|
+
and the audit can report a hand edit inside it; `llms.txt` is the exception and stays byte-identical.
|
|
41
|
+
Enrichment (`src/enrich/`, `src/schemas/enrichment.ts`) is advisory by construction. A proposal is
|
|
42
|
+
a typed claim with one deterministic validator per kind; a validator checks, it never judges. An
|
|
43
|
+
accepted entry binds to the content hash of the entity it describes and expires when that moves;
|
|
44
|
+
no entry deletes or alters observed data, and the projection asserts it. The overlay is read only
|
|
45
|
+
while `intelligence.registry.enabled` is true, a read never writes, and a corrupt overlay is no
|
|
46
|
+
overlay. No agent approves anything — not its own output, not another's — and human approvals go
|
|
47
|
+
through the ecosystem approval gate. Adding a proposal kind means a schema, a validator, a policy,
|
|
48
|
+
a valid fixture and a rejection fixture, and a row in `docs/spec/enrichment-overlay-v1.md`.
|
|
49
|
+
|
|
50
|
+
An enrichment run reports its own shape (`src/enrich/stats.ts`): counts per kind, a rejection
|
|
51
|
+
histogram, invented references counted apart from other rejections, cost, and stability against the
|
|
52
|
+
previous run. `stats` stays outside the overlay's content hash — two runs that decided identically
|
|
53
|
+
must agree on their hash while disagreeing about how long they took. The overlay may leave retrieval
|
|
54
|
+
unchanged or improve it; it may not lower hit@3, and `measureOverlayRetrievalDelta`
|
|
55
|
+
(`src/bench/overlay-delta.ts`) projects both indexes from one snapshot so the only difference
|
|
56
|
+
between the two runs is the overlay. A drop is a finding about the agent, never a new baseline.
|
|
57
|
+
See [Measured enrichment v1](../spec/measured-enrichment-v1.md).
|
|
@@ -3,8 +3,16 @@ type: module
|
|
|
3
3
|
id: doc-bridge-doctor
|
|
4
4
|
editRoot: src/doctor
|
|
5
5
|
humanDoc: /docs/spec/documentation-standard-v1
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/doctor
|
|
9
|
+
validationPath: pnpm test && node bin/ak-docs.js doctor --text
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# Doctor
|
|
9
13
|
|
|
10
14
|
Owns health scoring and remediation guidance. Every point must trace to meaningful repository evidence.
|
|
15
|
+
|
|
16
|
+
Reachability, connectivity and the retrieval benchmark are measured from the snapshot, the
|
|
17
|
+
projection and the golden suite, never declared; a missing suite is `not-analyzed`, not omitted.
|
|
18
|
+
An A requires all three, and the grade must fall while any document is outside the projection.
|
|
@@ -3,6 +3,10 @@ type: module
|
|
|
3
3
|
id: doc-bridge-gates
|
|
4
4
|
editRoot: src/gates
|
|
5
5
|
humanDoc: /docs/spec/documentation-standard-v1
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/gates
|
|
9
|
+
validationPath: pnpm test && node bin/ak-docs.js gate run
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# Gates
|
package/docs/agent-corpus/mcp.md
CHANGED
|
@@ -3,8 +3,21 @@ type: module
|
|
|
3
3
|
id: doc-bridge-mcp
|
|
4
4
|
editRoot: src/mcp
|
|
5
5
|
humanDoc: /docs/mcp
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/mcp
|
|
9
|
+
validationPath: pnpm test && pnpm typecheck && pnpm smoke:packaged
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# MCP
|
|
9
13
|
|
|
10
14
|
Owns the stdio server and public tool contracts. Preserve runtime validation and stable response shapes.
|
|
15
|
+
|
|
16
|
+
`knowledge.search` and `knowledge.lookup` (`src/mcp/knowledge.ts`) read the same projection and
|
|
17
|
+
rank with the same `searchIndex` as the CLI; a test compares the two. A `budgetTokens` drops
|
|
18
|
+
sections only in the declared order — evidence excerpts, related, neighbours, summaries — and
|
|
19
|
+
never the entity, the evidence paths and hashes, the handoff fields or the diagnostics; a payload
|
|
20
|
+
that cannot fit says `fits: false`. Existing tool names, arguments and payloads do not change.
|
|
21
|
+
`docbridge.proposals` carries enrichment review as `enrich-list`, `enrich-approve` and
|
|
22
|
+
`enrich-reject`; a decision goes through `decideEnrichment` and the shared approval gate, never
|
|
23
|
+
through a direct overlay edit.
|
|
@@ -3,8 +3,41 @@ type: module
|
|
|
3
3
|
id: doc-bridge-query
|
|
4
4
|
editRoot: src/query
|
|
5
5
|
humanDoc: /docs/query
|
|
6
|
+
owner: doc-bridge-maintainers
|
|
7
|
+
lifecycle: active
|
|
8
|
+
sourceOfTruth: src/query
|
|
9
|
+
validationPath: pnpm test && pnpm typecheck
|
|
6
10
|
---
|
|
7
11
|
|
|
8
12
|
# Query
|
|
9
13
|
|
|
10
14
|
Owns deterministic package and document resolution. Prefer an explicit miss over an invented answer.
|
|
15
|
+
|
|
16
|
+
Ranking (`src/retrieval/rank.ts`) reads the retrieval projection (`src/retrieval/project.ts`),
|
|
17
|
+
which is a function of the snapshot: never add a scanner to the query path, and never import
|
|
18
|
+
anything under `src/agents` from it. The score is field-weighted BM25 times a prior, plus absolute
|
|
19
|
+
boosts for exact identity, plus graph proximity and canonicality; every component is reported by
|
|
20
|
+
`--explain`, and explaining must never change the ranking. Keep the split: a prior must never be
|
|
21
|
+
able to rank a record that matched nothing, an exact id, path or exported symbol must win over
|
|
22
|
+
prose that mentions it, and a result only a relation surfaced earns proximity and nothing else. One
|
|
23
|
+
tokenizer (`searchTokens`) serves both indexing and querying — never tokenize one side differently.
|
|
24
|
+
Changing the stopword lists, folding, or token expansion means bumping `SEARCH_LEXICON_VERSION`,
|
|
25
|
+
because the index records it and the artifact hash depends on it.
|
|
26
|
+
|
|
27
|
+
Handoffs (`src/query/handoff.ts`) are derived from the projection's graph for any entity — package,
|
|
28
|
+
area, module, document — and every field says which relation produced it. `checks` must report
|
|
29
|
+
their origin in `metadata.checksSource`; `startHere` prefers a document that covers the target over
|
|
30
|
+
one that mentions it.
|
|
31
|
+
|
|
32
|
+
Re-run `pnpm bench:retrieval` after any ranking change; a hit@3 regression is a blocked change,
|
|
33
|
+
not a judgement call.
|
|
34
|
+
|
|
35
|
+
Accepted agent signals (`agentSignal` on an entry) are a share of `ACCEPTED_SIGNALS_WEIGHT`, which
|
|
36
|
+
is 15% of the exact-id boost and must stay there: a signal may reorder near-ties among lexical hits
|
|
37
|
+
and must never outrank an exact identifier, path or symbol match. Without an overlay the ranking is
|
|
38
|
+
byte-identical to the ranking before enrichment existed, and the bench baseline must not move.
|
|
39
|
+
|
|
40
|
+
`ak-docs bench retrieval <suite> --overlay` answers whether the accepted overlay earns its cost:
|
|
41
|
+
the same suite over the same snapshot, once with the overlay projected and once without. It needs
|
|
42
|
+
no index on disk, and it exits non-zero when hit@3 falls. Adding an overlay-consuming ranking
|
|
43
|
+
signal means re-running it, not only `pnpm bench:retrieval`.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Retrieval benchmark
|
|
3
|
+
description: A deterministic, model-free measurement of whether a query returns the right entity, and the gate that stops a ranking change from making it worse.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Retrieval benchmark
|
|
7
|
+
|
|
8
|
+
`ak-docs bench retrieval` answers one question: **does a query return the right thing?** It runs the deterministic retrieval path over a golden query suite, reports hit@1, hit@3, mean reciprocal rank and the context cost of the top three results, and compares the figures with an approved baseline. It uses no model, no network and no API key, and it finishes in well under a second on this repository, so it can gate every pull request.
|
|
9
|
+
|
|
10
|
+
This is workstream KR-07 of [#169](https://github.com/AgentsKit-io/doc-bridge/issues/169). It exists before the ranking work in KR-01 and KR-06 so those changes have a recorded "before" to improve on, rather than a claim.
|
|
11
|
+
|
|
12
|
+
## Run it
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
node bin/ak-docs.js index # the benchmark measures the current index
|
|
16
|
+
pnpm bench:retrieval # suite + committed baseline, text output
|
|
17
|
+
node bin/ak-docs.js bench retrieval docs/bench/retrieval-suite-v1.json --json
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Exit codes: `0` when the figures hold, `1` on a hit@3 regression or a changed suite, `2` on a usage or input error.
|
|
21
|
+
|
|
22
|
+
## What it measures
|
|
23
|
+
|
|
24
|
+
| Metric | Meaning |
|
|
25
|
+
| --- | --- |
|
|
26
|
+
| `hitAt1` | Share of cases whose first result is an expected target |
|
|
27
|
+
| `hitAt3` | Share of cases with an expected target in the top three. **The gated metric** |
|
|
28
|
+
| `meanReciprocalRank` | Mean of `1 / rank`, `0` when no expected target appears |
|
|
29
|
+
| `meanContextBytes` | Mean serialized size of the top-three payload an agent would receive |
|
|
30
|
+
| `meanApproxTokens` | `meanContextBytes / 4`, reported as `tokenMethod: 'approximate'` |
|
|
31
|
+
| `zeroResultRate` | Share of cases where the query returned nothing at all |
|
|
32
|
+
|
|
33
|
+
Results are also segmented `byLang` and `byKind`, because an aggregate average hides exactly the failures that matter: on the first run, ownership routing scored 100 percent while every exported-symbol query scored zero.
|
|
34
|
+
|
|
35
|
+
The result artifact carries **no timestamp and no latency**, so two runs over the same index and suite are byte-identical and their content hashes match. Wall time belongs in the terminal, never in a comparable artifact.
|
|
36
|
+
|
|
37
|
+
## The suite
|
|
38
|
+
|
|
39
|
+
[`retrieval-suite-v1.json`](./retrieval-suite-v1.json) is a valid [Open Eval Format](https://www.npmjs.com/package/@agentskit/core) document (`evalFormatVersion: 2026-04`), so the same dataset can be read by any runner that speaks the format. 60 cases across four kinds and two languages:
|
|
40
|
+
|
|
41
|
+
| Kind | Cases | What it asks |
|
|
42
|
+
| --- | --- | --- |
|
|
43
|
+
| `symbol` | 20 | An exported identifier, the way an agent types it: `reconcileKnowledge` |
|
|
44
|
+
| `path` | 8 | A file or directory: `src/mcp/server.ts` |
|
|
45
|
+
| `question` | 26 | Natural language, English and Portuguese |
|
|
46
|
+
| `ownership` | 6 | The routing question a handoff exists to answer |
|
|
47
|
+
|
|
48
|
+
Each case carries `metadata.expectedTargets`. **A target matches a result by entity id or by repository path**, which is what lets one suite survive the entity-identity changes coming in KR-01 through KR-06: `docs/mcp.md` keeps working when the index starts carrying `document:docs/mcp.md`, and a module target starts working once modules are projected into the index.
|
|
49
|
+
|
|
50
|
+
Each case also carries a portable `expected.regex` alternation over the same targets, so a generic Open Eval Format runner can compute pass or fail without knowing anything about Doc Bridge. A test asserts the two stay consistent, and another validates the whole suite with the real `validateEvalSuite` from `@agentskit/core/eval-format`, so the locally mirrored format cannot drift from the ecosystem one.
|
|
51
|
+
|
|
52
|
+
### Adding a case
|
|
53
|
+
|
|
54
|
+
Add it to the suite, then re-approve the baseline: a changed suite makes the old figures incomparable, so the gate fails closed rather than pretending otherwise.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
node bin/ak-docs.js bench retrieval docs/bench/retrieval-suite-v1.json \
|
|
58
|
+
--baseline docs/bench/retrieval-baseline-v1.json \
|
|
59
|
+
--update-baseline --by "<your name>" --reason "Added three cases for the area entities"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## The baseline and the gate
|
|
63
|
+
|
|
64
|
+
[`retrieval-baseline-v1.json`](./retrieval-baseline-v1.json) holds the approved figures plus an approval record: who approved them, when, why, and the hashes of the result and index they were measured from. Three properties make it a gate rather than a suggestion:
|
|
65
|
+
|
|
66
|
+
- **A normal run never writes it.** Recording a baseline requires `--update-baseline` *and* `--by <name>`. Without an approver the command refuses.
|
|
67
|
+
- **It cannot be edited.** The file carries its own content hash; a hand-edited figure is rejected before any comparison happens.
|
|
68
|
+
- **A changed suite fails closed.** Comparing figures measured over different questions would be meaningless, so it is an error that only an explicit re-approval clears.
|
|
69
|
+
|
|
70
|
+
Only `hitAt3` blocks. Every other metric is reported: an improvement as `improved`, a decline as `warning`. So a change that lifts hit@3 while wrecking hit@1 passes the gate but says so out loud.
|
|
71
|
+
|
|
72
|
+
## The baselines
|
|
73
|
+
|
|
74
|
+
### Before KR-01
|
|
75
|
+
|
|
76
|
+
Measured on `master` at 1.8.0, before any ranking work:
|
|
77
|
+
|
|
78
|
+
| Metric | Value |
|
|
79
|
+
| --- | --- |
|
|
80
|
+
| hit@1 | 21.7% |
|
|
81
|
+
| hit@3 | 23.3% |
|
|
82
|
+
| Mean reciprocal rank | 0.228 |
|
|
83
|
+
| Zero-result rate | 46.7% |
|
|
84
|
+
| Mean context | 243 bytes (~61 tokens) |
|
|
85
|
+
|
|
86
|
+
By kind: ownership 100%, question 26.9%, path 12.5%, **symbol 0%**. By language: English 20%, Portuguese 40%.
|
|
87
|
+
|
|
88
|
+
Those numbers are the diagnosis in the parent PRD turned into measurements. Nearly half of all queries return nothing, and not one of the twenty exported-symbol queries finds its module, because the index holds 11 agent sidecars while the snapshot holds 369 entities. KR-01 and KR-06 have to move these; this file is how anyone can check that they did.
|
|
89
|
+
|
|
90
|
+
### After KR-01
|
|
91
|
+
|
|
92
|
+
The current approved baseline, with the repository corpus projected into the index and ranked by
|
|
93
|
+
field-weighted BM25:
|
|
94
|
+
|
|
95
|
+
| Metric | Before | After |
|
|
96
|
+
| --- | --- | --- |
|
|
97
|
+
| hit@1 | 21.7% | **76.7%** |
|
|
98
|
+
| hit@3 | 23.3% | **83.3%** |
|
|
99
|
+
| Mean reciprocal rank | 0.228 | **0.812** |
|
|
100
|
+
| Zero-result rate | 46.7% | **5.0%** |
|
|
101
|
+
| Mean context | 243 bytes (~61 tokens) | 343 bytes (~86 tokens) |
|
|
102
|
+
|
|
103
|
+
The figures here are a summary; [`retrieval-baseline-v1.json`](./retrieval-baseline-v1.json) holds
|
|
104
|
+
the approved ones. They move slightly whenever the documentation changes, because the
|
|
105
|
+
documentation *is* part of the corpus being searched — which is why only hit@3 blocks.
|
|
106
|
+
|
|
107
|
+
By kind: symbol **0% → 100%**, path 12.5% → **100%**, question 26.9% → **61.5%**, ownership 100%
|
|
108
|
+
→ 100%. By language: English 20% → **92%**, Portuguese 40% → 40%.
|
|
109
|
+
|
|
110
|
+
**On the context figure.** Mean context rose, and that is not a regression hiding in a warning.
|
|
111
|
+
Before, 46.7% of queries returned nothing and cost nothing; the mean was low because retrieval
|
|
112
|
+
was failing. Per *answered* query the cost went down — 243 / 0.533 ≈ 456 bytes before, 343 /
|
|
113
|
+
0.95 ≈ 361 bytes now — while 78% more queries get answered. That is the trade the PRD asked
|
|
114
|
+
for; the metric is reported rather than gated precisely so the shape of such a change stays
|
|
115
|
+
visible.
|
|
116
|
+
|
|
117
|
+
**On Portuguese.** Unchanged at 40%, and three of the ten cases still return nothing. The lexicon
|
|
118
|
+
now handles Portuguese stopwords, accents and plurals, so a Portuguese query against Portuguese
|
|
119
|
+
documentation ranks by the same rules as English — but this repository's documentation is in
|
|
120
|
+
English, and matching `reconciliação` to `reconciliation` is a cross-language problem that
|
|
121
|
+
lexical retrieval cannot solve. It needs the semantic layer in a later workstream. The suite
|
|
122
|
+
segments by language so the gap stays measured instead of assumed.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "retrieval-benchmark-baseline",
|
|
3
|
+
"schemaVersion": 1,
|
|
4
|
+
"contentHash": "1fd663a565dfc9c1e19a12139d45f11a7a628ad5d0fec4e549c6148056c3ac3a",
|
|
5
|
+
"contentHashAlgo": "sha256-normalized-v1",
|
|
6
|
+
"suite": {
|
|
7
|
+
"name": "doc-bridge-retrieval-v1",
|
|
8
|
+
"caseCount": 60,
|
|
9
|
+
"contentHash": "8f789470ce0fca41154474191fed0163869518b278f7c91d6b5370b919cde072"
|
|
10
|
+
},
|
|
11
|
+
"metrics": {
|
|
12
|
+
"caseCount": 60,
|
|
13
|
+
"hitAt1": 0.783333,
|
|
14
|
+
"hitAt3": 0.883333,
|
|
15
|
+
"meanReciprocalRank": 0.827593,
|
|
16
|
+
"meanContextBytes": 428,
|
|
17
|
+
"meanApproxTokens": 108,
|
|
18
|
+
"zeroResultRate": 0.066667,
|
|
19
|
+
"tokenMethod": "approximate"
|
|
20
|
+
},
|
|
21
|
+
"approval": {
|
|
22
|
+
"approvedAt": "2026-09-14T20:12:19.833Z",
|
|
23
|
+
"approvedBy": "Emerson Braun",
|
|
24
|
+
"reason": "KR-06 ranks the retrieval projection of the snapshot with graph signals; hit@3 83.3% to 88.3%",
|
|
25
|
+
"resultHash": "77dbbd006ede58746913fdaf7741df48856daf5c09815c81eaac187eb9377399",
|
|
26
|
+
"indexHash": "c5600283b4c6cf5fc2546d0f8a2eb8158dcdd0e670bd1a7a070c68484dad709a"
|
|
27
|
+
}
|
|
28
|
+
}
|