@agentskit/doc-bridge 1.7.45 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +488 -0
- package/CONTRIBUTING.md +23 -0
- package/README.md +87 -40
- package/SECURITY.md +7 -0
- package/action.yml +1 -1
- package/bin/ak-docs.js +2 -2
- package/bin/ak-verify.js +13 -7
- package/dist/cli/program.d.ts +3 -1
- package/dist/cli/program.js +15871 -6218
- package/dist/cli/program.js.map +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +95 -6
- package/dist/config/index.js.map +1 -1
- package/dist/{index-BUL0q7s8.d.ts → index-7wYGbllW.d.ts} +134 -1
- package/dist/index.d.ts +9880 -1062
- package/dist/index.js +16143 -6127
- package/dist/index.js.map +1 -1
- package/docs/MARKETPLACE.md +1 -1
- package/docs/PRD-documentation-efficiency-study.md +406 -0
- package/docs/PRD-knowledge-retrieval-and-enrichment.md +466 -0
- package/docs/adr/0002-documentation-audit-boundary.md +22 -0
- package/docs/adr/0003-study-protocol-and-historical-evidence.md +40 -0
- package/docs/adr/0004-controlled-study-runner.md +25 -0
- package/docs/adr/0005-documentation-quality-and-criticality.md +20 -0
- package/docs/adr/0006-registry-semantic-grounding.md +20 -0
- package/docs/adr/0007-longitudinal-study-metrics.md +21 -0
- package/docs/adr/0008-study-verification-boundary.md +21 -0
- package/docs/adr/0009-study-provider-cli-adapter.md +25 -0
- package/docs/agent-corpus/INDEX.md +12 -1
- package/docs/agent-corpus/OVERVIEW.md +25 -0
- package/docs/agent-corpus/chat.md +5 -1
- package/docs/agent-corpus/cli.md +16 -0
- package/docs/agent-corpus/conformance.md +12 -0
- package/docs/agent-corpus/doc-bridge.md +47 -0
- package/docs/agent-corpus/doctor.md +8 -0
- package/docs/agent-corpus/gates.md +4 -0
- package/docs/agent-corpus/mcp.md +13 -0
- package/docs/agent-corpus/memory.md +4 -0
- package/docs/agent-corpus/query.md +33 -0
- package/docs/bench/README.md +122 -0
- package/docs/bench/retrieval-baseline-v1.json +28 -0
- package/docs/bench/retrieval-suite-v1.json +1033 -0
- package/docs/chat-and-rag.md +3 -2
- package/docs/for-agents.md +9 -1
- package/docs/getting-started.md +4 -11
- package/docs/guides/gate-ci.md +11 -1
- package/docs/guides/install-and-run.md +9 -65
- package/docs/index.md +22 -1
- package/docs/knowledge-engine-runbook.md +38 -3
- package/docs/landing/assets/context-payload-reduction.svg +21 -0
- package/docs/landing/assets/controlled-ab-comparison.svg +30 -0
- package/docs/landing/index.html +119 -5
- package/docs/loop-workflow.md +117 -0
- package/docs/mcp.md +6 -1
- package/docs/parity/public-claims-v1.json +145 -0
- package/docs/playbook/doc-bridge-pattern.md +1 -1
- package/docs/query.md +90 -2
- package/docs/recipes/index-pipeline.md +1 -1
- package/docs/schemas/agent-handoff-v1.md +15 -0
- package/docs/schemas/doc-bridge-index-v1.md +65 -0
- package/docs/spec/benchmark-v1.md +33 -1
- package/docs/spec/cli.md +30 -10
- package/docs/spec/config-v1.md +157 -8
- package/docs/spec/documentation-audit-v1.md +61 -0
- package/docs/spec/enrichment-overlay-v1.md +241 -0
- package/docs/spec/graph-signals-v1.md +92 -0
- package/docs/spec/incremental-scan-v1.md +102 -0
- package/docs/spec/markdown-analyzer-v1.md +73 -0
- package/docs/spec/mcp-knowledge-tools-v1.md +147 -0
- package/docs/spec/measured-enrichment-v1.md +229 -0
- package/docs/spec/public-parity-v1.md +119 -0
- package/docs/spec/registry-agents.md +6 -0
- package/docs/spec/render-v1.md +122 -0
- package/docs/spec/retrieval-index-v1.md +164 -0
- package/docs/spec/study-metrics-v1.md +58 -0
- package/docs/spec/study-protocol-v1.md +46 -0
- package/docs/spec/study-provider-cli-v1.md +116 -0
- package/docs/spec/study-runner-v1.md +35 -0
- package/docs/spec/study-task-suite-v1.md +41 -0
- package/docs/spec/study-verification-v1.md +40 -0
- package/docs/study/README.md +84 -0
- package/docs/study/ab-adjudicated-cost-analysis-v1.md +29 -0
- package/docs/study/ab-adjudicated-cost-plan-v1.json +33 -0
- package/docs/study/ab-adjudicated-cost-plan-v2-v1.json +33 -0
- package/docs/study/ab-adjudicated-cost-result-v1.json +80 -0
- package/docs/study/ab-baseline-analysis-v1.md +21 -0
- package/docs/study/ab-baseline-plan-v1.json +33 -0
- package/docs/study/ab-baseline-recovery-plan-v1.json +33 -0
- package/docs/study/ab-baseline-result-v1.json +79 -0
- package/docs/study/documentation-audit-round-2026-08-31.json +183 -0
- package/docs/study/historical-evidence-v1.json +252 -0
- package/docs/study/observation-ledger-v1.json +30632 -0
- package/docs/study/phase3-task-coverage-v1.json +34 -0
- package/docs/study/phase4-public-pilot-ledger-v1.json +1344 -0
- package/docs/study/phase4-public-pilot-result-v1.json +52 -0
- package/docs/study/phase4-public-pilot-run-plan-v1.json +26 -0
- package/docs/study/phase4-public-pilot-task-suite-v1.json +71 -0
- package/docs/study/pilot-round-2026-08-31.json +46 -0
- package/docs/study/protocol-v1.json +90 -0
- package/docs/study/publication-gate-v1.md +45 -0
- package/docs/study/quality-scorecard-cycle-plan.md +545 -0
- package/docs/study/quality-scorecard-v1.json +38 -0
- package/docs/study/round-1-adjudicated-smoke-v1.json +30642 -0
- package/docs/study/round-1-instrumentation-plan-v1.md +39 -0
- package/docs/study/round-2-expanded-adjudication-v1.json +91 -0
- package/docs/study/round-2-expanded-validation-v1.md +58 -0
- package/docs/study/round-3-evidence-contract-v1.json +75 -0
- package/docs/study/round-3-evidence-contract-v1.md +57 -0
- package/docs/study/round-4-confirmation-v1.json +75 -0
- package/docs/study/round-4-confirmation-v1.md +55 -0
- package/docs/study/run-plan-v1.json +33 -0
- package/docs/study/semantic-adjudication-cycle-8.md +20 -0
- package/docs/study/task-suite-v1.json +96 -0
- package/docs/study/token-efficiency-plan-v1.md +337 -0
- package/docs/study/token-efficiency-protocol-v2.json +62 -0
- package/docs/study/verification-binding-v1.json +27 -0
- package/docs/validation-cycle-plan.md +14 -0
- package/docs/verification-harness.md +11 -6
- package/ecosystem-claims.json +2 -2
- package/ecosystem-upstream.json +2 -2
- package/ecosystem.json +4 -4
- package/mcpb/manifest.json +9 -1
- package/package.json +34 -15
- package/scripts/check-ecosystem-upstream.mjs +34 -6
- package/skills/doc-bridge-handoff/fixtures/synthetic-repo/docs/for-agents/packages/payments.md +7 -0
- package/skills/doc-bridge-handoff/scripts/resolve-handoff.mjs +1 -1
- package/src/agents/registry-adapter.ts +192 -24
- package/src/audit/documentation.ts +513 -0
- package/src/bench/baseline.ts +198 -0
- package/src/bench/overlay-delta.ts +139 -0
- package/src/bench/retrieval.ts +319 -0
- package/src/budget/compile.ts +91 -0
- package/src/budget/sections.ts +70 -0
- package/src/cli/program.ts +684 -74
- package/src/cli/usage.ts +71 -0
- package/src/config/defaults.ts +1 -0
- package/src/config/index.ts +4 -0
- package/src/config/schema.ts +126 -2
- package/src/conformance/documentation-standard-v1.ts +8 -6
- package/src/discovery/areas.ts +197 -0
- package/src/discovery/documentation.ts +211 -5
- package/src/discovery/identity.ts +24 -0
- package/src/discovery/incremental.ts +314 -0
- package/src/discovery/inputs.ts +110 -0
- package/src/discovery/markdown.ts +481 -0
- package/src/discovery/repository.ts +485 -101
- package/src/doctor/run-doctor.ts +244 -12
- package/src/enrich/approvals.ts +190 -0
- package/src/enrich/cache.ts +93 -0
- package/src/enrich/context-pack.ts +272 -0
- package/src/enrich/overlay.ts +255 -0
- package/src/enrich/review.ts +106 -0
- package/src/enrich/stage.ts +374 -0
- package/src/enrich/stats.ts +100 -0
- package/src/enrich/validate.ts +410 -0
- package/src/federation/llms.ts +2 -4
- package/src/findings/report.ts +103 -0
- package/src/graph/build.ts +356 -0
- package/src/graph/memory.ts +208 -0
- package/src/index-builder/build-handoffs.ts +22 -11
- package/src/index-builder/build-index.ts +132 -3
- package/src/index-builder/llms-txt.ts +48 -8
- package/src/index-builder/project-corpus.ts +111 -0
- package/src/index.ts +630 -2
- package/src/lib/fuzzy-match.ts +235 -0
- package/src/mcp/knowledge.ts +554 -0
- package/src/mcp/server.ts +113 -18
- package/src/metrics/benchmark.ts +21 -0
- package/src/parity/check.ts +309 -0
- package/src/parity/claims.ts +259 -0
- package/src/parity/resolve.ts +160 -0
- package/src/query/handoff.ts +326 -0
- package/src/query/load-index.ts +53 -1
- package/src/query/query.ts +92 -59
- package/src/query/search.ts +289 -92
- package/src/query/text.ts +155 -0
- package/src/reconciliation/reconcile.ts +101 -10
- package/src/render/data.ts +356 -0
- package/src/render/engine.ts +398 -0
- package/src/render/generated.ts +77 -0
- package/src/render/render.ts +209 -0
- package/src/render/template-source.ts +52 -0
- package/src/render/templates.ts +289 -0
- package/src/report/html.ts +2 -2
- package/src/retrieval/bm25.ts +161 -0
- package/src/retrieval/project.ts +495 -0
- package/src/retrieval/rank.ts +383 -0
- package/src/retrieval/weights.ts +39 -0
- package/src/retriever/doc-bridge-retriever.ts +100 -15
- package/src/rules/engine.ts +30 -10
- package/src/schemas/agent-handoff.ts +56 -0
- package/src/schemas/budget.ts +37 -0
- package/src/schemas/doc-bridge-index.ts +53 -2
- package/src/schemas/enrichment.ts +369 -0
- package/src/schemas/json-schemas.ts +39 -2
- package/src/schemas/knowledge.ts +14 -1
- package/src/schemas/retrieval-index.ts +152 -0
- package/src/shims/graphology.d.ts +91 -0
- package/src/study/adjudication.ts +196 -0
- package/src/study/execution.ts +350 -0
- package/src/study/expectations.ts +219 -0
- package/src/study/metrics.ts +467 -0
- package/src/study/protocol.ts +271 -0
- package/src/study/provider-cli.ts +115 -0
- package/src/study/provider-telemetry.ts +47 -0
- package/src/study/quality-scorecard.ts +164 -0
- package/src/study/runner.ts +461 -0
- package/src/study/task-suite.ts +321 -0
- package/src/study/verification.ts +134 -0
- package/src/validate.ts +1 -4
- package/src/version.ts +1 -1
- package/src/workflow/engine.ts +36 -11
- package/scripts/verification-harness.mjs +0 -482
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "study-protocol",
|
|
3
|
+
"schemaVersion": 1,
|
|
4
|
+
"protocolVersion": "token-efficiency-v2",
|
|
5
|
+
"title": "Token efficiency and correct-action study",
|
|
6
|
+
"evidenceClasses": ["historical", "controlled"],
|
|
7
|
+
"repositories": [
|
|
8
|
+
{ "id": "consumer-01", "visibility": "anonymized", "role": "consumer" },
|
|
9
|
+
{ "id": "consumer-02", "visibility": "anonymized", "role": "consumer" },
|
|
10
|
+
{ "id": "consumer-03", "visibility": "anonymized", "role": "consumer" },
|
|
11
|
+
{ "id": "consumer-04", "visibility": "anonymized", "role": "consumer" },
|
|
12
|
+
{ "id": "consumer-05", "visibility": "anonymized", "role": "consumer" },
|
|
13
|
+
{ "id": "consumer-06", "visibility": "anonymized", "role": "consumer" }
|
|
14
|
+
],
|
|
15
|
+
"taskCategories": ["discovery", "architecture", "documentation", "implementation"],
|
|
16
|
+
"models": [
|
|
17
|
+
{ "id": "low-cost-model", "role": "low-cost", "status": "pinned", "provider": "codex", "model": "gpt-5.6-sol", "version": "codex-cli-0.149.0", "promptContractHash": "20897bddf03704d26f975003bf3ba0d4d8cb38559b382603d7a03cb17b487a7c" },
|
|
18
|
+
{ "id": "reference-model", "role": "reference", "status": "pinned", "provider": "codex", "model": "gpt-5.6-luna", "version": "codex-cli-0.149.0", "promptContractHash": "a0a90a4141327fef23e2d7ef99da79cd44cfe66fa65d7757484e95629b3fe7bd" }
|
|
19
|
+
],
|
|
20
|
+
"scenarios": [
|
|
21
|
+
{ "id": "repository-only", "label": "Repository only", "source": "repository-only", "modelIds": ["low-cost-model", "reference-model"], "requiresHumanApproval": true },
|
|
22
|
+
{ "id": "deterministic-doc-bridge", "label": "Deterministic Doc Bridge", "source": "deterministic-doc-bridge", "modelIds": ["low-cost-model", "reference-model"], "requiresHumanApproval": true },
|
|
23
|
+
{ "id": "registry-assisted", "label": "Doc Bridge plus Registry agent", "source": "doc-bridge-registry-agent", "modelIds": ["low-cost-model", "reference-model"], "registryAgent": "configured", "requiresHumanApproval": true }
|
|
24
|
+
],
|
|
25
|
+
"metrics": [
|
|
26
|
+
{ "id": "tokens-to-correct-action", "family": "discovery", "unit": "tokens", "source": "provider", "required": true, "description": "Provider tokens consumed until a correct evidence-backed action passes applicable acceptance checks." },
|
|
27
|
+
{ "id": "correct-action-rate", "family": "task", "unit": "ratio", "source": "adjudicator", "required": true, "description": "Proportion of eligible tasks with correct evidence and passing acceptance checks." },
|
|
28
|
+
{ "id": "context-tokens-p95", "family": "discovery", "unit": "tokens", "source": "provider", "required": true, "description": "P95 provider tokens in Doc Bridge context before task execution." },
|
|
29
|
+
{ "id": "time-to-first-evidence", "family": "discovery", "unit": "milliseconds", "source": "runner", "required": true, "description": "Elapsed time until the first required evidence item is found." },
|
|
30
|
+
{ "id": "time-to-correct-action-p95", "family": "discovery", "unit": "milliseconds", "source": "runner", "required": true, "description": "P95 elapsed time among tasks reaching a correct action." },
|
|
31
|
+
{ "id": "tool-calls-per-correct-action", "family": "operations", "unit": "count", "source": "runner", "required": true, "description": "Tool calls used by tasks that reach a correct action." },
|
|
32
|
+
{ "id": "clarification-rate", "family": "task", "unit": "ratio", "source": "runner", "required": true, "description": "Proportion of tasks requiring human clarification." },
|
|
33
|
+
{ "id": "rework-rate", "family": "task", "unit": "ratio", "source": "runner", "required": true, "description": "Proportion of tasks requiring a corrective second attempt." },
|
|
34
|
+
{ "id": "evidence-precision", "family": "task", "unit": "ratio", "source": "adjudicator", "required": true, "description": "Proportion of returned evidence relevant to the declared task." },
|
|
35
|
+
{ "id": "evidence-recall", "family": "task", "unit": "ratio", "source": "adjudicator", "required": true, "description": "Proportion of required evidence found by the workflow." },
|
|
36
|
+
{ "id": "safety-rate", "family": "task", "unit": "ratio", "source": "adjudicator", "required": true, "description": "Proportion of tasks with no unsafe or unauthorized action." },
|
|
37
|
+
{ "id": "total-cost-per-correct-action", "family": "cost", "unit": "currency", "source": "runner", "required": true, "description": "Doc Bridge, agent, and adjudicator cost for successful tasks." }
|
|
38
|
+
],
|
|
39
|
+
"outcomes": [
|
|
40
|
+
{ "id": "protocol-integrity", "statement": "The token-efficiency protocol is valid, versioned, and reproducible.", "checks": [{ "id": "protocol-validation", "command": "ak-docs study protocol docs/study/token-efficiency-protocol-v2.json --json" }] },
|
|
41
|
+
{ "id": "historical-separation", "statement": "Historical protocol and result artifacts remain unchanged and separately classified.", "checks": [{ "id": "historical-immutability", "command": "git diff --exit-code -- docs/study/protocol-v1.json docs/study/ab-baseline-result-v1.json docs/study/ab-adjudicated-cost-result-v1.json" }] },
|
|
42
|
+
{ "id": "metric-integrity", "statement": "Token methods, missing data, and correct-action nullability remain explicit.", "checks": [{ "id": "metric-contract-tests", "command": "pnpm exec vitest run tests/study-protocol.test.ts" }] },
|
|
43
|
+
{ "id": "privacy", "statement": "Publication-bound study artifacts contain no private repository data or raw agent material.", "checks": [{ "id": "privacy-scan", "command": "pnpm build && node scripts/study-privacy-gate.mjs docs/study" }] },
|
|
44
|
+
{ "id": "documentation-integrity", "statement": "The published study documentation remains type-safe and buildable.", "checks": [{ "id": "docs-typecheck", "command": "pnpm docs:typecheck" }] }
|
|
45
|
+
],
|
|
46
|
+
"budget": { "maxTokensPerTask": 400000, "maxRuntimeMsPerTask": 180000, "maxRuns": 288, "maxNetworkRequests": 0 },
|
|
47
|
+
"privacy": { "mode": "anonymized", "forbiddenFields": ["repository-content", "paths", "prompts", "credentials", "private-identifiers", "raw-agent-responses"], "publicationRequiresHumanReview": true },
|
|
48
|
+
"stopping": {
|
|
49
|
+
"minControlledRounds": 3,
|
|
50
|
+
"consecutiveNoMaterialImprovementRounds": 3,
|
|
51
|
+
"targets": [
|
|
52
|
+
{ "metricId": "tokens-to-correct-action", "direction": "decrease", "threshold": 0.25 },
|
|
53
|
+
{ "metricId": "time-to-correct-action-p95", "direction": "decrease", "threshold": 0.2 },
|
|
54
|
+
{ "metricId": "correct-action-rate", "direction": "no-regression", "threshold": 0 },
|
|
55
|
+
{ "metricId": "evidence-precision", "direction": "no-regression", "threshold": 0 },
|
|
56
|
+
{ "metricId": "evidence-recall", "direction": "no-regression", "threshold": 0 },
|
|
57
|
+
{ "metricId": "safety-rate", "direction": "no-regression", "threshold": 0 }
|
|
58
|
+
]
|
|
59
|
+
},
|
|
60
|
+
"contentHash": "97202235c4dbd556f41fe1aa08663d35b16d516de827aebae73b136bb6a538f6",
|
|
61
|
+
"contentHashAlgo": "sha256-normalized-v1"
|
|
62
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "controlled-study-verification",
|
|
3
|
+
"schemaVersion": 1,
|
|
4
|
+
"verificationRunId": "verification-fixture-01",
|
|
5
|
+
"sourceRevisionHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
6
|
+
"protocolHash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
7
|
+
"configurationHash": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
|
|
8
|
+
"baselineHash": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
9
|
+
"artifactHashes": {
|
|
10
|
+
"metrics": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
|
|
11
|
+
"ledger": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
|
|
12
|
+
},
|
|
13
|
+
"privacy": {
|
|
14
|
+
"mode": "anonymized",
|
|
15
|
+
"checkedArtifactCount": 2,
|
|
16
|
+
"forbiddenMatchCount": 0,
|
|
17
|
+
"publicationReview": "approved"
|
|
18
|
+
},
|
|
19
|
+
"budget": {
|
|
20
|
+
"maxTokens": 1000,
|
|
21
|
+
"usedTokens": 800,
|
|
22
|
+
"maxRuntimeMs": 2000,
|
|
23
|
+
"usedRuntimeMs": 1500
|
|
24
|
+
},
|
|
25
|
+
"contentHashAlgo": "sha256-normalized-v1",
|
|
26
|
+
"contentHash": "1c94cb0a2b70e18cffc99f4317d772760dfd9f29039009ece86d2031078bb781"
|
|
27
|
+
}
|
|
@@ -158,6 +158,20 @@ the evidence and requires a new workflow and verification run.
|
|
|
158
158
|
- issue/PR tracking is updated only after authorization and includes the exact run ID;
|
|
159
159
|
- task-owned temporary artifacts are cleaned while user-owned or ambiguous artifacts remain untouched;
|
|
160
160
|
- structural decisions are documented in the relevant ADR/RFC;
|
|
161
|
+
|
|
162
|
+
## Documentation quality audit baseline — 2026-08-29
|
|
163
|
+
|
|
164
|
+
The first deterministic audit run across the five consumer repositories used the same local Doc Bridge source revision. Repository names are intentionally anonymized in this study artifact; the evidence remains in each repository's local verification state.
|
|
165
|
+
|
|
166
|
+
| Consumer | Documents | Package coverage | Title rate | Example rate | Exact duplicate groups | Structure gaps | Contradictions | Stale |
|
|
167
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
168
|
+
| Consumer 01 | 4,098 | 82/83 (98.8%) | 32.4% | 15.0% | 23 | 2,712 | 0 | 0 |
|
|
169
|
+
| Consumer 02 | 1,537 | 26/26 (100%) | 71.2% | 42.0% | 2 | 0 | 0 | 0 |
|
|
170
|
+
| Consumer 03 | 169 | 20/21 (95.2%) | 96.4% | 59.8% | 0 | 1,083 | 0 | 0 |
|
|
171
|
+
| Consumer 04 | 163 | 0/2 (0%) | 98.2% | 58.3% | 0 | 512 | 0 | 0 |
|
|
172
|
+
| Consumer 05 | 416 | 0/1 (0%) | 99.8% | 23.6% | 0 | 5,118 | 0 | 0 |
|
|
173
|
+
|
|
174
|
+
These numbers are a baseline, not a quality score. The audit reports semantic contradiction, unnecessary content, and generated-document freshness as `not-analyzed` until a configured AgentsKit Registry agent or human review supplies evidence. Future rounds must retain the same metrics and compare source revision, configuration hash, audit hash, and verification run ID.
|
|
161
175
|
- final report states residual risks and the next human action; enterprise readiness is not claimed while any required gate is pending.
|
|
162
176
|
|
|
163
177
|
## Initial baseline to collect
|
|
@@ -5,18 +5,21 @@ description: Fail-closed, evidence-backed verification for humans and agents.
|
|
|
5
5
|
|
|
6
6
|
# Verification harness
|
|
7
7
|
|
|
8
|
-
`ak-verify` is the executable completion gate for work that must be proven, not merely compiled.
|
|
8
|
+
`ak-verify` delegates to the provider-neutral `@agentskit/harness@0.9.0`. It is the executable completion gate for work that must be proven, not merely compiled. The Doc Bridge contract enables YOLO autonomy for intermediate verification, while required UI review and external tracking remain explicit gates.
|
|
9
9
|
|
|
10
10
|
```bash
|
|
11
|
-
ak-verify
|
|
11
|
+
ak-verify plan approved --allow-dirty --config .codex/verification.json --json
|
|
12
|
+
ak-verify start --config .codex/verification.json --json
|
|
13
|
+
ak-verify verify --config .codex/verification.json --json
|
|
12
14
|
ak-verify status --config .codex/verification.json --json
|
|
13
15
|
ak-verify approve <run-id> approved --by human --config .codex/verification.json
|
|
14
16
|
ak-verify authorize <run-id> approved --by human --config .codex/verification.json
|
|
15
|
-
ak-verify baseline replace benchmarks/new.json approved --by human --config .codex/verification.json
|
|
16
17
|
ak-verify clean --periodic --config .codex/verification.json
|
|
17
18
|
```
|
|
18
19
|
|
|
19
|
-
The contract is JSON so it works without adding a YAML runtime. It declares the artifact surfaces that apply to the run, executable checks, explicit non-applicable reasons, the verification profile, and tracking policy.
|
|
20
|
+
The contract is JSON so it works without adding a YAML runtime. It declares the artifact surfaces that apply to the run, executable checks, explicit non-applicable reasons, the verification profile, YOLO autonomy, and tracking policy. This repository stores current official-harness artifacts under `.codex/verification-0.9/`; older `.codex/verification/` artifacts are historical and are not reused by the 0.9.0 CLI.
|
|
21
|
+
|
|
22
|
+
For audit-only work, set `mode: "discovery"`. Checks remain executable and their failures remain in the evidence ledger, but they are non-blocking by default. Mark only the minimum inventory/evidence-integrity checks with `blocking: true` when a failure must prevent advancing to the next audit unit. This keeps discovery complete without treating an observed quality defect as an implementation gate.
|
|
20
23
|
|
|
21
24
|
## Global policy and project contract
|
|
22
25
|
|
|
@@ -34,6 +37,8 @@ set of repository-specific checks fits every project.
|
|
|
34
37
|
|
|
35
38
|
Before implementation, the human intent and acceptance criteria must be explicit. Every criterion must map to an executable check and its expected evidence. If a criterion is not mapped, the run is `CLARIFYING` or `BLOCKED`; a project may not silently shrink the scope to the checks that are easiest to run.
|
|
36
39
|
|
|
40
|
+
An explicit human approval of the goal or plan authorizes the intermediate implementation, verification, tracking, and cleanup steps within that same contract. The harness must not repeatedly request approval for those intermediate steps. This approval does not waive evidence: failed, missing, stale, or newly out-of-scope work still blocks completion, and material artifact changes still require the applicable evidence review.
|
|
41
|
+
|
|
37
42
|
## States
|
|
38
43
|
|
|
39
44
|
`CLARIFYING` → `PLANNED` → `VERIFYING` → `AWAITING_HUMAN_APPROVAL` / `AWAITING_AUTHORIZATION` → `COMPLETE`.
|
|
@@ -46,7 +51,7 @@ The seven applicability surfaces are `logic`, `cli`, `mcp`, `ui`, `docs`, `endpo
|
|
|
46
51
|
|
|
47
52
|
## Evidence and recovery
|
|
48
53
|
|
|
49
|
-
Runs live under `.codex/verification/runs/<run-id>/run.json`. The latest pointer is
|
|
54
|
+
Runs live under the configured `stateDir`, currently `.codex/verification-0.9/runs/<run-id>/run.json`. The latest pointer is in that same state directory. Commands are captured with exit code, duration, stdout, stderr, source revision, configuration hash, and input hash. Re-running an unchanged pending or completed run is idempotent; changed source or contract creates a new run.
|
|
50
55
|
|
|
51
56
|
Checks may emit one final JSON line with `status` set to `passed`, `failed`, or `pending-human-review`. Structured `failed` evidence blocks the run even when the process exits with code 0; structured pending evidence remains explicitly awaiting human approval. This prevents a visual checker from being mistaken for a successful verification merely because it launched.
|
|
52
57
|
|
|
@@ -62,7 +67,7 @@ When a report is shared outside its repository, configure `report.privacy: 'anon
|
|
|
62
67
|
|
|
63
68
|
When `measurement.required` is enabled, the named required check must emit structured evidence with `status: "passed"`, a numeric `metrics` object, a `baselineHash`, and an empty `regressions` array. Missing or regressed measurements block completion. Baselines are explicit, versioned artifacts and are never updated implicitly by a verification run.
|
|
64
69
|
|
|
65
|
-
Baseline replacement is
|
|
70
|
+
Baseline replacement is outside the `@agentskit/harness@0.9.0` verification CLI. Doc Bridge study tooling must treat baseline replacement as a separate, explicit, human-authorized operation and record the new artifact hash, actor, intent, and timestamp in the study evidence ledger. A normal verification run never replaces a baseline.
|
|
66
71
|
|
|
67
72
|
The run JSON exposes `profile`, `profilePolicy`, `applicability`, `exemptions`, `checks`, `evidenceReferences`, `metrics`, `transitions`, `sourceRevision`, `contractHash`, `outputHash`, and the exact `runId`. Approval records are bound to the input, source, contract, and output hashes of that run.
|
|
68
73
|
|
package/ecosystem-claims.json
CHANGED
|
@@ -95,7 +95,7 @@
|
|
|
95
95
|
},
|
|
96
96
|
{
|
|
97
97
|
"id": "memory-backends",
|
|
98
|
-
"value":
|
|
98
|
+
"value": 18,
|
|
99
99
|
"noun": "memory backends",
|
|
100
100
|
"evidence": {
|
|
101
101
|
"type": "repository-derivation",
|
|
@@ -169,7 +169,7 @@
|
|
|
169
169
|
"productId": "code-review",
|
|
170
170
|
"source": {
|
|
171
171
|
"type": "repository",
|
|
172
|
-
"repo": "AgentsKit-io/code-review
|
|
172
|
+
"repo": "AgentsKit-io/code-review"
|
|
173
173
|
},
|
|
174
174
|
"verification": "declared",
|
|
175
175
|
"claims": []
|
package/ecosystem-upstream.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"repository": "AgentsKit-io/agentskit",
|
|
4
4
|
"ref": "main",
|
|
5
5
|
"files": {
|
|
6
|
-
"ecosystem.json": "
|
|
7
|
-
"ecosystem-claims.json": "
|
|
6
|
+
"ecosystem.json": "21f28c132685f9984979afc83b66a61310e2bb79666d826662f7e8f234fbf36f",
|
|
7
|
+
"ecosystem-claims.json": "6e77a801d93e58e6580fe5d69d8899bf8d396bffba367f3a705fa751aea83a06"
|
|
8
8
|
}
|
|
9
9
|
}
|
package/ecosystem.json
CHANGED
|
@@ -327,12 +327,12 @@
|
|
|
327
327
|
"role": "verification",
|
|
328
328
|
"promise": "Run deep, low-noise review with the model already in use.",
|
|
329
329
|
"maturity": "alpha",
|
|
330
|
-
"repo": "AgentsKit-io/code-review
|
|
330
|
+
"repo": "AgentsKit-io/code-review",
|
|
331
331
|
"accent": "#F97316",
|
|
332
332
|
"surfaces": {
|
|
333
|
-
"home": "https://github.com/AgentsKit-io/code-review
|
|
334
|
-
"docs": "https://github.com/AgentsKit-io/code-review
|
|
335
|
-
"llms": "https://raw.githubusercontent.com/AgentsKit-io/code-review
|
|
333
|
+
"home": "https://github.com/AgentsKit-io/code-review",
|
|
334
|
+
"docs": "https://github.com/AgentsKit-io/code-review#readme",
|
|
335
|
+
"llms": "https://raw.githubusercontent.com/AgentsKit-io/code-review/main/llms.txt",
|
|
336
336
|
"documentation": "repository",
|
|
337
337
|
"chat": "none"
|
|
338
338
|
},
|
package/mcpb/manifest.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"manifest_version": "0.3",
|
|
3
3
|
"name": "doc-bridge",
|
|
4
4
|
"display_name": "Doc Bridge",
|
|
5
|
-
"version": "1.
|
|
5
|
+
"version": "1.10.0",
|
|
6
6
|
"description": "Deterministic repository handoffs for coding agents, running locally without an LLM or API key.",
|
|
7
7
|
"long_description": "Doc Bridge turns a repository's own documentation and ownership metadata into deterministic handoffs: where an agent should start, which paths it may edit, which checks it must run, and when a human must take over. The local connector exposes the same read-only contract available through Doc Bridge CLI and CI.",
|
|
8
8
|
"author": {
|
|
@@ -87,6 +87,14 @@
|
|
|
87
87
|
{
|
|
88
88
|
"name": "docbridge.proposals",
|
|
89
89
|
"description": "Create, inspect, approve, and apply human-gated Doc Bridge proposals."
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"name": "knowledge.search",
|
|
93
|
+
"description": "Rank every indexed entity for a query, optionally by kind, with an explanation and within a token budget."
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"name": "knowledge.lookup",
|
|
97
|
+
"description": "Return an entity with its neighbours, documents, handoff, open diagnostics and evidence, within a token budget."
|
|
90
98
|
}
|
|
91
99
|
],
|
|
92
100
|
"tools_generated": false,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentskit/doc-bridge",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0",
|
|
4
4
|
"mcpName": "io.github.AgentsKit-io/doc-bridge",
|
|
5
5
|
"description": "Human↔agent documentation bridge — deterministic handoffs, doc-site links, memory→docs, optional AgentsKit RAG/chat.",
|
|
6
6
|
"type": "module",
|
|
@@ -25,7 +25,6 @@
|
|
|
25
25
|
"bin",
|
|
26
26
|
"action.yml",
|
|
27
27
|
"scripts/prepare.mjs",
|
|
28
|
-
"scripts/verification-harness.mjs",
|
|
29
28
|
"scripts/report-visual-check.mjs",
|
|
30
29
|
"scripts/check-ecosystem-upstream.mjs",
|
|
31
30
|
"ecosystem.json",
|
|
@@ -74,11 +73,25 @@
|
|
|
74
73
|
"access": "public"
|
|
75
74
|
},
|
|
76
75
|
"dependencies": {
|
|
76
|
+
"@agentskit/harness": "0.11.0",
|
|
77
77
|
"github-slugger": "^2.0.0",
|
|
78
|
+
"graphology": "~0.26.0",
|
|
79
|
+
"graphology-communities-louvain": "~2.0.2",
|
|
80
|
+
"graphology-dag": "~0.4.1",
|
|
81
|
+
"graphology-metrics": "~2.4.2",
|
|
82
|
+
"graphology-shortest-path": "~2.1.0",
|
|
83
|
+
"knap": "~0.5.1",
|
|
84
|
+
"mdast-util-to-string": "^4.0.0",
|
|
78
85
|
"mermaid": "^11.17.2",
|
|
79
86
|
"minimatch": "^10.2.6",
|
|
87
|
+
"remark-frontmatter": "^5.0.0",
|
|
88
|
+
"remark-gfm": "^4.0.1",
|
|
89
|
+
"remark-parse": "^11.0.0",
|
|
80
90
|
"typescript": "^6.0.3",
|
|
81
|
-
"
|
|
91
|
+
"unified": "^11.0.5",
|
|
92
|
+
"unist-util-visit": "^5.1.0",
|
|
93
|
+
"yaml": "^2.9.0",
|
|
94
|
+
"zod": "^4.5.4"
|
|
82
95
|
},
|
|
83
96
|
"peerDependencies": {
|
|
84
97
|
"@agentskit/adapters": ">=0.12.0 <1",
|
|
@@ -110,27 +123,28 @@
|
|
|
110
123
|
},
|
|
111
124
|
"devDependencies": {
|
|
112
125
|
"@agentskit/chat": "0.4.1",
|
|
113
|
-
"@agentskit/core": "1.12.
|
|
126
|
+
"@agentskit/core": "1.12.9",
|
|
114
127
|
"@agentskit/ink": "0.10.9",
|
|
115
|
-
"@agentskit/react": "0.
|
|
128
|
+
"@agentskit/react": "0.8.3",
|
|
116
129
|
"@anthropic-ai/mcpb": "2.1.2",
|
|
117
130
|
"@changesets/cli": "^3.0.1",
|
|
118
131
|
"@lhci/cli": "^0.15.1",
|
|
119
|
-
"@playwright/test": "1.
|
|
132
|
+
"@playwright/test": "1.63.0",
|
|
120
133
|
"@tailwindcss/postcss": "4.3.3",
|
|
134
|
+
"@types/mdast": "^4.0.4",
|
|
121
135
|
"@types/node": "^26.4.0",
|
|
122
|
-
"@types/react": "19.
|
|
123
|
-
"@types/react-dom": "19.
|
|
136
|
+
"@types/react": "19.3.0",
|
|
137
|
+
"@types/react-dom": "19.3.0",
|
|
124
138
|
"@vitest/coverage-v8": "4.1.11",
|
|
125
139
|
"esbuild": "^0.28.2",
|
|
126
|
-
"fumadocs-core": "16.15.
|
|
140
|
+
"fumadocs-core": "16.15.10",
|
|
127
141
|
"fumadocs-mdx": "15.4.0",
|
|
128
|
-
"fumadocs-ui": "16.15.
|
|
129
|
-
"lucide-react": "1.
|
|
130
|
-
"next": "16.3.
|
|
142
|
+
"fumadocs-ui": "16.15.10",
|
|
143
|
+
"lucide-react": "1.45.0",
|
|
144
|
+
"next": "16.3.5",
|
|
131
145
|
"proxy-agent": "8.0.2",
|
|
132
|
-
"react": "19.
|
|
133
|
-
"react-dom": "19.
|
|
146
|
+
"react": "19.3.0",
|
|
147
|
+
"react-dom": "19.3.0",
|
|
134
148
|
"tailwindcss": "4.3.3",
|
|
135
149
|
"tsup": "^8.5.0",
|
|
136
150
|
"vitest": "^4.1.11"
|
|
@@ -151,6 +165,7 @@
|
|
|
151
165
|
"test": "vitest run && pnpm test:cursor-plugin && pnpm test:claude-plugin && pnpm test:copilot-plugin && pnpm test:portable-skill",
|
|
152
166
|
"test:watch": "vitest",
|
|
153
167
|
"coverage": "vitest run --coverage",
|
|
168
|
+
"bench:retrieval": "node bin/ak-docs.js bench retrieval docs/bench/retrieval-suite-v1.json --baseline docs/bench/retrieval-baseline-v1.json --text",
|
|
154
169
|
"check:ecosystem-upstream": "node scripts/check-ecosystem-upstream.mjs",
|
|
155
170
|
"check:no-legacy-chat-imports": "node scripts/check-no-legacy-chat-imports.mjs",
|
|
156
171
|
"typecheck": "tsc --noEmit",
|
|
@@ -176,9 +191,13 @@
|
|
|
176
191
|
"docs:artifacts": "node scripts/build-docs-artifacts.mjs",
|
|
177
192
|
"docs:build": "pnpm build && node bin/ak-docs.js index && pnpm docs:artifacts && cd apps/docs && next build",
|
|
178
193
|
"docs:typecheck": "cd apps/docs && fumadocs-mdx && tsc --noEmit",
|
|
194
|
+
"docs:freshness": "node scripts/verify-generated-docs.mjs",
|
|
195
|
+
"docs:semantic-review": "node scripts/run-registry-semantic-review.mjs",
|
|
196
|
+
"docs:semantic-adjudication": "node scripts/run-semantic-adjudication.mjs",
|
|
197
|
+
"quality:scorecard": "pnpm build && node scripts/quality-scorecard-eval.mjs",
|
|
179
198
|
"docs:e2e": "playwright test",
|
|
180
199
|
"report:visual": "node scripts/report-visual-check.mjs",
|
|
181
|
-
"test:verification-harness": "node --
|
|
200
|
+
"test:verification-harness": "node bin/ak-verify.js doctor --config .codex/verification.json --json",
|
|
182
201
|
"docs:lighthouse": "lhci autorun",
|
|
183
202
|
"test:readme-standard": "node --test scripts/readme-standard.test.mjs",
|
|
184
203
|
"test:marketplace": "node --test scripts/marketplace-contract.test.mjs",
|
|
@@ -6,7 +6,12 @@ import { fileURLToPath } from 'node:url'
|
|
|
6
6
|
|
|
7
7
|
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|
8
8
|
const metadata = JSON.parse(readFileSync(join(root, 'ecosystem-upstream.json'), 'utf8'))
|
|
9
|
-
if (
|
|
9
|
+
if (
|
|
10
|
+
metadata.schemaVersion !== 1 ||
|
|
11
|
+
typeof metadata.repository !== 'string' ||
|
|
12
|
+
!/^[^/]+\/[^/]+$/.test(metadata.repository) ||
|
|
13
|
+
typeof metadata.ref !== 'string'
|
|
14
|
+
) {
|
|
10
15
|
throw new Error('Invalid ecosystem-upstream.json metadata.')
|
|
11
16
|
}
|
|
12
17
|
|
|
@@ -20,18 +25,39 @@ if (
|
|
|
20
25
|
throw new Error('Upstream metadata must contain SHA-256 digests for both canonical files.')
|
|
21
26
|
}
|
|
22
27
|
|
|
23
|
-
const fetchText = async (url) => {
|
|
28
|
+
const fetchText = async (url, apiUrl) => {
|
|
24
29
|
let lastError
|
|
25
|
-
for (let attempt = 1; attempt <=
|
|
30
|
+
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
26
31
|
try {
|
|
27
|
-
const response = await fetch(url, {
|
|
32
|
+
const response = await fetch(url, {
|
|
33
|
+
headers: { 'user-agent': 'doc-bridge-ecosystem-check' },
|
|
34
|
+
signal: AbortSignal.timeout(10_000),
|
|
35
|
+
})
|
|
28
36
|
if (!response.ok) throw new Error(`HTTP ${response.status}`)
|
|
29
37
|
return await response.text()
|
|
30
38
|
} catch (error) {
|
|
31
39
|
lastError = error
|
|
32
40
|
}
|
|
33
41
|
}
|
|
34
|
-
|
|
42
|
+
try {
|
|
43
|
+
const response = await fetch(apiUrl, {
|
|
44
|
+
headers: {
|
|
45
|
+
accept: 'application/vnd.github+json',
|
|
46
|
+
'user-agent': 'doc-bridge-ecosystem-check',
|
|
47
|
+
},
|
|
48
|
+
signal: AbortSignal.timeout(10_000),
|
|
49
|
+
})
|
|
50
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}`)
|
|
51
|
+
const payload = await response.json()
|
|
52
|
+
if (payload?.encoding !== 'base64' || typeof payload.content !== 'string') {
|
|
53
|
+
throw new Error('GitHub API response did not contain a base64 file.')
|
|
54
|
+
}
|
|
55
|
+
return Buffer.from(payload.content.replace(/\s+/g, ''), 'base64').toString('utf8')
|
|
56
|
+
} catch (error) {
|
|
57
|
+
const fallback = error instanceof Error ? error.message : String(error)
|
|
58
|
+
const primary = lastError instanceof Error ? lastError.message : String(lastError)
|
|
59
|
+
throw new Error(`Unable to verify ${url}: ${primary}; API fallback: ${fallback}`)
|
|
60
|
+
}
|
|
35
61
|
}
|
|
36
62
|
|
|
37
63
|
for (const file of requiredFiles) {
|
|
@@ -43,7 +69,9 @@ for (const file of requiredFiles) {
|
|
|
43
69
|
}
|
|
44
70
|
const url = new URL(`https://raw.githubusercontent.com/${metadata.repository}/${metadata.ref}/${file}`)
|
|
45
71
|
if (url.hostname !== 'raw.githubusercontent.com') throw new Error('Unexpected upstream host.')
|
|
46
|
-
const
|
|
72
|
+
const apiUrl = new URL(`https://api.github.com/repos/${metadata.repository}/contents/${file}`)
|
|
73
|
+
apiUrl.searchParams.set('ref', metadata.ref)
|
|
74
|
+
const upstream = await fetchText(url.href, apiUrl.href)
|
|
47
75
|
if (sha256(upstream) !== expectedDigest || upstream !== local) {
|
|
48
76
|
throw new Error(`${file} is stale against ${metadata.repository}@${metadata.ref}. Sync the canonical snapshot and digest.`)
|
|
49
77
|
}
|
package/skills/doc-bridge-handoff/fixtures/synthetic-repo/docs/for-agents/packages/payments.md
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
---
|
|
2
|
+
owner: fixture-maintainers
|
|
3
|
+
lifecycle: active
|
|
4
|
+
sourceOfTruth: skills/doc-bridge-handoff/fixtures/synthetic-repo
|
|
5
|
+
validationPath: pnpm test:portable-skill
|
|
6
|
+
---
|
|
7
|
+
|
|
1
8
|
# Payments package
|
|
2
9
|
|
|
3
10
|
This is public synthetic guidance for the portable skill compatibility test.
|