docguard-cli 0.39.0 → 0.40.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -20
- package/cli/commands/agent.mjs +47 -1
- package/cli/commands/explain.mjs +16 -0
- package/cli/commands/fix.mjs +13 -11
- package/cli/commands/generate.mjs +52 -18
- package/cli/commands/guard.mjs +13 -2
- package/cli/commands/mcp.mjs +22 -2
- package/cli/commands/score.mjs +13 -1
- package/cli/commands/sync.mjs +20 -7
- package/cli/commands/verify.mjs +65 -2
- package/cli/config.mjs +3 -0
- package/cli/docguard.mjs +33 -12
- package/cli/evidence/adapters.mjs +200 -0
- package/cli/evidence/evaluate.mjs +185 -0
- package/cli/evidence/manifest.mjs +194 -0
- package/cli/evidence/markdown.mjs +107 -0
- package/cli/findings.mjs +31 -0
- package/cli/release-pr-policy.mjs +107 -0
- package/cli/repository-root.mjs +159 -0
- package/cli/scanners/py-ast.mjs +39 -2
- package/cli/scanners/task-context.mjs +312 -0
- package/cli/shared-doc-roles.mjs +44 -1
- package/cli/shared-source.mjs +101 -28
- package/cli/validators/architecture.mjs +186 -13
- package/cli/validators/environment.mjs +14 -1
- package/cli/validators/evidence.mjs +52 -0
- package/cli/validators/todo-tracking.mjs +45 -2
- package/cli/writers/doc-generators.mjs +31 -17
- package/cli/writers/mechanical.mjs +44 -14
- package/cli/writers/sections.mjs +31 -3
- package/docs/ai-integration.md +31 -6
- package/docs/commands.md +43 -5
- package/docs/configuration.md +11 -3
- package/docs/quickstart.md +1 -1
- package/extensions/spec-kit-docguard/commands/fix.md +4 -2
- package/extensions/spec-kit-docguard/commands/generate.md +6 -1
- package/extensions/spec-kit-docguard/commands/guard.md +3 -2
- package/extensions/spec-kit-docguard/commands/sync.md +1 -1
- package/extensions/spec-kit-docguard/extension.yml +1 -1
- package/extensions/spec-kit-docguard/skills/docguard-fix/SKILL.md +14 -3
- package/extensions/spec-kit-docguard/skills/docguard-guard/SKILL.md +16 -5
- package/extensions/spec-kit-docguard/skills/docguard-review/SKILL.md +8 -3
- package/extensions/spec-kit-docguard/skills/docguard-score/SKILL.md +3 -2
- package/extensions/spec-kit-docguard/skills/docguard-sync/SKILL.md +6 -3
- package/extensions/spec-kit-docguard/templates/github-workflows/docguard-autofix.yml +2 -2
- package/extensions/spec-kit-docguard/templates/github-workflows/docguard-guard.yml +4 -4
- package/package.json +2 -1
- package/schemas/docguard-agent-context-benchmark.schema.json +92 -0
- package/schemas/docguard-agent-context-result.schema.json +95 -0
- package/schemas/docguard-config.schema.json +1 -0
- package/schemas/docguard-evidence.schema.json +169 -0
- package/schemas/docguard-task-context.schema.json +144 -0
- package/templates/AGENTS.md.template +9 -4
- package/templates/ci/github-actions.yml +4 -4
- package/templates/commands/docguard.guard.md +5 -1
- package/templates/commands/docguard.review.md +6 -1
- package/templates/evidence-manifest.json +21 -0
|
@@ -28,13 +28,13 @@ You **MUST** consider the user input before proceeding (if not empty).
|
|
|
28
28
|
|
|
29
29
|
1. Run DocGuard guard validation:
|
|
30
30
|
```bash
|
|
31
|
-
npx --yes docguard-cli@latest guard $ARGUMENTS
|
|
31
|
+
npx --yes docguard-cli@latest guard --format json $ARGUMENTS
|
|
32
32
|
```
|
|
33
33
|
|
|
34
34
|
2. Parse each validator's result and build a severity-ranked findings table. Status glyphs: ✅ pass, ⚠️ warning, ❌ fail, ➖ N/A (nothing to validate — NOT a pass; the dimension was not assessed).
|
|
35
35
|
|
|
36
36
|
3. **Triage by severity**:
|
|
37
|
-
- **CRITICAL**: Structure, Security, Test-Spec failures → fix immediately
|
|
37
|
+
- **CRITICAL**: Structure, Security, Test-Spec, Evidence contradiction/manifest failures → fix immediately
|
|
38
38
|
- **HIGH**: Doc Sections, Drift-Comments, Changelog, Traceability, API-Surface → fix before commit
|
|
39
39
|
- **MEDIUM**: Freshness, Docs-Coverage, Doc-Quality, Metrics → fix this sprint
|
|
40
40
|
- **LOW**: TODO-Tracking, Schema-Sync, Spec-Kit, Metadata → fix when convenient
|
|
@@ -66,6 +66,7 @@ npx --yes docguard-cli@latest guard $ARGUMENTS
|
|
|
66
66
|
| TODO-Tracking | TODOs are tracked |
|
|
67
67
|
| Schema-Sync | Schema documentation matches code |
|
|
68
68
|
| Spec-Kit | Spec quality (FR-IDs, sections) |
|
|
69
|
+
| Evidence | Exact declared statements match current local sources or saved compatibility reports |
|
|
69
70
|
| Metrics-Consistency | Internal counts are accurate |
|
|
70
71
|
|
|
71
72
|
## Flags
|
|
@@ -46,7 +46,7 @@ npx --yes docguard-cli@latest guard
|
|
|
46
46
|
|
|
47
47
|
- `--since <ref>` — also report which code files changed since this git ref (context for prose updates).
|
|
48
48
|
- `--write` — apply the mechanical refreshes. Default is a dry-run preview.
|
|
49
|
-
- `--force` — sync
|
|
49
|
+
- `--force` — sync unmarked default-path docs. It never bypasses mapped-document section ownership.
|
|
50
50
|
- `--format json` — machine-readable output (`updates`, `reviews`, `skipped`).
|
|
51
51
|
|
|
52
52
|
## When to use
|
|
@@ -3,7 +3,7 @@ schema_version: "1.0"
|
|
|
3
3
|
extension:
|
|
4
4
|
id: "docguard"
|
|
5
5
|
name: "DocGuard — CDD Enforcement"
|
|
6
|
-
version: "0.
|
|
6
|
+
version: "0.40.1"
|
|
7
7
|
description: "Canonical-Driven Development enforcement as a true spec-kit extension. LLM-first design with automated validators, 5 AI behavior skills, spec-kit skill chaining, and workflow hooks. One pinned runtime dependency (@babel/parser); pure Node.js otherwise."
|
|
8
8
|
author: "Ricardo Accioly"
|
|
9
9
|
repository: "https://github.com/raccioly/docguard"
|
|
@@ -6,10 +6,10 @@ description: AI-driven documentation repair with structured research workflow, t
|
|
|
6
6
|
compatibility: Requires DocGuard CLI installed (npm i -g docguard-cli or npx docguard-cli)
|
|
7
7
|
metadata:
|
|
8
8
|
author: docguard
|
|
9
|
-
version: 0.
|
|
9
|
+
version: 0.40.1
|
|
10
10
|
source: extensions/spec-kit-docguard/skills/docguard-fix
|
|
11
11
|
---
|
|
12
|
-
<!-- docguard:version: 0.
|
|
12
|
+
<!-- docguard:version: 0.40.1 -->
|
|
13
13
|
|
|
14
14
|
# DocGuard Fix Skill
|
|
15
15
|
|
|
@@ -44,7 +44,9 @@ npx docguard-cli fix --write 2>&1
|
|
|
44
44
|
|
|
45
45
|
Removes endpoints documented in `docs-canonical/API-REFERENCE.md` that the OpenAPI
|
|
46
46
|
spec confirms no longer exist (table row + detail block). Only edits
|
|
47
|
-
`docguard:generated`
|
|
47
|
+
`docguard:generated` whole documents. Mapped human docs expose only unique
|
|
48
|
+
`source=code` sections, and `--force` cannot bypass ownership. The operation is
|
|
49
|
+
idempotent and prints what changed. Don't hand-edit generated sections.
|
|
48
50
|
|
|
49
51
|
### Step 1: Diagnose Current State
|
|
50
52
|
|
|
@@ -217,6 +219,15 @@ After all fixes are applied, output:
|
|
|
217
219
|
- **Log deviations** — if you deviate from canonical expectations, add `// DRIFT: reason` in DRIFT-LOG.md
|
|
218
220
|
- **Never include secrets** — document variable/secret NAMES only, never actual values
|
|
219
221
|
|
|
222
|
+
## Evidence Findings
|
|
223
|
+
|
|
224
|
+
Run `docguard verify --evidence --format json` when guard returns an `EVD` code.
|
|
225
|
+
Repair invalid, missing, ambiguous, stale, or unsupported evidence at the named
|
|
226
|
+
source. For `EVD002`, inspect the approved requirement and implementation before
|
|
227
|
+
editing either side; current code does not automatically override canonical
|
|
228
|
+
intent. Rerun evidence verification after the targeted change. Never broaden a
|
|
229
|
+
statement template or remove an evidence declaration merely to make guard pass.
|
|
230
|
+
|
|
220
231
|
## Integration with Spec Kit
|
|
221
232
|
|
|
222
233
|
If `.specify/` directory exists:
|
|
@@ -7,10 +7,10 @@ description: Run DocGuard guard validation against Canonical-Driven Development
|
|
|
7
7
|
compatibility: Requires DocGuard CLI installed (npm i -g docguard-cli or npx docguard-cli)
|
|
8
8
|
metadata:
|
|
9
9
|
author: docguard
|
|
10
|
-
version: 0.
|
|
10
|
+
version: 0.40.1
|
|
11
11
|
source: extensions/spec-kit-docguard/skills/docguard-guard
|
|
12
12
|
---
|
|
13
|
-
<!-- docguard:version: 0.
|
|
13
|
+
<!-- docguard:version: 0.40.1 -->
|
|
14
14
|
|
|
15
15
|
# DocGuard Guard Skill
|
|
16
16
|
|
|
@@ -44,12 +44,12 @@ Execute DocGuard's full guard validator suite against the current project, parse
|
|
|
44
44
|
Execute the guard command and capture full output:
|
|
45
45
|
|
|
46
46
|
```bash
|
|
47
|
-
npx docguard-cli guard
|
|
47
|
+
npx docguard-cli guard --format json
|
|
48
48
|
```
|
|
49
49
|
|
|
50
50
|
If in a DocGuard development environment (cli/docguard.mjs exists), use:
|
|
51
51
|
```bash
|
|
52
|
-
node cli/docguard.mjs guard
|
|
52
|
+
node cli/docguard.mjs guard --format json
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
### Step 2: Parse Validator Results
|
|
@@ -75,6 +75,7 @@ Classify every non-passing check using this priority matrix:
|
|
|
75
75
|
- Structure failures (missing canonical docs)
|
|
76
76
|
- Security failures (hardcoded secrets, missing SECURITY.md)
|
|
77
77
|
- Test-Spec failures (tests don't match spec)
|
|
78
|
+
- Evidence contradictions or an invalid evidence manifest
|
|
78
79
|
|
|
79
80
|
**HIGH (fix before commit)**:
|
|
80
81
|
- Doc Sections failures (missing required sections)
|
|
@@ -139,7 +140,7 @@ For each finding, provide a **specific, actionable fix** — not "fix the issue"
|
|
|
139
140
|
|
|
140
141
|
Based on the triage results:
|
|
141
142
|
|
|
142
|
-
- **If all PASS**: "All validators passed.
|
|
143
|
+
- **If all PASS**: "All configured validators passed. Report declared evidence coverage and any remaining heuristic claims; uncaptured prose is still unverified."
|
|
143
144
|
- **If only MEDIUM/LOW warnings**: "Non-blocking warnings found. Safe to commit, but consider running `/docguard.fix` for automated remediation."
|
|
144
145
|
- **If HIGH or CRITICAL failures**: "Blocking issues found. Fix these before committing. Suggest running `/docguard.fix --doc [most impactful doc]` next."
|
|
145
146
|
|
|
@@ -157,6 +158,16 @@ Present the user with options:
|
|
|
157
158
|
- **Track progress** — if user runs guard multiple times, compare before/after
|
|
158
159
|
- If user provides `$ARGUMENTS` like "just structure" or "only security", filter report to those validators
|
|
159
160
|
|
|
161
|
+
## Evidence State Rules
|
|
162
|
+
|
|
163
|
+
When the JSON payload contains `evidence`, preserve its state names exactly.
|
|
164
|
+
Treat `verified-within-scope` as a pass only for that declaration. Treat
|
|
165
|
+
`contradicted` as a high-confidence failure. For `stale`, regenerate the saved
|
|
166
|
+
upstream report and hashes before reviewing prose. For `inconclusive`, restore
|
|
167
|
+
or narrow the missing/ambiguous input. For `unsupported`, keep the finding
|
|
168
|
+
visible and request a paired synthetic fixture before expanding support. Never
|
|
169
|
+
rewrite approved intent from current code automatically.
|
|
170
|
+
|
|
160
171
|
## Integration with Spec Kit (Extension-First)
|
|
161
172
|
|
|
162
173
|
DocGuard is a spec-kit extension. When this project has a `.specify/` directory:
|
|
@@ -6,10 +6,10 @@ description: Cross-document consistency analysis and quality assessment. Perform
|
|
|
6
6
|
compatibility: Requires DocGuard CLI installed (npm i -g docguard-cli or npx docguard-cli)
|
|
7
7
|
metadata:
|
|
8
8
|
author: docguard
|
|
9
|
-
version: 0.
|
|
9
|
+
version: 0.40.1
|
|
10
10
|
source: extensions/spec-kit-docguard/skills/docguard-review
|
|
11
11
|
---
|
|
12
|
-
<!-- docguard:version: 0.
|
|
12
|
+
<!-- docguard:version: 0.40.1 -->
|
|
13
13
|
|
|
14
14
|
# DocGuard Review Skill
|
|
15
15
|
|
|
@@ -51,7 +51,9 @@ Execute DocGuard's diagnostic and scoring tools:
|
|
|
51
51
|
```bash
|
|
52
52
|
npx docguard-cli diagnose 2>&1
|
|
53
53
|
npx docguard-cli score 2>&1
|
|
54
|
-
npx docguard-cli guard
|
|
54
|
+
npx docguard-cli guard --format json
|
|
55
|
+
npx docguard-cli verify --evidence --format json
|
|
56
|
+
npx docguard-cli verify --semantic --format json
|
|
55
57
|
```
|
|
56
58
|
|
|
57
59
|
Record:
|
|
@@ -59,6 +61,7 @@ Record:
|
|
|
59
61
|
- CDD maturity score (0-100)
|
|
60
62
|
- ALCOA+ compliance attributes
|
|
61
63
|
- Category breakdown
|
|
64
|
+
- Exact declared evidence states and the remaining heuristic claim tasks
|
|
62
65
|
|
|
63
66
|
### Step 3: Semantic Cross-Document Analysis
|
|
64
67
|
|
|
@@ -180,6 +183,8 @@ Ask: "Would you like me to fix the top N issues? (I'll show you what I plan to c
|
|
|
180
183
|
- **Compare actual code vs docs** — don't just validate formatting
|
|
181
184
|
- **Limit findings to 50** — aggregate overflow in a summary count
|
|
182
185
|
- **Prioritize high-signal findings** — one CRITICAL finding is worth ten LOW findings
|
|
186
|
+
- **Preserve evidence scope** — a verified declaration covers one selected statement, while undeclared prose and whole-document accuracy remain review work
|
|
187
|
+
- **Preserve approved intent** — a contradiction can mean implementation regressed; determine which side owns truth before recommending an edit
|
|
183
188
|
|
|
184
189
|
## Context
|
|
185
190
|
|
|
@@ -6,10 +6,10 @@ description: CDD maturity assessment with category-aware improvement roadmap. Ru
|
|
|
6
6
|
compatibility: Requires DocGuard CLI installed (npm i -g docguard-cli or npx docguard-cli)
|
|
7
7
|
metadata:
|
|
8
8
|
author: docguard
|
|
9
|
-
version: 0.
|
|
9
|
+
version: 0.40.1
|
|
10
10
|
source: extensions/spec-kit-docguard/skills/docguard-score
|
|
11
11
|
---
|
|
12
|
-
<!-- docguard:version: 0.
|
|
12
|
+
<!-- docguard:version: 0.40.1 -->
|
|
13
13
|
|
|
14
14
|
# DocGuard Score Skill
|
|
15
15
|
|
|
@@ -173,6 +173,7 @@ If user has run score before (check git log for score badge changes):
|
|
|
173
173
|
- **Be actionable** — every recommendation must have a specific action
|
|
174
174
|
- **Compare before/after** — if user has previously run score in this session, show improvement
|
|
175
175
|
- **Focus on ROI** — surface the cheapest fixes with the biggest score impact first
|
|
176
|
+
- **Keep assurance separate** — report `assurance.declaredEvidence` and remaining `unverifiedClaims` beside the structural score; never convert a scoped evidence pass into an accuracy score for a document or project
|
|
176
177
|
|
|
177
178
|
## Context
|
|
178
179
|
|
|
@@ -4,10 +4,10 @@ description: Keep canonical documentation ALWAYS UP TO DATE. Refreshes code-trut
|
|
|
4
4
|
compatibility: Requires DocGuard CLI installed (npm i -g docguard-cli or npx docguard-cli)
|
|
5
5
|
metadata:
|
|
6
6
|
author: docguard
|
|
7
|
-
version: 0.
|
|
7
|
+
version: 0.40.1
|
|
8
8
|
source: extensions/spec-kit-docguard/skills/docguard-sync
|
|
9
9
|
---
|
|
10
|
-
<!-- docguard:version: 0.
|
|
10
|
+
<!-- docguard:version: 0.40.1 -->
|
|
11
11
|
|
|
12
12
|
# DocGuard Sync Skill
|
|
13
13
|
|
|
@@ -76,6 +76,7 @@ For each `🤖 Prose to review` entry:
|
|
|
76
76
|
|
|
77
77
|
```bash
|
|
78
78
|
npx --yes docguard-cli@latest guard
|
|
79
|
+
npx --yes docguard-cli@latest verify --evidence
|
|
79
80
|
```
|
|
80
81
|
|
|
81
82
|
Confirm there are no errors. If the API surface drifted (`API-Surface` failures),
|
|
@@ -102,7 +103,8 @@ that ref. Use that diff to:
|
|
|
102
103
|
|---|---|
|
|
103
104
|
| `↻ docs-canonical/API-REFERENCE.md → endpoints` | Code-truth refreshed by `--write`. No action needed. |
|
|
104
105
|
| `🤖 docs-canonical/API-REFERENCE.md → overview` | Open the doc; update the `overview` prose to reflect the new endpoint set. |
|
|
105
|
-
| `Skipped … not marked docguard:generated` |
|
|
106
|
+
| `Skipped … not marked docguard:generated` | A default-path doc lacks whole-document ownership. Review before using `--force`; a mapped human doc still requires a unique `source=code` section. |
|
|
107
|
+
| `malformed or duplicate docguard:section markers` | Stop. Repair ownership markers explicitly; `--force` cannot authorize the write. |
|
|
106
108
|
| `Documentation memory is up to date` | Done — no drift. |
|
|
107
109
|
|
|
108
110
|
## Anti-patterns (do NOT do these)
|
|
@@ -110,3 +112,4 @@ that ref. Use that diff to:
|
|
|
110
112
|
- ❌ Editing inside `<!-- docguard:section source=code -->` — DocGuard will rewrite it on the next sync.
|
|
111
113
|
- ❌ Removing the markers to "make the doc look cleaner" — that breaks future sync/regeneration.
|
|
112
114
|
- ❌ Skipping `sync --write` and editing the code section by hand — let DocGuard do it.
|
|
115
|
+
- ❌ Rewriting an evidence-bound human statement from generated code alone — rerun the declared predicate and review approved intent first.
|
|
@@ -35,7 +35,7 @@ jobs:
|
|
|
35
35
|
if: github.event.pull_request.head.repo.full_name == github.repository
|
|
36
36
|
steps:
|
|
37
37
|
- name: Checkout PR branch (with write token)
|
|
38
|
-
uses: actions/checkout@
|
|
38
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
39
39
|
with:
|
|
40
40
|
ref: ${{ github.event.pull_request.head.ref }}
|
|
41
41
|
# Default GITHUB_TOKEN has the contents:write scope granted above.
|
|
@@ -44,7 +44,7 @@ jobs:
|
|
|
44
44
|
fetch-depth: 0
|
|
45
45
|
|
|
46
46
|
- name: Run DocGuard fix --write + auto-commit + PR comment
|
|
47
|
-
uses: raccioly/docguard@v0.
|
|
47
|
+
uses: raccioly/docguard@v0.40.1
|
|
48
48
|
with:
|
|
49
49
|
command: fix
|
|
50
50
|
auto-commit: 'true'
|
|
@@ -25,17 +25,17 @@ jobs:
|
|
|
25
25
|
echo "DOCGUARD_LOG=$RUNNER_TEMP/docguard-stderr.log" >> "$GITHUB_ENV"
|
|
26
26
|
echo "DOCGUARD_SCORE=$RUNNER_TEMP/docguard-score.json" >> "$GITHUB_ENV"
|
|
27
27
|
|
|
28
|
-
- uses: actions/checkout@
|
|
28
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
29
29
|
with:
|
|
30
30
|
fetch-depth: 0 # Freshness needs the complete commit history.
|
|
31
31
|
persist-credentials: false
|
|
32
32
|
|
|
33
|
-
- uses: actions/setup-node@
|
|
33
|
+
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
34
34
|
with:
|
|
35
35
|
node-version: '20'
|
|
36
36
|
|
|
37
37
|
- name: Install DocGuard
|
|
38
|
-
run: npm install --global --ignore-scripts docguard-cli@0.
|
|
38
|
+
run: npm install --global --ignore-scripts docguard-cli@0.40.1
|
|
39
39
|
|
|
40
40
|
- name: Run DocGuard
|
|
41
41
|
shell: bash
|
|
@@ -83,7 +83,7 @@ jobs:
|
|
|
83
83
|
|
|
84
84
|
- name: Upload Report
|
|
85
85
|
if: always()
|
|
86
|
-
uses: actions/upload-artifact@
|
|
86
|
+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
87
87
|
with:
|
|
88
88
|
name: docguard-report
|
|
89
89
|
path: |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "docguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.40.1",
|
|
4
4
|
"description": "The enforcement tool for Canonical-Driven Development (CDD). Audit, generate, and guard your project documentation.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
"init": "node cli/docguard.mjs init",
|
|
14
14
|
"score": "node cli/docguard.mjs score",
|
|
15
15
|
"benchmark": "node benchmarks/run.mjs",
|
|
16
|
+
"benchmark:agent-context": "node benchmarks/agent-context/run.mjs",
|
|
16
17
|
"diff": "node cli/docguard.mjs diff",
|
|
17
18
|
"generate": "node cli/docguard.mjs generate",
|
|
18
19
|
"hooks": "node cli/docguard.mjs hooks",
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://raccioly.github.io/docguard/schemas/docguard-agent-context-benchmark.schema.json",
|
|
4
|
+
"title": "DocGuard task-specific agent context benchmark",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["$schema", "schemaVersion", "protocol", "conditions", "promotion", "tasks"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"$schema": { "const": "https://raccioly.github.io/docguard/schemas/docguard-agent-context-benchmark.schema.json" },
|
|
10
|
+
"schemaVersion": { "const": 1 },
|
|
11
|
+
"protocol": {
|
|
12
|
+
"type": "object",
|
|
13
|
+
"additionalProperties": false,
|
|
14
|
+
"required": ["id", "frozenAt", "seed", "repetitions", "timeoutMs", "harness", "model"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"id": { "const": "docguard-agent-context-v1" },
|
|
17
|
+
"frozenAt": { "type": "string", "format": "date" },
|
|
18
|
+
"seed": { "type": "string", "minLength": 1, "maxLength": 128 },
|
|
19
|
+
"repetitions": { "type": "integer", "minimum": 3, "maximum": 20 },
|
|
20
|
+
"timeoutMs": { "type": "integer", "minimum": 1000, "maximum": 900000 },
|
|
21
|
+
"harness": { "type": "string", "minLength": 1, "maxLength": 128 },
|
|
22
|
+
"model": {
|
|
23
|
+
"type": "object",
|
|
24
|
+
"additionalProperties": false,
|
|
25
|
+
"required": ["cliVersion", "id", "reasoningEffort", "sandbox"],
|
|
26
|
+
"properties": {
|
|
27
|
+
"cliVersion": { "type": "string", "minLength": 1, "maxLength": 128 },
|
|
28
|
+
"id": { "type": "string", "minLength": 1, "maxLength": 128 },
|
|
29
|
+
"reasoningEffort": { "enum": ["low", "medium", "high"] },
|
|
30
|
+
"sandbox": { "const": "workspace-write" }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"conditions": {
|
|
36
|
+
"type": "array",
|
|
37
|
+
"minItems": 3,
|
|
38
|
+
"maxItems": 3,
|
|
39
|
+
"uniqueItems": true,
|
|
40
|
+
"prefixItems": [
|
|
41
|
+
{ "const": "task-only" },
|
|
42
|
+
{ "const": "context-pack" },
|
|
43
|
+
{ "const": "targeted-packet" }
|
|
44
|
+
],
|
|
45
|
+
"items": false
|
|
46
|
+
},
|
|
47
|
+
"promotion": {
|
|
48
|
+
"type": "object",
|
|
49
|
+
"additionalProperties": false,
|
|
50
|
+
"required": ["nonInferiorityFailures", "maxAdditionalRequirementViolations", "maxAdditionalUnnecessaryEdits", "minimumMedianReductionPercent", "successGainQualifies"],
|
|
51
|
+
"properties": {
|
|
52
|
+
"nonInferiorityFailures": { "const": 1 },
|
|
53
|
+
"maxAdditionalRequirementViolations": { "const": 0 },
|
|
54
|
+
"maxAdditionalUnnecessaryEdits": { "const": 0 },
|
|
55
|
+
"minimumMedianReductionPercent": { "const": 15 },
|
|
56
|
+
"successGainQualifies": { "const": true }
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"tasks": {
|
|
60
|
+
"type": "array",
|
|
61
|
+
"minItems": 3,
|
|
62
|
+
"maxItems": 3,
|
|
63
|
+
"items": { "$ref": "#/$defs/task" }
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
"$defs": {
|
|
67
|
+
"path": { "type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*\\\\).+" },
|
|
68
|
+
"task": {
|
|
69
|
+
"type": "object",
|
|
70
|
+
"additionalProperties": false,
|
|
71
|
+
"required": ["id", "fixture", "hiddenEvaluator", "reference", "prompt", "allowedChanges", "visibleTest", "expectedInitial"],
|
|
72
|
+
"properties": {
|
|
73
|
+
"id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,63}$" },
|
|
74
|
+
"fixture": { "$ref": "#/$defs/path" },
|
|
75
|
+
"hiddenEvaluator": { "$ref": "#/$defs/path" },
|
|
76
|
+
"reference": { "$ref": "#/$defs/path" },
|
|
77
|
+
"prompt": { "type": "string", "minLength": 20, "maxLength": 2000 },
|
|
78
|
+
"allowedChanges": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "$ref": "#/$defs/path" } },
|
|
79
|
+
"visibleTest": { "type": "array", "minItems": 2, "items": { "type": "string", "minLength": 1, "maxLength": 512 } },
|
|
80
|
+
"expectedInitial": {
|
|
81
|
+
"type": "object",
|
|
82
|
+
"additionalProperties": false,
|
|
83
|
+
"required": ["passed", "failed"],
|
|
84
|
+
"properties": {
|
|
85
|
+
"passed": { "type": "integer", "minimum": 1 },
|
|
86
|
+
"failed": { "type": "integer", "minimum": 1 }
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://raccioly.github.io/docguard/schemas/docguard-agent-context-result.schema.json",
|
|
4
|
+
"title": "DocGuard task-specific agent context benchmark result",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["$schema", "schemaVersion", "core", "observations"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"$schema": { "const": "https://raccioly.github.io/docguard/schemas/docguard-agent-context-result.schema.json" },
|
|
10
|
+
"schemaVersion": { "const": 1 },
|
|
11
|
+
"core": {
|
|
12
|
+
"type": "object",
|
|
13
|
+
"additionalProperties": false,
|
|
14
|
+
"required": ["protocolId", "manifestDigest", "harnessDigest", "analysisDigest", "selectorDigest", "fixtureDigests", "trialOrder", "aggregate", "decision"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"protocolId": { "const": "docguard-agent-context-v1" },
|
|
17
|
+
"manifestDigest": { "$ref": "#/$defs/digest" },
|
|
18
|
+
"harnessDigest": { "$ref": "#/$defs/digest" },
|
|
19
|
+
"analysisDigest": { "$ref": "#/$defs/digest" },
|
|
20
|
+
"selectorDigest": { "$ref": "#/$defs/digest" },
|
|
21
|
+
"fixtureDigests": { "type": "object", "minProperties": 3, "additionalProperties": { "$ref": "#/$defs/digest" } },
|
|
22
|
+
"trialOrder": { "type": "array", "minItems": 27, "items": { "type": "string" } },
|
|
23
|
+
"aggregate": { "type": "object" },
|
|
24
|
+
"decision": {
|
|
25
|
+
"type": "object",
|
|
26
|
+
"additionalProperties": false,
|
|
27
|
+
"required": ["status", "reasons"],
|
|
28
|
+
"properties": {
|
|
29
|
+
"status": { "enum": ["promote", "reject", "incomplete"] },
|
|
30
|
+
"reasons": { "type": "array", "items": { "type": "string", "minLength": 1 } },
|
|
31
|
+
"reductions": {
|
|
32
|
+
"type": "object",
|
|
33
|
+
"additionalProperties": false,
|
|
34
|
+
"required": ["uncachedInputTokens", "steps", "latency"],
|
|
35
|
+
"properties": {
|
|
36
|
+
"uncachedInputTokens": { "type": ["number", "null"] },
|
|
37
|
+
"steps": { "type": ["number", "null"] },
|
|
38
|
+
"latency": { "type": ["number", "null"] }
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
"observations": {
|
|
46
|
+
"type": "object",
|
|
47
|
+
"additionalProperties": false,
|
|
48
|
+
"required": ["generatedAt", "environment", "trials"],
|
|
49
|
+
"properties": {
|
|
50
|
+
"generatedAt": { "type": "string", "format": "date-time" },
|
|
51
|
+
"environment": { "type": "object" },
|
|
52
|
+
"trials": { "type": "array", "minItems": 27, "items": { "$ref": "#/$defs/trial" } }
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"$defs": {
|
|
57
|
+
"digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
|
|
58
|
+
"nullableMetric": { "type": ["integer", "null"], "minimum": 0 },
|
|
59
|
+
"trial": {
|
|
60
|
+
"type": "object",
|
|
61
|
+
"additionalProperties": false,
|
|
62
|
+
"required": ["id", "task", "condition", "repetition", "status", "success", "requirementsPassed", "requirementsTotal", "requirementViolations", "unnecessaryEdits", "changedFiles", "steps", "usage", "latencyMs", "humanIntervention", "patchDigest", "failure"],
|
|
63
|
+
"properties": {
|
|
64
|
+
"id": { "type": "string" },
|
|
65
|
+
"task": { "type": "string" },
|
|
66
|
+
"condition": { "enum": ["task-only", "context-pack", "targeted-packet"] },
|
|
67
|
+
"repetition": { "type": "integer", "minimum": 1 },
|
|
68
|
+
"status": { "enum": ["completed", "agent-failed", "timed-out", "infrastructure-failed"] },
|
|
69
|
+
"success": { "type": "boolean" },
|
|
70
|
+
"requirementsPassed": { "type": "integer", "minimum": 0 },
|
|
71
|
+
"requirementsTotal": { "type": "integer", "minimum": 1 },
|
|
72
|
+
"requirementViolations": { "type": "integer", "minimum": 0 },
|
|
73
|
+
"unnecessaryEdits": { "type": "integer", "minimum": 0 },
|
|
74
|
+
"changedFiles": { "type": "array", "uniqueItems": true, "items": { "type": "string" } },
|
|
75
|
+
"steps": { "type": "integer", "minimum": 0 },
|
|
76
|
+
"usage": {
|
|
77
|
+
"type": "object",
|
|
78
|
+
"additionalProperties": false,
|
|
79
|
+
"required": ["inputTokens", "cachedInputTokens", "uncachedInputTokens", "outputTokens", "reasoningTokens"],
|
|
80
|
+
"properties": {
|
|
81
|
+
"inputTokens": { "$ref": "#/$defs/nullableMetric" },
|
|
82
|
+
"cachedInputTokens": { "$ref": "#/$defs/nullableMetric" },
|
|
83
|
+
"uncachedInputTokens": { "$ref": "#/$defs/nullableMetric" },
|
|
84
|
+
"outputTokens": { "$ref": "#/$defs/nullableMetric" },
|
|
85
|
+
"reasoningTokens": { "$ref": "#/$defs/nullableMetric" }
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
"latencyMs": { "type": "integer", "minimum": 0 },
|
|
89
|
+
"humanIntervention": { "type": "integer", "minimum": 0, "maximum": 1 },
|
|
90
|
+
"patchDigest": { "anyOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }] },
|
|
91
|
+
"failure": { "type": ["string", "null"] }
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -107,6 +107,7 @@
|
|
|
107
107
|
"specKit": { "type": "boolean" },
|
|
108
108
|
"documentLifecycle": { "type": "boolean" },
|
|
109
109
|
"specRegistry": { "type": "boolean" },
|
|
110
|
+
"evidence": { "type": "boolean" },
|
|
110
111
|
"crossReference": { "type": "boolean" },
|
|
111
112
|
"generatedStaleness":{ "type": "boolean" },
|
|
112
113
|
"canonicalSync": { "type": "boolean" },
|