canary-test-cli 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +308 -0
- package/agents/skills/canary:generate.md +49 -0
- package/agents/skills/canary:init.md +37 -0
- package/agents/skills/canary:migrate.md +66 -0
- package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
- package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
- package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
- package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
- package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
- package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
- package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
- package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
- package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
- package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
- package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
- package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
- package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
- package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
- package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
- package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
- package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
- package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
- package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
- package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
- package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
- package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
- package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
- package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
- package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
- package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
- package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
- package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
- package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
- package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
- package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
- package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
- package/agents/skills/lib/parse-args.mjs +214 -0
- package/dist/engine/analysis/cli.js +116 -54
- package/dist/engine/analysis/engine.js +34 -16
- package/dist/engine/analysis/reports.js +5 -4
- package/dist/engine/cli-commands.js +252 -106
- package/dist/engine/cli-common.js +15 -24
- package/dist/engine/cli.core.js +37 -11
- package/dist/engine/cli.js +2 -2
- package/dist/engine/company-knowledge-cli.js +2 -2
- package/dist/engine/core/adoption.js +408 -0
- package/dist/engine/core/framework-probes.js +7 -7
- package/dist/engine/core/fs-glob.js +2 -2
- package/dist/engine/core/gate-result.js +17 -0
- package/dist/engine/core/migrator.js +9 -17
- package/dist/engine/core/pattern-matcher.js +23 -5
- package/dist/engine/core/persona.js +421 -0
- package/dist/engine/core/promotion-verdict.js +261 -0
- package/dist/engine/core/quality-scorer.js +15 -2
- package/dist/engine/core/reporter.js +1 -9
- package/dist/engine/core/skill-dispatch.js +115 -0
- package/dist/engine/core/skill-examples.js +392 -0
- package/dist/engine/core/skill-registry.js +59 -4
- package/dist/engine/core/skill-surfaces.js +307 -0
- package/dist/engine/core/static-linter.js +310 -38
- package/dist/engine/core/test-files.js +77 -0
- package/dist/engine/core/ticket-updater.js +1 -7
- package/dist/engine/core/vacuity-scanner.js +726 -0
- package/dist/engine/core/workflow-discovery.js +2 -8
- package/dist/engine/core/workspace-detect.js +7 -6
- package/dist/engine/data/personas/registry.json +36 -0
- package/dist/engine/guardian/adjudication.js +5 -5
- package/dist/engine/guardian/analysis-emit.js +19 -28
- package/dist/engine/guardian/cli.js +150 -51
- package/dist/engine/guardian/coverage.js +1 -1
- package/dist/engine/guardian/diff-coverage/heuristic-tier.js +1 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +2 -2
- package/dist/engine/guardian/pr-check.js +322 -33
- package/dist/engine/guardian/pr-comment.js +4 -3
- package/dist/engine/history/cli.js +210 -6
- package/dist/engine/history/ndjson-store.js +9 -5
- package/dist/engine/history/record.js +34 -5
- package/dist/engine/history/run-recorder.js +165 -0
- package/dist/engine/history/schema.js +25 -7
- package/dist/engine/history/store.js +9 -0
- package/dist/engine/mcp-server.js +35 -13
- package/dist/engine/skills-cli.js +249 -23
- package/dist/engine/util/ensure-ascii.js +37 -0
- package/dist/engine/workflow-cli.js +6 -6
- package/dist/gate-result.d.ts +11 -0
- package/dist/gate-result.js +18 -0
- package/dist/reporters/testtracker.d.ts +1 -1
- package/dist/reporters/testtracker.js +1 -1
- package/dist/uninstall.js +12 -5
- package/package.json +3 -2
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: canary-setup-harness
|
|
3
|
+
description: >
|
|
4
|
+
Configure the Harness Engineering guardrails in a new Canary project (or a
|
|
5
|
+
fork) — installs the harness CLI, initialises `harness.config.json`, wires
|
|
6
|
+
up the five required CI workflows, generates the initial security ledger,
|
|
7
|
+
and verifies all gates pass on a clean repository. Use for "set up harness
|
|
8
|
+
for this project", "onboard this repo with harness", "install the harness
|
|
9
|
+
CLI", "configure the CI gates", or when a fork has drifted from the
|
|
10
|
+
harness config and needs re-alignment. Not for day-to-day harness updates,
|
|
11
|
+
adding a single new gate to an existing config, or troubleshooting a
|
|
12
|
+
failing CI check.
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
# Canary: Setup Harness
|
|
16
|
+
|
|
17
|
+
> Configure the Harness Engineering guardrails in a new Canary
|
|
18
|
+
> project (or a fork). Installs the harness CLI, initialises the
|
|
19
|
+
> config, wires up CI workflows, and verifies all gates pass on
|
|
20
|
+
> a clean repository.
|
|
21
|
+
|
|
22
|
+
## When to Use
|
|
23
|
+
|
|
24
|
+
- When setting up Canary (or a fork) in a new repository for
|
|
25
|
+
the first time
|
|
26
|
+
- When onboarding a new project that wants Canary's CI gate
|
|
27
|
+
structure
|
|
28
|
+
- When a fork has drifted from the harness config and needs
|
|
29
|
+
to be re-aligned
|
|
30
|
+
- NOT for day-to-day harness updates — use the harness CLI's
|
|
31
|
+
own update flow for that
|
|
32
|
+
- NOT for adding a single new gate to an existing config —
|
|
33
|
+
edit `harness.config.json` directly
|
|
34
|
+
- NOT for troubleshooting a failing CI check — see the
|
|
35
|
+
[Understanding the Harness][harness-wiki] wiki page
|
|
36
|
+
|
|
37
|
+
[harness-wiki]: ../../../../docs/wiki/Understanding-the-Harness.md
|
|
38
|
+
|
|
39
|
+
## Process
|
|
40
|
+
|
|
41
|
+
### Phase 1: PREREQUISITES — Confirm the Environment
|
|
42
|
+
|
|
43
|
+
1. **Verify Node.js is available** (harness CLI requires it):
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
node --version
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Version 18 or later is required.
|
|
50
|
+
|
|
51
|
+
2. **Verify Python is available** (security ledger script):
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
python3 --version
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Version 3.11 or later is required.
|
|
58
|
+
|
|
59
|
+
3. **Confirm the repo has a `harness.config.json`** at the
|
|
60
|
+
root. If it doesn't, the harness is not yet configured —
|
|
61
|
+
continue to Phase 2. If it does, validate it first:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
npx --yes -p @harness-engineering/cli harness validate
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
If validation passes, the harness is already set up. Stop
|
|
68
|
+
here unless re-alignment is the goal.
|
|
69
|
+
|
|
70
|
+
### Phase 2: INSTALL — Run the Harness Init
|
|
71
|
+
|
|
72
|
+
1. **Initialise the harness** from the repo root:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
npx --yes -p @harness-engineering/cli harness init
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
This creates `harness.config.json` with default gates and
|
|
79
|
+
generates the `.harness/` hooks directory.
|
|
80
|
+
|
|
81
|
+
2. **Review the generated `harness.config.json`.** The defaults
|
|
82
|
+
are sensible, but confirm:
|
|
83
|
+
|
|
84
|
+
- `layers` matches the project's actual package structure
|
|
85
|
+
(for Canary: `llm`, `core`, `cli`)
|
|
86
|
+
- `entrypoints` lists the correct top-level packages
|
|
87
|
+
- `telemetry` and `adoption` settings match team policy
|
|
88
|
+
|
|
89
|
+
3. **Inspect `.harness/hooks/`.** The hooks directory contains
|
|
90
|
+
scripts that run locally on git events. Review each file
|
|
91
|
+
briefly — they should not require network access outside
|
|
92
|
+
the harness telemetry endpoint.
|
|
93
|
+
|
|
94
|
+
### Phase 3: CONFIGURE — Wire Up CI Workflows
|
|
95
|
+
|
|
96
|
+
1. **Check which workflows already exist:**
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
ls .github/workflows/
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
2. **Required workflows for a full harness setup.** Each
|
|
103
|
+
should exist as a `.yml` file in `.github/workflows/`:
|
|
104
|
+
|
|
105
|
+
- `harness.yml` — core phase-gate checks
|
|
106
|
+
- `harness-architecture.yml` — layer dependency validation
|
|
107
|
+
- `harness-quality.yml` — quality and integrity checks
|
|
108
|
+
- `harness-security.yml` — security scan + ledger refresh
|
|
109
|
+
- `docs-lint.yml` — markdown formatting enforcement
|
|
110
|
+
|
|
111
|
+
If any are missing, copy them from an upstream Canary
|
|
112
|
+
reference install or generate them:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
npx --yes -p @harness-engineering/cli harness \
|
|
116
|
+
generate-workflows
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
3. **Confirm `harness-security.yml` refreshes the ledger.**
|
|
120
|
+
The final steps of the security job must run the ledger
|
|
121
|
+
script and commit if changed. Without this, the security
|
|
122
|
+
ledger goes stale and the Quality gate fails.
|
|
123
|
+
|
|
124
|
+
4. **Confirm `docs-lint.yml` covers all doc paths.** The
|
|
125
|
+
workflow's `paths` filter should include:
|
|
126
|
+
|
|
127
|
+
- `docs/**`
|
|
128
|
+
- `agents/**`
|
|
129
|
+
- `AGENTS.md`
|
|
130
|
+
|
|
131
|
+
5. **Set required permissions.** Workflows that commit
|
|
132
|
+
back to the repository need `contents: write`:
|
|
133
|
+
|
|
134
|
+
```yaml
|
|
135
|
+
permissions:
|
|
136
|
+
contents: write
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
This applies to `harness-security.yml` at minimum.
|
|
140
|
+
|
|
141
|
+
### Phase 4: BASELINE — Generate Initial Artifacts
|
|
142
|
+
|
|
143
|
+
1. **Run the security scan to create the initial ledger:**
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
npx --yes -p @harness-engineering/cli harness check-security
|
|
147
|
+
python3 scripts/security_ledger.py
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
2. **Commit the generated baseline files:**
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
git add harness.config.json .harness/ \
|
|
154
|
+
.harness/security/timeline.json \
|
|
155
|
+
docs/SECURITY_LEDGER.md
|
|
156
|
+
git commit -m "chore: initialise harness configuration"
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
3. **Push and confirm all CI gates pass** on the resulting
|
|
160
|
+
commit before calling setup complete.
|
|
161
|
+
|
|
162
|
+
### Phase 5: VERIFY — Confirm All Gates Pass
|
|
163
|
+
|
|
164
|
+
1. **Open a pull request** (or push to the configured branch)
|
|
165
|
+
to trigger CI.
|
|
166
|
+
2. **Check each workflow:**
|
|
167
|
+
|
|
168
|
+
- Architecture Enforcer: green
|
|
169
|
+
- Quality & Integrity: green
|
|
170
|
+
- Docs Lint: green
|
|
171
|
+
- Security Reviewer: green (ledger will be auto-refreshed
|
|
172
|
+
by the workflow on first scan)
|
|
173
|
+
|
|
174
|
+
3. **If any gate fails:** Read the error, fix the root cause,
|
|
175
|
+
push again. Do not suppress gates or skip hooks to make CI
|
|
176
|
+
green — fix the actual issue.
|
|
177
|
+
|
|
178
|
+
4. **Log to `docs/CANARY_STATE.md`** that harness setup was
|
|
179
|
+
completed, including the date and commit SHA.
|
|
180
|
+
|
|
181
|
+
## Canary Integration
|
|
182
|
+
|
|
183
|
+
- **`harness.config.json`** — Layer rules, entrypoints,
|
|
184
|
+
telemetry/adoption settings. Source of truth for all
|
|
185
|
+
harness gates.
|
|
186
|
+
- **`.harness/hooks/`** — Local git hooks generated during
|
|
187
|
+
`harness init`. Committed so all contributors share them.
|
|
188
|
+
- **`.github/workflows/`** — CI workflows that run the harness
|
|
189
|
+
gates on every PR.
|
|
190
|
+
- **`scripts/security_ledger.py`** — Regenerates
|
|
191
|
+
`docs/SECURITY_LEDGER.md` from `.harness/security/timeline.json`.
|
|
192
|
+
Must be run after every security scan.
|
|
193
|
+
- **`docs/CANARY_STATE.md`** — Project ledger. Log setup
|
|
194
|
+
completion here.
|
|
195
|
+
|
|
196
|
+
## Success Criteria
|
|
197
|
+
|
|
198
|
+
- `harness validate` exits cleanly with no violations
|
|
199
|
+
- All five CI workflows exist and pass on a clean push
|
|
200
|
+
- The security ledger (`docs/SECURITY_LEDGER.md`) exists and
|
|
201
|
+
is not stale
|
|
202
|
+
- `.harness/hooks/` is committed and present in the repo
|
|
203
|
+
- A setup entry exists in `docs/CANARY_STATE.md`
|
|
204
|
+
|
|
205
|
+
## Rationalizations to Reject
|
|
206
|
+
|
|
207
|
+
| Rationalization | Why It Is Wrong |
|
|
208
|
+
| --- | --- |
|
|
209
|
+
| "CI is red but the code is fine, I'll add `--no-verify`" | Skipping hooks defeats the entire purpose of the harness. Fix the root cause. |
|
|
210
|
+
| "I'll set up the workflows later — the config is good enough" | A config with no CI enforcement is ornamental. The gates only protect the team when they run on every PR. |
|
|
211
|
+
| "The security ledger step seems redundant — the scan already ran" | The scan writes to a JSON timeline; the ledger script produces the human-readable summary. Both are needed. |
|
|
212
|
+
| "I'll commit the hooks directory but not review the files" | Hook files run on every developer's machine. Review them before committing — they're executable scripts. |
|
|
213
|
+
|
|
214
|
+
## Examples
|
|
215
|
+
|
|
216
|
+
### Example: Fresh install on a new fork
|
|
217
|
+
|
|
218
|
+
**Scenario:** `cap-canary` is a new fork of `canary-test-ai-agent`.
|
|
219
|
+
It has no harness config yet.
|
|
220
|
+
|
|
221
|
+
**Steps:**
|
|
222
|
+
|
|
223
|
+
1. Clone the fork and run `harness init`.
|
|
224
|
+
2. Review `harness.config.json` — layers match `llm`, `core`,
|
|
225
|
+
`cli`. Entrypoints set to `agent`.
|
|
226
|
+
3. Copy the five CI workflow files from upstream.
|
|
227
|
+
4. Run the security scan, generate the baseline ledger, commit.
|
|
228
|
+
5. Push. All five gates pass. Log to `CANARY_STATE.md`.
|
|
229
|
+
|
|
230
|
+
### Example: Re-aligning a drift after harness update
|
|
231
|
+
|
|
232
|
+
**Scenario:** A harness CLI update added a new hook file to
|
|
233
|
+
`.harness/hooks/`. The local copy doesn't have it.
|
|
234
|
+
|
|
235
|
+
**Steps:**
|
|
236
|
+
|
|
237
|
+
1. Run `npx --yes -p @harness-engineering/cli harness update`.
|
|
238
|
+
2. Review the diff in `.harness/hooks/` — confirm the new
|
|
239
|
+
file is expected.
|
|
240
|
+
3. Stage and commit the new hook file.
|
|
241
|
+
4. Push. CI passes.
|
|
242
|
+
|
|
243
|
+
## Escalation
|
|
244
|
+
|
|
245
|
+
- **When `harness init` fails or produces unexpected output:**
|
|
246
|
+
Check the harness CLI version. A major version mismatch
|
|
247
|
+
between the CLI and the `harness.config.json` schema is the
|
|
248
|
+
most common cause.
|
|
249
|
+
- **When a CI gate fails after a clean setup:** Don't assume
|
|
250
|
+
the setup is wrong. Read the failure output — it often points
|
|
251
|
+
to a pre-existing issue in the repo that the harness is now
|
|
252
|
+
surfacing for the first time.
|
|
253
|
+
- **When a fork needs company-specific config injected:**
|
|
254
|
+
Set up the base harness using this skill, then apply the
|
|
255
|
+
company-specific overrides as a follow-up commit. Don't
|
|
256
|
+
mix upstream setup with fork-specific changes.
|
|
257
|
+
- **When the wiki-sync workflow fails on first run:** The
|
|
258
|
+
GitHub wiki is a separate git repository that GitHub creates
|
|
259
|
+
lazily — it doesn't exist until someone saves the first page
|
|
260
|
+
via the UI. Go to the repo → Wiki → create any placeholder
|
|
261
|
+
page. After that, the wiki-sync workflow handles the wiki
|
|
262
|
+
repo correctly on all subsequent runs. Each new fork needs
|
|
263
|
+
this one-time step.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: canary-shadow
|
|
3
|
+
description: >
|
|
4
|
+
Differential ("shadow") parity testing — run the SAME invocation through a
|
|
5
|
+
baseline and a candidate, normalize away irrelevant noise (ANSI, timestamps,
|
|
6
|
+
temp paths, version banners, SHAs), diff, and flag divergences. Use to prove a
|
|
7
|
+
behavior-preserving change actually preserves behavior: a language/framework
|
|
8
|
+
port (Python→TS, typer→commander), a skill rewrite (old vs new), a refactor
|
|
9
|
+
claimed equivalent, or "does this still match the old CLI?". Runs both sides
|
|
10
|
+
live, so it needs no golden capture — just the two versions. NOT a correctness
|
|
11
|
+
oracle: it proves the two AGREE, not that either is right, so pair it with a
|
|
12
|
+
few asserted cases.
|
|
13
|
+
cli: scripts/cli.mjs
|
|
14
|
+
requires: [node>=20]
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
# Canary: Shadow (differential parity testing)
|
|
18
|
+
|
|
19
|
+
> Run one invocation through two implementations, normalize the noise, diff the
|
|
20
|
+
> rest. Every un-accepted divergence is a behavior change to explain or fix.
|
|
21
|
+
|
|
22
|
+
## When to Use
|
|
23
|
+
|
|
24
|
+
- **Ports / rewrites** where output should be identical: Python→TS, a BoP skill
|
|
25
|
+
re-port, typer→commander, one engine replacing another. (This is exactly how
|
|
26
|
+
the v6 cutover was validated: run `canary <cmd>` through the TS engine AND the
|
|
27
|
+
Python reference, diff. It surfaced a divergence the unit tests + golden suite
|
|
28
|
+
missed — Python's `rich` console renders the ticket-marker hint
|
|
29
|
+
`# canary:ticket:` as `# canary🎫` via emoji-shortcode substitution, mangling
|
|
30
|
+
a hint the parser can't match; the TS port printed the literal, correct
|
|
31
|
+
marker. A console-rendering artifact only a live side-by-side run exposes.)
|
|
32
|
+
- **Skill changes** — old skill vs new skill on the same inputs: "prove the
|
|
33
|
+
rewrite is equivalent."
|
|
34
|
+
- **Refactors** asserted to be behavior-preserving.
|
|
35
|
+
- **Framework / dependency swaps** where the observable surface should not move.
|
|
36
|
+
- NOT for greenfield behavior (nothing to compare against) — use asserted tests.
|
|
37
|
+
- NOT as a sole correctness check — see the caveat at the bottom.
|
|
38
|
+
|
|
39
|
+
## The idea (differential / shadow testing)
|
|
40
|
+
|
|
41
|
+
For each case, run the SAME arguments through a `baseline` command and a
|
|
42
|
+
`candidate` command, capture `{exitCode, stdout, stderr}`, **normalize** both,
|
|
43
|
+
and diff. Identical (post-normalization) ⇒ `ok`. Different ⇒ `DIVERGE` — a real
|
|
44
|
+
behavior change, unless it's a _documented, intentional_ difference recorded in
|
|
45
|
+
the accepted-divergence allowlist.
|
|
46
|
+
|
|
47
|
+
The value is not the runner (two `spawn`s and a diff) — it's the two disciplines
|
|
48
|
+
below.
|
|
49
|
+
|
|
50
|
+
### 1. The normalization ruleset (get this right, or drown)
|
|
51
|
+
|
|
52
|
+
Output has meaningful content and irrelevant noise. Normalize the noise; keep
|
|
53
|
+
everything else. Default masks (see `scripts/cli.mjs`): ANSI SGR codes, ISO
|
|
54
|
+
timestamps, temp paths (`/tmp`, `/var/folders`, `/private`), version banners
|
|
55
|
+
(`vX.Y.Z[-rc.N]`), commit SHAs, run-ids.
|
|
56
|
+
|
|
57
|
+
- **Under-normalize** and every run is a wall of false DIVERGE (a timestamp, a
|
|
58
|
+
temp dir) — you stop reading them.
|
|
59
|
+
- **Over-normalize** and you mask a real bug (don't blanket-strip numbers if the
|
|
60
|
+
command emits counts/scores that matter).
|
|
61
|
+
- Make masking **visible**: the runner reports what each mask touched, so a
|
|
62
|
+
reviewer can see a real difference wasn't hidden. Tune masks per project in
|
|
63
|
+
the cases file's `normalize` list, don't hardcode.
|
|
64
|
+
|
|
65
|
+
### 2. The accepted-divergence allowlist
|
|
66
|
+
|
|
67
|
+
Some differences are intentional and permanent (e.g. Python `rich` soft-wraps
|
|
68
|
+
prose at 80 cols; a TS port emits it unwrapped — same content, different line
|
|
69
|
+
breaks). Record each in the cases file with a **reason**, keyed by case label.
|
|
70
|
+
This is the differential-test analog of `// harness-ignore`: an allowlisted
|
|
71
|
+
DIVERGE is reported as `accept` (with its reason), never as a failure, and a NEW
|
|
72
|
+
divergence still fails loudly. Review the allowlist like code — an entry with no
|
|
73
|
+
reason, or one that quietly grows, is a smell.
|
|
74
|
+
|
|
75
|
+
## Process
|
|
76
|
+
|
|
77
|
+
### Phase 1: SCOPE — define baseline, candidate, and cases
|
|
78
|
+
|
|
79
|
+
1. Identify the two implementations and how to invoke each (argv-preserving):
|
|
80
|
+
`baseline` = the trusted side (old skill, Python reference, prior version),
|
|
81
|
+
`candidate` = the new side.
|
|
82
|
+
2. Enumerate cases that cover the observable surface: the happy paths, the JSON
|
|
83
|
+
contracts, the error/exit-code paths (a good differential run checks exit
|
|
84
|
+
codes AND stdout), and any command that needs a fixture (a sample file, a
|
|
85
|
+
temp project). Prefer read-only/deterministic cases first; give
|
|
86
|
+
state-changing ones (`init`, `migrate`) an isolated `cwd`.
|
|
87
|
+
|
|
88
|
+
### Phase 2: RUN — cycle until clean
|
|
89
|
+
|
|
90
|
+
1. Write a cases file (see `scripts/cases.example.json`) and run:
|
|
91
|
+
`node scripts/cli.mjs --cases <file>` (or via the skill `cli`). It prints
|
|
92
|
+
`ok` / `DIVERGE` / `accept` per case and a summary; exit is non-zero if any
|
|
93
|
+
un-accepted divergence remains.
|
|
94
|
+
2. For each DIVERGE: read the diff. Decide — **bug** (fix the candidate),
|
|
95
|
+
**noise** (add/adjust a normalize rule), or **intentional** (add an allowlist
|
|
96
|
+
entry WITH a reason). Re-run. Iterate in cycles until the only remaining
|
|
97
|
+
divergences are `accept`ed.
|
|
98
|
+
3. Do multiple cycles broadening coverage (add sub-commands, more fixtures,
|
|
99
|
+
adversarial inputs) — the first cycle finds the obvious breaks; the tail
|
|
100
|
+
finds the subtle ones.
|
|
101
|
+
|
|
102
|
+
### Phase 3: LOCK — keep it honest
|
|
103
|
+
|
|
104
|
+
1. Commit the cases file + allowlist alongside the change so the parity contract
|
|
105
|
+
is reviewable and re-runnable in CI.
|
|
106
|
+
2. Add a handful of **asserted** cases (expected exact output/exit for a few
|
|
107
|
+
invocations) so the suite also catches "both sides are wrong the same way."
|
|
108
|
+
|
|
109
|
+
## Invocation
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
# Usage, flags, and the cases-file contract (exits 0):
|
|
113
|
+
canary skills run canary-shadow -- --help
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
A run needs a cases file, so the shape below is illustrative — copy
|
|
117
|
+
`scripts/cases.example.json`, point it at your two implementations, and
|
|
118
|
+
substitute the path:
|
|
119
|
+
|
|
120
|
+
<!-- canary:illustrative -->
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
canary skills run canary-shadow -- --cases <cases.json>
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## Caveat — agreement is not correctness
|
|
127
|
+
|
|
128
|
+
Differential testing proves the two implementations AGREE. If the baseline has a
|
|
129
|
+
bug the candidate faithfully reproduces, both pass. So it is a powerful
|
|
130
|
+
regression net, not a correctness oracle — always pair it with a few asserted
|
|
131
|
+
cases and human review of the allowlist.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"//": "Example shadow-parity config: the released CLI (baseline) vs the local build (candidate) — does this branch still agree with what users have? Copy, point baseline/candidate at your two implementations, and edit `cases`. Run: node cli.mjs --cases cases.example.json",
|
|
3
|
+
"baseline": ["npx", "--yes", "canary-test-cli@6.6.0"],
|
|
4
|
+
"candidate": ["node", "ts/bin/canary.js"],
|
|
5
|
+
"env": { "NO_COLOR": "1" },
|
|
6
|
+
"normalize": ["ansi", "version", "timestamp", "tmppath", "sha"],
|
|
7
|
+
"accept": {
|
|
8
|
+
"skills list": "The released build soft-wraps long skill descriptions at 80 cols; the local build emits them unwrapped. Same content + names, different line breaks — a documented, intentional presentation deviation, not a contract."
|
|
9
|
+
},
|
|
10
|
+
"cases": [
|
|
11
|
+
{ "label": "frameworks --json", "argv": ["frameworks", "--json"] },
|
|
12
|
+
{ "label": "frameworks", "argv": ["frameworks"] },
|
|
13
|
+
{
|
|
14
|
+
"label": "recommend e2e --json",
|
|
15
|
+
"argv": ["recommend", "test a login form", "--json"]
|
|
16
|
+
},
|
|
17
|
+
{ "label": "recommend prose", "argv": ["recommend", "test a login form"] },
|
|
18
|
+
{ "label": "skills list", "argv": ["skills", "list"] },
|
|
19
|
+
{
|
|
20
|
+
"label": "bad: unknown option (exit 2)",
|
|
21
|
+
"argv": ["frameworks", "--nope"]
|
|
22
|
+
},
|
|
23
|
+
{ "label": "bad: missing arg (exit 2)", "argv": ["recommend"] },
|
|
24
|
+
{ "label": "bad: unknown command (exit 2)", "argv": ["frobnicate"] },
|
|
25
|
+
{ "label": "analyze flaky --json", "argv": ["analyze", "flaky", "--json"] },
|
|
26
|
+
{
|
|
27
|
+
"label": "init playwright (isolated cwd)",
|
|
28
|
+
"argv": ["init", "playwright"],
|
|
29
|
+
"isolate": true
|
|
30
|
+
}
|
|
31
|
+
]
|
|
32
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// canary-shadow — differential ("shadow") parity runner.
|
|
3
|
+
//
|
|
4
|
+
// Runs each case's argv through a `baseline` and a `candidate` command,
|
|
5
|
+
// normalizes away irrelevant noise, and diffs exit code + stdout. Prints
|
|
6
|
+
// ok / accept / DIVERGE per case; exits non-zero if any un-accepted divergence
|
|
7
|
+
// remains. See ../SKILL.md for the methodology and cases.example.json for the
|
|
8
|
+
// config shape.
|
|
9
|
+
//
|
|
10
|
+
// Usage: node cli.mjs --cases <cases.json> [--verbose]
|
|
11
|
+
import { spawnSync } from 'node:child_process';
|
|
12
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
13
|
+
import { tmpdir } from 'node:os';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
|
|
16
|
+
import {
|
|
17
|
+
createParser,
|
|
18
|
+
formatUsageError,
|
|
19
|
+
EXIT_USAGE,
|
|
20
|
+
} from '../../../lib/parse-args.mjs';
|
|
21
|
+
|
|
22
|
+
const USAGE =
|
|
23
|
+
'usage: canary-shadow [-h] --cases PATH [--verbose]\n' +
|
|
24
|
+
'\n' +
|
|
25
|
+
'Run each case through a baseline and a candidate command, normalize away\n' +
|
|
26
|
+
'irrelevant noise, and diff exit code + stdout.\n' +
|
|
27
|
+
'\n' +
|
|
28
|
+
'options:\n' +
|
|
29
|
+
' -h, --help show this help message and exit\n' +
|
|
30
|
+
' --cases PATH cases file describing baseline, candidate, and cases\n' +
|
|
31
|
+
' --verbose print a line per matching case, not just divergences';
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* canary-shadow shipped BEFORE the family had a shared contract, and it showed:
|
|
35
|
+
* no `--help`, no unknown-flag rejection, and a usage line naming `cli.mjs`
|
|
36
|
+
* rather than the skill. Nobody noticed, because there was nothing to violate
|
|
37
|
+
* (#479).
|
|
38
|
+
*/
|
|
39
|
+
export const CLI_SPEC = {
|
|
40
|
+
prog: 'canary-shadow',
|
|
41
|
+
booleans: { '--verbose': 'verbose' },
|
|
42
|
+
values: { '--cases': { key: 'cases' } },
|
|
43
|
+
required: ['--cases'],
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const parseArgs = createParser(CLI_SPEC);
|
|
47
|
+
|
|
48
|
+
// --- built-in normalization masks (name -> {re, replace}). Each records
|
|
49
|
+
// whether it fired so the reviewer can see what was hidden. Extend/override per
|
|
50
|
+
// project via the cases file's `normalize` list. ---
|
|
51
|
+
const MASKS = {
|
|
52
|
+
ansi: { re: /\x1b\[[0-9;]*m/g, replace: '' },
|
|
53
|
+
version: { re: /v\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?/g, replace: 'vX' },
|
|
54
|
+
timestamp: {
|
|
55
|
+
re: /\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g,
|
|
56
|
+
replace: 'TS',
|
|
57
|
+
},
|
|
58
|
+
tmppath: { re: /(?:\/private)?\/(?:var|tmp)\/[^\s"']+/g, replace: 'TMP' },
|
|
59
|
+
sha: { re: /\b[0-9a-f]{7,40}\b/g, replace: 'SHA' },
|
|
60
|
+
runid: { re: /\b(?:run|job)[-_]?\d{3,}\b/gi, replace: 'RUNID' },
|
|
61
|
+
};
|
|
62
|
+
const DEFAULT_MASKS = ['ansi', 'version', 'timestamp', 'tmppath', 'sha'];
|
|
63
|
+
|
|
64
|
+
function buildMasks(spec) {
|
|
65
|
+
// spec: array of built-in names and/or {name, pattern, flags?, replace}
|
|
66
|
+
const list = [];
|
|
67
|
+
for (const m of spec ?? DEFAULT_MASKS) {
|
|
68
|
+
if (typeof m === 'string') {
|
|
69
|
+
if (!MASKS[m]) throw new Error(`unknown built-in mask: ${m}`);
|
|
70
|
+
list.push({ name: m, ...MASKS[m] });
|
|
71
|
+
} else {
|
|
72
|
+
list.push({
|
|
73
|
+
name: m.name ?? m.pattern,
|
|
74
|
+
re: new RegExp(m.pattern, m.flags ?? 'g'),
|
|
75
|
+
replace: m.replace ?? '',
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return list;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function normalize(text, masks, fired) {
|
|
83
|
+
let out = text;
|
|
84
|
+
for (const m of masks) {
|
|
85
|
+
if (m.re.test(out)) fired.add(m.name);
|
|
86
|
+
m.re.lastIndex = 0;
|
|
87
|
+
out = out.replace(m.re, m.replace);
|
|
88
|
+
}
|
|
89
|
+
return out.replace(/[ \t]+$/gm, '').trim();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function run(cmd, argv, cwd, env) {
|
|
93
|
+
const r = spawnSync(cmd[0], [...cmd.slice(1), ...argv], {
|
|
94
|
+
cwd,
|
|
95
|
+
env: { ...process.env, NO_COLOR: '1', ...env },
|
|
96
|
+
encoding: 'utf-8',
|
|
97
|
+
maxBuffer: Infinity,
|
|
98
|
+
timeout: 120_000,
|
|
99
|
+
});
|
|
100
|
+
return {
|
|
101
|
+
code: r.status ?? 1,
|
|
102
|
+
stdout: r.stdout ?? '',
|
|
103
|
+
stderr: r.stderr ?? '',
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function unifiedDiff(a, b, max = 30) {
|
|
108
|
+
const A = a.split('\n');
|
|
109
|
+
const B = b.split('\n');
|
|
110
|
+
// minimal LCS-free line diff: good enough for review, keeps the runner small.
|
|
111
|
+
const out = [];
|
|
112
|
+
const bSet = new Set(B);
|
|
113
|
+
const aSet = new Set(A);
|
|
114
|
+
for (const line of A) if (!bSet.has(line)) out.push('- ' + line);
|
|
115
|
+
for (const line of B) if (!aSet.has(line)) out.push('+ ' + line);
|
|
116
|
+
return out.slice(0, max);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function main(argv = []) {
|
|
120
|
+
const { opts, help, error } = parseArgs(argv);
|
|
121
|
+
|
|
122
|
+
if (help) {
|
|
123
|
+
console.log(USAGE);
|
|
124
|
+
return 0;
|
|
125
|
+
}
|
|
126
|
+
if (error) {
|
|
127
|
+
console.error(formatUsageError(CLI_SPEC.prog, error));
|
|
128
|
+
return EXIT_USAGE;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const { cases: casesPath, verbose } = opts;
|
|
132
|
+
const cfg = JSON.parse(readFileSync(casesPath, 'utf-8'));
|
|
133
|
+
const masks = buildMasks(cfg.normalize);
|
|
134
|
+
// Null-prototype: `accept` comes straight from JSON, so on a plain object a
|
|
135
|
+
// case labelled `toString` resolved to Object.prototype.toString and was
|
|
136
|
+
// reported `accept` instead of DIVERGE -- a parity tool silently suppressing
|
|
137
|
+
// a parity failure, which is the worst place in the family for this bug to
|
|
138
|
+
// have landed (#479).
|
|
139
|
+
const accept = Object.assign(Object.create(null), cfg.accept ?? {});
|
|
140
|
+
const fired = new Set();
|
|
141
|
+
|
|
142
|
+
let ok = 0;
|
|
143
|
+
let accepted = 0;
|
|
144
|
+
let diverged = 0;
|
|
145
|
+
for (const c of cfg.cases) {
|
|
146
|
+
let cwd = cfg.cwd;
|
|
147
|
+
let tmp;
|
|
148
|
+
if (c.isolate) {
|
|
149
|
+
tmp = mkdtempSync(join(tmpdir(), 'shadow-'));
|
|
150
|
+
cwd = tmp;
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
const base = run(cfg.baseline, c.argv, cwd, cfg.env);
|
|
154
|
+
const cand = run(cfg.candidate, c.argv, cwd, cfg.env);
|
|
155
|
+
const bOut = normalize(base.stdout, masks, fired);
|
|
156
|
+
const cOut = normalize(cand.stdout, masks, fired);
|
|
157
|
+
const match = base.code === cand.code && bOut === cOut;
|
|
158
|
+
if (match) {
|
|
159
|
+
ok++;
|
|
160
|
+
if (verbose)
|
|
161
|
+
process.stdout.write(`ok ${c.label} (exit ${base.code})\n`);
|
|
162
|
+
} else if (accept[c.label]) {
|
|
163
|
+
accepted++;
|
|
164
|
+
process.stdout.write(`accept ${c.label} — ${accept[c.label]}\n`);
|
|
165
|
+
} else {
|
|
166
|
+
diverged++;
|
|
167
|
+
process.stdout.write(
|
|
168
|
+
`DIVERGE ${c.label} (exit base=${base.code} cand=${cand.code})\n`,
|
|
169
|
+
);
|
|
170
|
+
if (base.code !== cand.code)
|
|
171
|
+
process.stdout.write(
|
|
172
|
+
` exit differs: base=${base.code} cand=${cand.code}\n`,
|
|
173
|
+
);
|
|
174
|
+
for (const line of unifiedDiff(bOut, cOut))
|
|
175
|
+
process.stdout.write(' ' + line + '\n');
|
|
176
|
+
}
|
|
177
|
+
} finally {
|
|
178
|
+
if (tmp) rmSync(tmp, { recursive: true, force: true });
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
process.stdout.write(
|
|
182
|
+
`\n=== ${ok} ok, ${accepted} accepted, ${diverged} DIVERGE ` +
|
|
183
|
+
`(${cfg.cases.length} cases) — masks fired: ${[...fired].join(', ') || 'none'} ===\n`,
|
|
184
|
+
);
|
|
185
|
+
return diverged > 0 ? 1 : 0;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Direct execution (the skill runner execs this file via its shebang).
|
|
189
|
+
//
|
|
190
|
+
// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
|
|
191
|
+
// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
|
|
192
|
+
// truncated JSON that still exits 0 (#791).
|
|
193
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
194
|
+
process.exitCode = main(process.argv.slice(2));
|
|
195
|
+
}
|