canary-test-cli 7.1.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/agents/skills/README.md +308 -0
  2. package/agents/skills/canary:generate.md +49 -0
  3. package/agents/skills/canary:init.md +37 -0
  4. package/agents/skills/canary:migrate.md +66 -0
  5. package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
  6. package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
  7. package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
  8. package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
  9. package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
  10. package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
  11. package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
  12. package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
  13. package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
  14. package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
  15. package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
  16. package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
  17. package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
  18. package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
  19. package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
  20. package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
  21. package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
  22. package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
  23. package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
  24. package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
  25. package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
  26. package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
  27. package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
  28. package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
  29. package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
  30. package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
  31. package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
  32. package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
  33. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
  34. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
  35. package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
  36. package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
  37. package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
  38. package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
  39. package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
  40. package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
  41. package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
  42. package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
  43. package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
  44. package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
  45. package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
  46. package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
  47. package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
  48. package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
  49. package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
  50. package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
  51. package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
  52. package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
  53. package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
  54. package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
  55. package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
  56. package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
  57. package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
  58. package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
  59. package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
  60. package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
  61. package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
  62. package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
  63. package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
  64. package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
  65. package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
  66. package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
  67. package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
  68. package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
  69. package/agents/skills/lib/parse-args.mjs +214 -0
  70. package/dist/engine/cli-commands.js +3 -65
  71. package/dist/engine/core/quality-scorer.js +15 -2
  72. package/dist/engine/core/skill-dispatch.js +115 -0
  73. package/dist/engine/core/skill-examples.js +103 -3
  74. package/dist/engine/core/skill-registry.js +59 -4
  75. package/dist/engine/core/test-files.js +77 -0
  76. package/dist/engine/core/vacuity-scanner.js +184 -14
  77. package/dist/engine/guardian/analysis-emit.js +7 -2
  78. package/dist/engine/guardian/cli.js +121 -9
  79. package/dist/engine/guardian/pr-check.js +317 -18
  80. package/dist/engine/skills-cli.js +123 -19
  81. package/dist/reporters/testtracker.d.ts +1 -1
  82. package/dist/reporters/testtracker.js +1 -1
  83. package/package.json +3 -2
@@ -0,0 +1,263 @@
1
+ ---
2
+ name: canary-setup-harness
3
+ description: >
4
+ Configure the Harness Engineering guardrails in a new Canary project (or a
5
+ fork) — installs the harness CLI, initialises `harness.config.json`, wires
6
+ up the five required CI workflows, generates the initial security ledger,
7
+ and verifies all gates pass on a clean repository. Use for "set up harness
8
+ for this project", "onboard this repo with harness", "install the harness
9
+ CLI", "configure the CI gates", or when a fork has drifted from the
10
+ harness config and needs re-alignment. Not for day-to-day harness updates,
11
+ adding a single new gate to an existing config, or troubleshooting a
12
+ failing CI check.
13
+ ---
14
+
15
+ # Canary: Setup Harness
16
+
17
+ > Configure the Harness Engineering guardrails in a new Canary
18
+ > project (or a fork). Installs the harness CLI, initialises the
19
+ > config, wires up CI workflows, and verifies all gates pass on
20
+ > a clean repository.
21
+
22
+ ## When to Use
23
+
24
+ - When setting up Canary (or a fork) in a new repository for
25
+ the first time
26
+ - When onboarding a new project that wants Canary's CI gate
27
+ structure
28
+ - When a fork has drifted from the harness config and needs
29
+ to be re-aligned
30
+ - NOT for day-to-day harness updates — use the harness CLI's
31
+ own update flow for that
32
+ - NOT for adding a single new gate to an existing config —
33
+ edit `harness.config.json` directly
34
+ - NOT for troubleshooting a failing CI check — see the
35
+ [Understanding the Harness][harness-wiki] wiki page
36
+
37
+ [harness-wiki]: ../../../../docs/wiki/Understanding-the-Harness.md
38
+
39
+ ## Process
40
+
41
+ ### Phase 1: PREREQUISITES — Confirm the Environment
42
+
43
+ 1. **Verify Node.js is available** (harness CLI requires it):
44
+
45
+ ```bash
46
+ node --version
47
+ ```
48
+
49
+ Version 18 or later is required.
50
+
51
+ 2. **Verify Python is available** (security ledger script):
52
+
53
+ ```bash
54
+ python3 --version
55
+ ```
56
+
57
+ Version 3.11 or later is required.
58
+
59
+ 3. **Confirm the repo has a `harness.config.json`** at the
60
+ root. If it doesn't, the harness is not yet configured —
61
+ continue to Phase 2. If it does, validate it first:
62
+
63
+ ```bash
64
+ npx --yes -p @harness-engineering/cli harness validate
65
+ ```
66
+
67
+ If validation passes, the harness is already set up. Stop
68
+ here unless re-alignment is the goal.
69
+
70
+ ### Phase 2: INSTALL — Run the Harness Init
71
+
72
+ 1. **Initialise the harness** from the repo root:
73
+
74
+ ```bash
75
+ npx --yes -p @harness-engineering/cli harness init
76
+ ```
77
+
78
+ This creates `harness.config.json` with default gates and
79
+ generates the `.harness/` hooks directory.
80
+
81
+ 2. **Review the generated `harness.config.json`.** The defaults
82
+ are sensible, but confirm:
83
+
84
+ - `layers` matches the project's actual package structure
85
+ (for Canary: `llm`, `core`, `cli`)
86
+ - `entrypoints` lists the correct top-level packages
87
+ - `telemetry` and `adoption` settings match team policy
88
+
89
+ 3. **Inspect `.harness/hooks/`.** The hooks directory contains
90
+ scripts that run locally on git events. Review each file
91
+ briefly — they should not require network access outside
92
+ the harness telemetry endpoint.
93
+
94
+ ### Phase 3: CONFIGURE — Wire Up CI Workflows
95
+
96
+ 1. **Check which workflows already exist:**
97
+
98
+ ```bash
99
+ ls .github/workflows/
100
+ ```
101
+
102
+ 2. **Required workflows for a full harness setup.** Each
103
+ should exist as a `.yml` file in `.github/workflows/`:
104
+
105
+ - `harness.yml` — core phase-gate checks
106
+ - `harness-architecture.yml` — layer dependency validation
107
+ - `harness-quality.yml` — quality and integrity checks
108
+ - `harness-security.yml` — security scan + ledger refresh
109
+ - `docs-lint.yml` — markdown formatting enforcement
110
+
111
+ If any are missing, copy them from an upstream Canary
112
+ reference install or generate them:
113
+
114
+ ```bash
115
+ npx --yes -p @harness-engineering/cli harness \
116
+ generate-workflows
117
+ ```
118
+
119
+ 3. **Confirm `harness-security.yml` refreshes the ledger.**
120
+ The final steps of the security job must run the ledger
121
+ script and commit if changed. Without this, the security
122
+ ledger goes stale and the Quality gate fails.
123
+
124
+ 4. **Confirm `docs-lint.yml` covers all doc paths.** The
125
+ workflow's `paths` filter should include:
126
+
127
+ - `docs/**`
128
+ - `agents/**`
129
+ - `AGENTS.md`
130
+
131
+ 5. **Set required permissions.** Workflows that commit
132
+ back to the repository need `contents: write`:
133
+
134
+ ```yaml
135
+ permissions:
136
+ contents: write
137
+ ```
138
+
139
+ This applies to `harness-security.yml` at minimum.
140
+
141
+ ### Phase 4: BASELINE — Generate Initial Artifacts
142
+
143
+ 1. **Run the security scan to create the initial ledger:**
144
+
145
+ ```bash
146
+ npx --yes -p @harness-engineering/cli harness check-security
147
+ python3 scripts/security_ledger.py
148
+ ```
149
+
150
+ 2. **Commit the generated baseline files:**
151
+
152
+ ```bash
153
+ git add harness.config.json .harness/ \
154
+ .harness/security/timeline.json \
155
+ docs/SECURITY_LEDGER.md
156
+ git commit -m "chore: initialise harness configuration"
157
+ ```
158
+
159
+ 3. **Push and confirm all CI gates pass** on the resulting
160
+ commit before calling setup complete.
161
+
162
+ ### Phase 5: VERIFY — Confirm All Gates Pass
163
+
164
+ 1. **Open a pull request** (or push to the configured branch)
165
+ to trigger CI.
166
+ 2. **Check each workflow:**
167
+
168
+ - Architecture Enforcer: green
169
+ - Quality & Integrity: green
170
+ - Docs Lint: green
171
+ - Security Reviewer: green (ledger will be auto-refreshed
172
+ by the workflow on first scan)
173
+
174
+ 3. **If any gate fails:** Read the error, fix the root cause,
175
+ push again. Do not suppress gates or skip hooks to make CI
176
+ green — fix the actual issue.
177
+
178
+ 4. **Log to `docs/CANARY_STATE.md`** that harness setup was
179
+ completed, including the date and commit SHA.
180
+
181
+ ## Canary Integration
182
+
183
+ - **`harness.config.json`** — Layer rules, entrypoints,
184
+ telemetry/adoption settings. Source of truth for all
185
+ harness gates.
186
+ - **`.harness/hooks/`** — Local git hooks generated during
187
+ `harness init`. Committed so all contributors share them.
188
+ - **`.github/workflows/`** — CI workflows that run the harness
189
+ gates on every PR.
190
+ - **`scripts/security_ledger.py`** — Regenerates
191
+ `docs/SECURITY_LEDGER.md` from `.harness/security/timeline.json`.
192
+ Must be run after every security scan.
193
+ - **`docs/CANARY_STATE.md`** — Project ledger. Log setup
194
+ completion here.
195
+
196
+ ## Success Criteria
197
+
198
+ - `harness validate` exits cleanly with no violations
199
+ - All five CI workflows exist and pass on a clean push
200
+ - The security ledger (`docs/SECURITY_LEDGER.md`) exists and
201
+ is not stale
202
+ - `.harness/hooks/` is committed and present in the repo
203
+ - A setup entry exists in `docs/CANARY_STATE.md`
204
+
205
+ ## Rationalizations to Reject
206
+
207
+ | Rationalization | Why It Is Wrong |
208
+ | --- | --- |
209
+ | "CI is red but the code is fine, I'll add `--no-verify`" | Skipping hooks defeats the entire purpose of the harness. Fix the root cause. |
210
+ | "I'll set up the workflows later — the config is good enough" | A config with no CI enforcement is ornamental. The gates only protect the team when they run on every PR. |
211
+ | "The security ledger step seems redundant — the scan already ran" | The scan writes to a JSON timeline; the ledger script produces the human-readable summary. Both are needed. |
212
+ | "I'll commit the hooks directory but not review the files" | Hook files run on every developer's machine. Review them before committing — they're executable scripts. |
213
+
214
+ ## Examples
215
+
216
+ ### Example: Fresh install on a new fork
217
+
218
+ **Scenario:** `cap-canary` is a new fork of `canary-test-ai-agent`.
219
+ It has no harness config yet.
220
+
221
+ **Steps:**
222
+
223
+ 1. Clone the fork and run `harness init`.
224
+ 2. Review `harness.config.json` — layers match `llm`, `core`,
225
+ `cli`. Entrypoints set to `agent`.
226
+ 3. Copy the five CI workflow files from upstream.
227
+ 4. Run the security scan, generate the baseline ledger, commit.
228
+ 5. Push. All five gates pass. Log to `CANARY_STATE.md`.
229
+
230
+ ### Example: Re-aligning a drift after harness update
231
+
232
+ **Scenario:** A harness CLI update added a new hook file to
233
+ `.harness/hooks/`. The local copy doesn't have it.
234
+
235
+ **Steps:**
236
+
237
+ 1. Run `npx --yes -p @harness-engineering/cli harness update`.
238
+ 2. Review the diff in `.harness/hooks/` — confirm the new
239
+ file is expected.
240
+ 3. Stage and commit the new hook file.
241
+ 4. Push. CI passes.
242
+
243
+ ## Escalation
244
+
245
+ - **When `harness init` fails or produces unexpected output:**
246
+ Check the harness CLI version. A major version mismatch
247
+ between the CLI and the `harness.config.json` schema is the
248
+ most common cause.
249
+ - **When a CI gate fails after a clean setup:** Don't assume
250
+ the setup is wrong. Read the failure output — it often points
251
+ to a pre-existing issue in the repo that the harness is now
252
+ surfacing for the first time.
253
+ - **When a fork needs company-specific config injected:**
254
+ Set up the base harness using this skill, then apply the
255
+ company-specific overrides as a follow-up commit. Don't
256
+ mix upstream setup with fork-specific changes.
257
+ - **When the wiki-sync workflow fails on first run:** The
258
+ GitHub wiki is a separate git repository that GitHub creates
259
+ lazily — it doesn't exist until someone saves the first page
260
+ via the UI. Go to the repo → Wiki → create any placeholder
261
+ page. After that, the wiki-sync workflow handles the wiki
262
+ repo correctly on all subsequent runs. Each new fork needs
263
+ this one-time step.
@@ -0,0 +1,131 @@
1
+ ---
2
+ name: canary-shadow
3
+ description: >
4
+ Differential ("shadow") parity testing — run the SAME invocation through a
5
+ baseline and a candidate, normalize away irrelevant noise (ANSI, timestamps,
6
+ temp paths, version banners, SHAs), diff, and flag divergences. Use to prove a
7
+ behavior-preserving change actually preserves behavior: a language/framework
8
+ port (Python→TS, typer→commander), a skill rewrite (old vs new), a refactor
9
+ claimed equivalent, or "does this still match the old CLI?". Runs both sides
10
+ live, so it needs no golden capture — just the two versions. NOT a correctness
11
+ oracle: it proves the two AGREE, not that either is right, so pair it with a
12
+ few asserted cases.
13
+ cli: scripts/cli.mjs
14
+ requires: [node>=20]
15
+ ---
16
+
17
+ # Canary: Shadow (differential parity testing)
18
+
19
+ > Run one invocation through two implementations, normalize the noise, diff the
20
+ > rest. Every un-accepted divergence is a behavior change to explain or fix.
21
+
22
+ ## When to Use
23
+
24
+ - **Ports / rewrites** where output should be identical: Python→TS, a BoP skill
25
+ re-port, typer→commander, one engine replacing another. (This is exactly how
26
+ the v6 cutover was validated: run `canary <cmd>` through the TS engine AND the
27
+ Python reference, diff. It surfaced a divergence the unit tests + golden suite
28
+ missed — Python's `rich` console renders the ticket-marker hint
29
+ `# canary:ticket:` as `# canary🎫` via emoji-shortcode substitution, mangling
30
+ a hint the parser can't match; the TS port printed the literal, correct
31
+ marker. A console-rendering artifact only a live side-by-side run exposes.)
32
+ - **Skill changes** — old skill vs new skill on the same inputs: "prove the
33
+ rewrite is equivalent."
34
+ - **Refactors** asserted to be behavior-preserving.
35
+ - **Framework / dependency swaps** where the observable surface should not move.
36
+ - NOT for greenfield behavior (nothing to compare against) — use asserted tests.
37
+ - NOT as a sole correctness check — see the caveat at the bottom.
38
+
39
+ ## The idea (differential / shadow testing)
40
+
41
+ For each case, run the SAME arguments through a `baseline` command and a
42
+ `candidate` command, capture `{exitCode, stdout, stderr}`, **normalize** both,
43
+ and diff. Identical (post-normalization) ⇒ `ok`. Different ⇒ `DIVERGE` — a real
44
+ behavior change, unless it's a _documented, intentional_ difference recorded in
45
+ the accepted-divergence allowlist.
46
+
47
+ The value is not the runner (two `spawn`s and a diff) — it's the two disciplines
48
+ below.
49
+
50
+ ### 1. The normalization ruleset (get this right, or drown)
51
+
52
+ Output has meaningful content and irrelevant noise. Normalize the noise; keep
53
+ everything else. Default masks (see `scripts/cli.mjs`): ANSI SGR codes, ISO
54
+ timestamps, temp paths (`/tmp`, `/var/folders`, `/private`), version banners
55
+ (`vX.Y.Z[-rc.N]`), commit SHAs, run-ids.
56
+
57
+ - **Under-normalize** and every run is a wall of false DIVERGE (a timestamp, a
58
+ temp dir) — you stop reading them.
59
+ - **Over-normalize** and you mask a real bug (don't blanket-strip numbers if the
60
+ command emits counts/scores that matter).
61
+ - Make masking **visible**: the runner reports what each mask touched, so a
62
+ reviewer can see a real difference wasn't hidden. Tune masks per project in
63
+ the cases file's `normalize` list, don't hardcode.
64
+
65
+ ### 2. The accepted-divergence allowlist
66
+
67
+ Some differences are intentional and permanent (e.g. Python `rich` soft-wraps
68
+ prose at 80 cols; a TS port emits it unwrapped — same content, different line
69
+ breaks). Record each in the cases file with a **reason**, keyed by case label.
70
+ This is the differential-test analog of `// harness-ignore`: an allowlisted
71
+ DIVERGE is reported as `accept` (with its reason), never as a failure, and a NEW
72
+ divergence still fails loudly. Review the allowlist like code — an entry with no
73
+ reason, or one that quietly grows, is a smell.
74
+
75
+ ## Process
76
+
77
+ ### Phase 1: SCOPE — define baseline, candidate, and cases
78
+
79
+ 1. Identify the two implementations and how to invoke each (argv-preserving):
80
+ `baseline` = the trusted side (old skill, Python reference, prior version),
81
+ `candidate` = the new side.
82
+ 2. Enumerate cases that cover the observable surface: the happy paths, the JSON
83
+ contracts, the error/exit-code paths (a good differential run checks exit
84
+ codes AND stdout), and any command that needs a fixture (a sample file, a
85
+ temp project). Prefer read-only/deterministic cases first; give
86
+ state-changing ones (`init`, `migrate`) an isolated `cwd`.
87
+
88
+ ### Phase 2: RUN — cycle until clean
89
+
90
+ 1. Write a cases file (see `scripts/cases.example.json`) and run:
91
+ `node scripts/cli.mjs --cases <file>` (or via the skill `cli`). It prints
92
+ `ok` / `DIVERGE` / `accept` per case and a summary; exit is non-zero if any
93
+ un-accepted divergence remains.
94
+ 2. For each DIVERGE: read the diff. Decide — **bug** (fix the candidate),
95
+ **noise** (add/adjust a normalize rule), or **intentional** (add an allowlist
96
+ entry WITH a reason). Re-run. Iterate in cycles until the only remaining
97
+ divergences are `accept`ed.
98
+ 3. Do multiple cycles broadening coverage (add sub-commands, more fixtures,
99
+ adversarial inputs) — the first cycle finds the obvious breaks; the tail
100
+ finds the subtle ones.
101
+
102
+ ### Phase 3: LOCK — keep it honest
103
+
104
+ 1. Commit the cases file + allowlist alongside the change so the parity contract
105
+ is reviewable and re-runnable in CI.
106
+ 2. Add a handful of **asserted** cases (expected exact output/exit for a few
107
+ invocations) so the suite also catches "both sides are wrong the same way."
108
+
109
+ ## Invocation
110
+
111
+ ```bash
112
+ # Usage, flags, and the cases-file contract (exits 0):
113
+ canary skills run canary-shadow -- --help
114
+ ```
115
+
116
+ A run needs a cases file, so the shape below is illustrative — copy
117
+ `scripts/cases.example.json`, point it at your two implementations, and
118
+ substitute the path:
119
+
120
+ <!-- canary:illustrative -->
121
+
122
+ ```bash
123
+ canary skills run canary-shadow -- --cases <cases.json>
124
+ ```
125
+
126
+ ## Caveat — agreement is not correctness
127
+
128
+ Differential testing proves the two implementations AGREE. If the baseline has a
129
+ bug the candidate faithfully reproduces, both pass. So it is a powerful
130
+ regression net, not a correctness oracle — always pair it with a few asserted
131
+ cases and human review of the allowlist.
@@ -0,0 +1,32 @@
1
+ {
2
+ "//": "Example shadow-parity config: the released CLI (baseline) vs the local build (candidate) — does this branch still agree with what users have? Copy, point baseline/candidate at your two implementations, and edit `cases`. Run: node cli.mjs --cases cases.example.json",
3
+ "baseline": ["npx", "--yes", "canary-test-cli@6.6.0"],
4
+ "candidate": ["node", "ts/bin/canary.js"],
5
+ "env": { "NO_COLOR": "1" },
6
+ "normalize": ["ansi", "version", "timestamp", "tmppath", "sha"],
7
+ "accept": {
8
+ "skills list": "The released build soft-wraps long skill descriptions at 80 cols; the local build emits them unwrapped. Same content + names, different line breaks — a documented, intentional presentation deviation, not a contract."
9
+ },
10
+ "cases": [
11
+ { "label": "frameworks --json", "argv": ["frameworks", "--json"] },
12
+ { "label": "frameworks", "argv": ["frameworks"] },
13
+ {
14
+ "label": "recommend e2e --json",
15
+ "argv": ["recommend", "test a login form", "--json"]
16
+ },
17
+ { "label": "recommend prose", "argv": ["recommend", "test a login form"] },
18
+ { "label": "skills list", "argv": ["skills", "list"] },
19
+ {
20
+ "label": "bad: unknown option (exit 2)",
21
+ "argv": ["frameworks", "--nope"]
22
+ },
23
+ { "label": "bad: missing arg (exit 2)", "argv": ["recommend"] },
24
+ { "label": "bad: unknown command (exit 2)", "argv": ["frobnicate"] },
25
+ { "label": "analyze flaky --json", "argv": ["analyze", "flaky", "--json"] },
26
+ {
27
+ "label": "init playwright (isolated cwd)",
28
+ "argv": ["init", "playwright"],
29
+ "isolate": true
30
+ }
31
+ ]
32
+ }
@@ -0,0 +1,195 @@
1
+ #!/usr/bin/env node
2
+ // canary-shadow — differential ("shadow") parity runner.
3
+ //
4
+ // Runs each case's argv through a `baseline` and a `candidate` command,
5
+ // normalizes away irrelevant noise, and diffs exit code + stdout. Prints
6
+ // ok / accept / DIVERGE per case; exits non-zero if any un-accepted divergence
7
+ // remains. See ../SKILL.md for the methodology and cases.example.json for the
8
+ // config shape.
9
+ //
10
+ // Usage: node cli.mjs --cases <cases.json> [--verbose]
11
+ import { spawnSync } from 'node:child_process';
12
+ import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
13
+ import { tmpdir } from 'node:os';
14
+ import { join } from 'node:path';
15
+
16
+ import {
17
+ createParser,
18
+ formatUsageError,
19
+ EXIT_USAGE,
20
+ } from '../../../lib/parse-args.mjs';
21
+
22
+ const USAGE =
23
+ 'usage: canary-shadow [-h] --cases PATH [--verbose]\n' +
24
+ '\n' +
25
+ 'Run each case through a baseline and a candidate command, normalize away\n' +
26
+ 'irrelevant noise, and diff exit code + stdout.\n' +
27
+ '\n' +
28
+ 'options:\n' +
29
+ ' -h, --help show this help message and exit\n' +
30
+ ' --cases PATH cases file describing baseline, candidate, and cases\n' +
31
+ ' --verbose print a line per matching case, not just divergences';
32
+
33
+ /**
34
+ * canary-shadow shipped BEFORE the family had a shared contract, and it showed:
35
+ * no `--help`, no unknown-flag rejection, and a usage line naming `cli.mjs`
36
+ * rather than the skill. Nobody noticed, because there was nothing to violate
37
+ * (#479).
38
+ */
39
+ export const CLI_SPEC = {
40
+ prog: 'canary-shadow',
41
+ booleans: { '--verbose': 'verbose' },
42
+ values: { '--cases': { key: 'cases' } },
43
+ required: ['--cases'],
44
+ };
45
+
46
+ const parseArgs = createParser(CLI_SPEC);
47
+
48
+ // --- built-in normalization masks (name -> {re, replace}). Each records
49
+ // whether it fired so the reviewer can see what was hidden. Extend/override per
50
+ // project via the cases file's `normalize` list. ---
51
+ const MASKS = {
52
+ ansi: { re: /\x1b\[[0-9;]*m/g, replace: '' },
53
+ version: { re: /v\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?/g, replace: 'vX' },
54
+ timestamp: {
55
+ re: /\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g,
56
+ replace: 'TS',
57
+ },
58
+ tmppath: { re: /(?:\/private)?\/(?:var|tmp)\/[^\s"']+/g, replace: 'TMP' },
59
+ sha: { re: /\b[0-9a-f]{7,40}\b/g, replace: 'SHA' },
60
+ runid: { re: /\b(?:run|job)[-_]?\d{3,}\b/gi, replace: 'RUNID' },
61
+ };
62
+ const DEFAULT_MASKS = ['ansi', 'version', 'timestamp', 'tmppath', 'sha'];
63
+
64
+ function buildMasks(spec) {
65
+ // spec: array of built-in names and/or {name, pattern, flags?, replace}
66
+ const list = [];
67
+ for (const m of spec ?? DEFAULT_MASKS) {
68
+ if (typeof m === 'string') {
69
+ if (!MASKS[m]) throw new Error(`unknown built-in mask: ${m}`);
70
+ list.push({ name: m, ...MASKS[m] });
71
+ } else {
72
+ list.push({
73
+ name: m.name ?? m.pattern,
74
+ re: new RegExp(m.pattern, m.flags ?? 'g'),
75
+ replace: m.replace ?? '',
76
+ });
77
+ }
78
+ }
79
+ return list;
80
+ }
81
+
82
+ function normalize(text, masks, fired) {
83
+ let out = text;
84
+ for (const m of masks) {
85
+ if (m.re.test(out)) fired.add(m.name);
86
+ m.re.lastIndex = 0;
87
+ out = out.replace(m.re, m.replace);
88
+ }
89
+ return out.replace(/[ \t]+$/gm, '').trim();
90
+ }
91
+
92
+ function run(cmd, argv, cwd, env) {
93
+ const r = spawnSync(cmd[0], [...cmd.slice(1), ...argv], {
94
+ cwd,
95
+ env: { ...process.env, NO_COLOR: '1', ...env },
96
+ encoding: 'utf-8',
97
+ maxBuffer: Infinity,
98
+ timeout: 120_000,
99
+ });
100
+ return {
101
+ code: r.status ?? 1,
102
+ stdout: r.stdout ?? '',
103
+ stderr: r.stderr ?? '',
104
+ };
105
+ }
106
+
107
+ function unifiedDiff(a, b, max = 30) {
108
+ const A = a.split('\n');
109
+ const B = b.split('\n');
110
+ // minimal LCS-free line diff: good enough for review, keeps the runner small.
111
+ const out = [];
112
+ const bSet = new Set(B);
113
+ const aSet = new Set(A);
114
+ for (const line of A) if (!bSet.has(line)) out.push('- ' + line);
115
+ for (const line of B) if (!aSet.has(line)) out.push('+ ' + line);
116
+ return out.slice(0, max);
117
+ }
118
+
119
+ export function main(argv = []) {
120
+ const { opts, help, error } = parseArgs(argv);
121
+
122
+ if (help) {
123
+ console.log(USAGE);
124
+ return 0;
125
+ }
126
+ if (error) {
127
+ console.error(formatUsageError(CLI_SPEC.prog, error));
128
+ return EXIT_USAGE;
129
+ }
130
+
131
+ const { cases: casesPath, verbose } = opts;
132
+ const cfg = JSON.parse(readFileSync(casesPath, 'utf-8'));
133
+ const masks = buildMasks(cfg.normalize);
134
+ // Null-prototype: `accept` comes straight from JSON, so on a plain object a
135
+ // case labelled `toString` resolved to Object.prototype.toString and was
136
+ // reported `accept` instead of DIVERGE -- a parity tool silently suppressing
137
+ // a parity failure, which is the worst place in the family for this bug to
138
+ // have landed (#479).
139
+ const accept = Object.assign(Object.create(null), cfg.accept ?? {});
140
+ const fired = new Set();
141
+
142
+ let ok = 0;
143
+ let accepted = 0;
144
+ let diverged = 0;
145
+ for (const c of cfg.cases) {
146
+ let cwd = cfg.cwd;
147
+ let tmp;
148
+ if (c.isolate) {
149
+ tmp = mkdtempSync(join(tmpdir(), 'shadow-'));
150
+ cwd = tmp;
151
+ }
152
+ try {
153
+ const base = run(cfg.baseline, c.argv, cwd, cfg.env);
154
+ const cand = run(cfg.candidate, c.argv, cwd, cfg.env);
155
+ const bOut = normalize(base.stdout, masks, fired);
156
+ const cOut = normalize(cand.stdout, masks, fired);
157
+ const match = base.code === cand.code && bOut === cOut;
158
+ if (match) {
159
+ ok++;
160
+ if (verbose)
161
+ process.stdout.write(`ok ${c.label} (exit ${base.code})\n`);
162
+ } else if (accept[c.label]) {
163
+ accepted++;
164
+ process.stdout.write(`accept ${c.label} — ${accept[c.label]}\n`);
165
+ } else {
166
+ diverged++;
167
+ process.stdout.write(
168
+ `DIVERGE ${c.label} (exit base=${base.code} cand=${cand.code})\n`,
169
+ );
170
+ if (base.code !== cand.code)
171
+ process.stdout.write(
172
+ ` exit differs: base=${base.code} cand=${cand.code}\n`,
173
+ );
174
+ for (const line of unifiedDiff(bOut, cOut))
175
+ process.stdout.write(' ' + line + '\n');
176
+ }
177
+ } finally {
178
+ if (tmp) rmSync(tmp, { recursive: true, force: true });
179
+ }
180
+ }
181
+ process.stdout.write(
182
+ `\n=== ${ok} ok, ${accepted} accepted, ${diverged} DIVERGE ` +
183
+ `(${cfg.cases.length} cases) — masks fired: ${[...fired].join(', ') || 'none'} ===\n`,
184
+ );
185
+ return diverged > 0 ? 1 : 0;
186
+ }
187
+
188
+ // Direct execution (the skill runner execs this file via its shebang).
189
+ //
190
+ // `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
191
+ // the pipe buffer, and `process.exit` tears the process down mid-write, leaving
192
+ // truncated JSON that still exits 0 (#791).
193
+ if (import.meta.url === `file://${process.argv[1]}`) {
194
+ process.exitCode = main(process.argv.slice(2));
195
+ }