canary-test-cli 7.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/agents/skills/README.md +308 -0
  2. package/agents/skills/canary:generate.md +49 -0
  3. package/agents/skills/canary:init.md +37 -0
  4. package/agents/skills/canary:migrate.md +66 -0
  5. package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
  6. package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
  7. package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
  8. package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
  9. package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
  10. package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
  11. package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
  12. package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
  13. package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
  14. package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
  15. package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
  16. package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
  17. package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
  18. package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
  19. package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
  20. package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
  21. package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
  22. package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
  23. package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
  24. package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
  25. package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
  26. package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
  27. package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
  28. package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
  29. package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
  30. package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
  31. package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
  32. package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
  33. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
  34. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
  35. package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
  36. package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
  37. package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
  38. package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
  39. package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
  40. package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
  41. package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
  42. package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
  43. package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
  44. package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
  45. package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
  46. package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
  47. package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
  48. package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
  49. package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
  50. package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
  51. package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
  52. package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
  53. package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
  54. package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
  55. package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
  56. package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
  57. package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
  58. package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
  59. package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
  60. package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
  61. package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
  62. package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
  63. package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
  64. package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
  65. package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
  66. package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
  67. package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
  68. package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
  69. package/agents/skills/lib/parse-args.mjs +214 -0
  70. package/dist/engine/analysis/cli.js +116 -54
  71. package/dist/engine/analysis/engine.js +34 -16
  72. package/dist/engine/analysis/reports.js +5 -4
  73. package/dist/engine/cli-commands.js +252 -106
  74. package/dist/engine/cli-common.js +15 -24
  75. package/dist/engine/cli.core.js +37 -11
  76. package/dist/engine/cli.js +2 -2
  77. package/dist/engine/company-knowledge-cli.js +2 -2
  78. package/dist/engine/core/adoption.js +408 -0
  79. package/dist/engine/core/framework-probes.js +7 -7
  80. package/dist/engine/core/fs-glob.js +2 -2
  81. package/dist/engine/core/gate-result.js +17 -0
  82. package/dist/engine/core/migrator.js +9 -17
  83. package/dist/engine/core/pattern-matcher.js +23 -5
  84. package/dist/engine/core/persona.js +421 -0
  85. package/dist/engine/core/promotion-verdict.js +261 -0
  86. package/dist/engine/core/quality-scorer.js +15 -2
  87. package/dist/engine/core/reporter.js +1 -9
  88. package/dist/engine/core/skill-dispatch.js +115 -0
  89. package/dist/engine/core/skill-examples.js +392 -0
  90. package/dist/engine/core/skill-registry.js +59 -4
  91. package/dist/engine/core/skill-surfaces.js +307 -0
  92. package/dist/engine/core/static-linter.js +310 -38
  93. package/dist/engine/core/test-files.js +77 -0
  94. package/dist/engine/core/ticket-updater.js +1 -7
  95. package/dist/engine/core/vacuity-scanner.js +726 -0
  96. package/dist/engine/core/workflow-discovery.js +2 -8
  97. package/dist/engine/core/workspace-detect.js +7 -6
  98. package/dist/engine/data/personas/registry.json +36 -0
  99. package/dist/engine/guardian/adjudication.js +5 -5
  100. package/dist/engine/guardian/analysis-emit.js +19 -28
  101. package/dist/engine/guardian/cli.js +150 -51
  102. package/dist/engine/guardian/coverage.js +1 -1
  103. package/dist/engine/guardian/diff-coverage/heuristic-tier.js +1 -1
  104. package/dist/engine/guardian/diff-coverage/orchestrator.js +2 -2
  105. package/dist/engine/guardian/pr-check.js +322 -33
  106. package/dist/engine/guardian/pr-comment.js +4 -3
  107. package/dist/engine/history/cli.js +210 -6
  108. package/dist/engine/history/ndjson-store.js +9 -5
  109. package/dist/engine/history/record.js +34 -5
  110. package/dist/engine/history/run-recorder.js +165 -0
  111. package/dist/engine/history/schema.js +25 -7
  112. package/dist/engine/history/store.js +9 -0
  113. package/dist/engine/mcp-server.js +35 -13
  114. package/dist/engine/skills-cli.js +249 -23
  115. package/dist/engine/util/ensure-ascii.js +37 -0
  116. package/dist/engine/workflow-cli.js +6 -6
  117. package/dist/gate-result.d.ts +11 -0
  118. package/dist/gate-result.js +18 -0
  119. package/dist/reporters/testtracker.d.ts +1 -1
  120. package/dist/reporters/testtracker.js +1 -1
  121. package/dist/uninstall.js +12 -5
  122. package/package.json +3 -2
@@ -0,0 +1,248 @@
1
+ ---
2
+ name: canary-add-framework
3
+ description: >
4
+ Add a new testing framework to Canary's registry end-to-end — aligns the
5
+ classifier↔registry contract, authors the
6
+ `ts/src/data/frameworks/registry.json` entry, validates the execution command,
7
+ and ensures every classifier `test_type` still resolves to a non-null
8
+ framework. Use for "add support for a new framework", "add k6", "support
9
+ cypress", "we need Locust", or when a classifier `test_type` exists with no
10
+ framework backing it (registry gap). Not for choosing between existing
11
+ frameworks at runtime (that's the recommender's job) or adding a one-off
12
+ framework name with no real CLI.
13
+ ---
14
+
15
+ # Canary: Add Framework
16
+
17
+ > Add a new testing framework to Canary's registry end-to-end. Confirms the
18
+ > classifier↔registry contract, authors the registry entry, validates the
19
+ > execution command, and ensures every classifier `test_type` still resolves to
20
+ > a non-null framework.
21
+
22
+ ## When to Use
23
+
24
+ - When the user asks to add support for a new framework ("add k6", "support
25
+ cypress", "we need Locust")
26
+ - When a classifier `test_type` exists but has no framework backing it (registry
27
+ gap)
28
+ - When migrating from a deprecated framework — adding the new entry before
29
+ removing the old one
30
+ - NOT for choosing between existing frameworks at runtime — that's the
31
+ recommender's job
32
+ - NOT for adding a one-off LLM-generated framework name that doesn't exist as a
33
+ real tool — frameworks must be real, runnable CLIs
34
+
35
+ ## Process
36
+
37
+ ### Phase 1: SCOPE — Confirm the Framework Belongs in the Registry
38
+
39
+ 1. **Verify the framework is real and installable.** It needs a runnable CLI
40
+ invocation (the `execution_command` template). If you can't write a one-line
41
+ shell command that runs a single test file, it doesn't fit Canary's model.
42
+ 2. **Identify the category.** Match to an existing `category` value (`e2e_ui`,
43
+ `python_unit`, `api`, `performance`, `frontend_unit`, etc.). New categories
44
+ require a coordinated classifier update — see Phase 2.
45
+ 3. **Check for overlap.** Is there already a `preferred` framework in this
46
+ category? If yes, decide intentionally: is the new framework a replacement
47
+ (demote the old to `legacy`), a peer (add as `supported`), or the new
48
+ preference (demote the old, mark new as `preferred`)?
49
+ 4. **Confirm the user wants this maintained.** Each registry entry is a
50
+ maintenance commitment. A framework added casually but never validated rots
51
+ into a routing trap.
52
+
53
+ ### Phase 2: ALIGN — Classifier ↔ Registry Contract
54
+
55
+ 1. **Find the classifier rule that emits this `test_type`.** Open
56
+ `ts/src/core/classifier.ts` and locate the heuristic block that produces the
57
+ target category.
58
+ 2. **If no classifier rule exists for the category:** Add one _before_ the
59
+ registry entry. A framework with no routing path is a dead entry. Add a
60
+ heuristic that maps natural-language signals (keywords, phrases) to the new
61
+ `test_type`.
62
+ 3. **If the category exists but the classifier never emits it confidently:**
63
+ Strengthen the rule's heuristics. Confidence below 0.7 should trigger a
64
+ clarification, not a generation.
65
+ 4. **Run the contract tests:**
66
+
67
+ ```bash
68
+ pytest tests/unit/test_orchestrator.py
69
+ ```
70
+
71
+ Every `test_type` the classifier can emit must resolve to ≥1 framework via
72
+ `get_by_category` — the orchestrator tests assert non-null framework
73
+ resolution per category. This is the gate.
74
+
75
+ ### Phase 3: AUTHOR — Write the Registry Entry
76
+
77
+ 1. **Open `ts/src/data/frameworks/registry.json`.** Add a new object to
78
+ `frameworks[]`. Required fields:
79
+ - `name` (unique slug)
80
+ - `display_name`
81
+ - `category` (must match the classifier emission)
82
+ - `languages`
83
+ - `file_extensions`
84
+ - `execution_command` (with `{file}` placeholder)
85
+ - `status` (`preferred` / `supported` / `legacy`)
86
+ 2. **Add recommended metadata.** `maturity`, `community_size`,
87
+ `recommended_for`, `strengths`, `avoid_when`, `ecosystems`. These show up in
88
+ the recommender's reasoning string — sparse entries produce sparse
89
+ recommendations.
90
+ 3. **Validate the `execution_command`.** It must run a single test file when
91
+ `{file}` is substituted. Test manually:
92
+
93
+ ```bash
94
+ echo '<minimal test>' > /tmp/sample.<ext>
95
+ <execution_command with /tmp/sample.<ext> substituted>
96
+ ```
97
+
98
+ Confirm exit code 0 (or expected non-zero) and no missing-config errors.
99
+
100
+ 4. **Confirm the `{file}` placeholder is the only substitution.** The executor
101
+ tokenizes the template with `shlex.split` before substituting `{file}`, so
102
+ the file path stays a single argv element; paths with spaces must remain
103
+ intact.
104
+
105
+ ### Phase 4: VERIFY — Generate, Execute, Promote-Dry
106
+
107
+ 1. **Generate a test that should route to the new framework.** Run the
108
+ `/canary-write-test` slash command in Claude Code with a prompt that should
109
+ hit the new `test_type`.
110
+
111
+ 2. **Check the trace.** Classification matches the expected category,
112
+ recommendation picks the new framework (or the existing preferred — confirm
113
+ this matches your intent), output file has the right extension.
114
+ 3. **Execute with `--execute`.** Confirm the framework's CLI actually runs the
115
+ generated file. A passing execution validates the `execution_command`
116
+ template under the real Canary invocation path.
117
+ 4. **Don't promote.** This is a registry-validation run, not a real test
118
+ promotion. Leave the artifact in `tests/generated/`.
119
+
120
+ ### Phase 5: DOCUMENT — Update Guides and State
121
+
122
+ 1. **Update `docs/guides/framework-registry.md`** if you added a new category,
123
+ changed the contract surface, or introduced a new optional field schema.
124
+ 2. **Update the LLM-providers guide** if the new framework only works with
125
+ specific provider output styles (rare; flag this as a smell if it's true).
126
+ 3. **Log to `docs/CANARY_STATE.md`.** One line: framework name, category,
127
+ status, date added. This is the project ledger; future maintainers grep here
128
+ first.
129
+ 4. **Open a PR.** Title: `feat(registry): add <framework-name> support`. Body
130
+ should include the validation steps you ran and the generated/executed
131
+ sample.
132
+
133
+ ## Canary Integration
134
+
135
+ - **`ts/src/data/frameworks/registry.json`** — Source of truth for entries.
136
+ - **`ts/src/core/framework-registry.ts`** — Loader and lookup methods. Don't add
137
+ lookup helpers here unless the existing five (`get_all_frameworks`,
138
+ `get_by_category`, `get_preferred_by_category`, `find_by_name`,
139
+ `match_by_language`) genuinely don't fit.
140
+ - **`ts/src/core/classifier.ts`** — Routing rules. Coordinate changes with
141
+ registry entries.
142
+ - **`tests/unit/test_orchestrator.py`** — Contract enforcement. Non-null
143
+ resolution per `test_type` (api → pytest, e2e_ui → playwright, performance →
144
+ k6, etc.) is asserted here. `tests/unit/test_factory.py` separately enforces
145
+ the LLM provider matrix.
146
+
147
+ ## Success Criteria
148
+
149
+ - New entry validates against the contract test (no orphaned `test_type` in
150
+ classifier, no null framework resolution in registry)
151
+ - The `/canary-write-test` slash command with a routing prompt resolves to the
152
+ new framework
153
+ - `--execute` runs the generated file successfully via the new
154
+ `execution_command`
155
+ - `framework-registry.md` documents any new category or schema field
156
+ - `CANARY_STATE.md` ledger entry is present
157
+ - PR includes the validation transcript
158
+
159
+ ## Rationalizations to Reject
160
+
161
+ | Rationalization | Why It Is Wrong |
162
+ | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
163
+ | "I'll add the registry entry now and update the classifier later" | Orphaned registry entries are routing traps. Either both land together or neither does. |
164
+ | "Marking it `preferred` is fine, no need to demote the existing entry" | Multiple `preferred` entries in one category means `get_preferred_by_category` picks by registry order, not by merit. Demote the loser explicitly. |
165
+ | "I tested the command with my path — Canary's substitution will work too" | Test the substituted command with a path that contains spaces. The executor's `shlex.split`-then-substitute order exists specifically to handle this case; a broken template will silently corrupt argv. |
166
+ | "It only needs to work for the happy-path prompt I tried" | If the classifier emits this `test_type` with confidence ≥0.7 for _any_ phrasing, the framework must handle the resulting generated file. Test at least two distinct prompts that route to the new entry. |
167
+ | "Sparse metadata is fine, we can fill it in later" | The recommender's reasoning string is read by humans and agents. Sparse metadata produces unhelpful recommendations — fill it in now while you have the context. |
168
+
169
+ ## Examples
170
+
171
+ ### Example: Adding Cypress as a peer to Playwright
172
+
173
+ **Scope:** Cypress already has community demand and a CI image. Category
174
+ `e2e_ui` already has Playwright as `preferred`. Decision: add Cypress as
175
+ `supported`, keep Playwright as `preferred`.
176
+
177
+ **Classifier:** Already emits `test_type=e2e_ui` on UI-test prompts. No
178
+ classifier change needed.
179
+
180
+ **Entry (abridged):**
181
+
182
+ ```json
183
+ {
184
+ "name": "cypress",
185
+ "display_name": "Cypress",
186
+ "category": "e2e_ui",
187
+ "languages": ["javascript", "typescript"],
188
+ "file_extensions": ["cy.ts", "cy.js"],
189
+ "execution_command": "npx --yes cypress run --spec {file}",
190
+ "status": "supported",
191
+ "strengths": ["Time-travel debugging", "Strong DX for UI tests"],
192
+ "avoid_when": ["Cross-browser parity required (Playwright is stronger)"]
193
+ }
194
+ ```
195
+
196
+ **Validation:** Generated a UI test prompt, confirmed recommender still picked
197
+ Playwright (correct — `preferred` wins). Verified Cypress is accessible by
198
+ routing a prompt with explicit Cypress mention through a future
199
+ explicit-framework override.
200
+
201
+ ### Example: Adding k6 to a new performance category
202
+
203
+ **Scope:** Performance category doesn't exist in the registry yet. Classifier
204
+ emits `test_type=performance` on load/stress prompts, but the registry has no
205
+ matching entry — this is the orphan case.
206
+
207
+ **Phase 2 first:** Confirmed `ts/src/core/classifier.ts` already has the
208
+ performance rule (`if "performance" in p or "load test" in p ...`). No
209
+ classifier change needed.
210
+
211
+ **Phase 3:**
212
+
213
+ ```json
214
+ {
215
+ "name": "k6",
216
+ "display_name": "k6",
217
+ "category": "performance",
218
+ "languages": ["javascript"],
219
+ "file_extensions": ["js"],
220
+ "execution_command": "k6 run {file}",
221
+ "status": "preferred",
222
+ "recommended_for": ["HTTP load testing", "Stress and spike tests"],
223
+ "avoid_when": ["Browser-level performance (use Playwright tracing)"]
224
+ }
225
+ ```
226
+
227
+ **Validation:** Ran the `/canary-write-test` slash command with "Load test
228
+ /v1/search at 200 RPS for 5 minutes". Confirmed classifier emits
229
+ `test_type=performance` at 0.95 confidence, recommender picks k6, k6 CLI runs
230
+ the generated file without error.
231
+
232
+ ## Escalation
233
+
234
+ - **When the framework requires a non-trivial config file (e.g.,
235
+ `playwright.config.ts`):** Document the setup in `avoid_when` or in
236
+ `recommended_for`. If Canary is expected to generate the config too, that's a
237
+ scaffolder change, not just a registry change.
238
+ - **When two frameworks legitimately should both be `preferred` for different
239
+ sub-cases:** Split the category. A single category should have a single
240
+ `preferred`. If the split isn't clean, surface this to the user as a design
241
+ decision rather than fudging the registry.
242
+ - **When the classifier doesn't reliably emit the target `test_type`:** Don't
243
+ add the registry entry yet. Strengthen the classifier first — an unreachable
244
+ entry is dead weight.
245
+ - **When the framework's CLI doesn't accept a single-file argument:** Canary's
246
+ model is one-test-per-file generation. Frameworks that only run by directory
247
+ or by tag don't fit cleanly; raise this with the user before forcing a
248
+ workaround.
@@ -0,0 +1,170 @@
1
+ ---
2
+ name: canary-blackhawk
3
+ description:
4
+ Temporal-dependency linter for test files — statically flags tests that lean
5
+ on wall-clock time, a real delay, or the local timezone, the ones that pass
6
+ all day and fail at midnight, across a DST boundary, or on Feb 29. Suppresses
7
+ itself when the file already installs a frozen clock (fake timers, freezegun,
8
+ time_machine, MockDate). Self-contained, deterministic, advisory by default.
9
+ cli: scripts/cli.mjs
10
+ requires: [node>=20]
11
+ ---
12
+
13
+ # Canary Blackhawk
14
+
15
+ A test that reads the wall clock is a test with a scheduled outage. It passes
16
+ every run you watch, then fails at 23:59:59, on the Sunday the clocks move, or
17
+ on Feb 29. Blackhawk finds those lines before the calendar does.
18
+
19
+ Tier-0 deterministic analysis: no LLM, no network, no secrets, no dependency on
20
+ any other skill.
21
+
22
+ ## Rules
23
+
24
+ | Rule | Severity | Fires on |
25
+ | ------------------------------ | -------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
26
+ | `BH001-wall-clock` | high | `Date.now()`, bare `new Date()`, `moment()`, `datetime.now()` / `.today()` / `.utcnow()`, `date.today()`, `time.time()`, `pd.Timestamp.now()` |
27
+ | `BH002-real-delay` | medium | `time.sleep(n)` and `setTimeout(fn, n)` with a literal `n > 0` |
28
+ | `BH003-local-timezone` | medium | `toLocaleString` / `toLocaleDateString` / `toLocaleTimeString`, `strftime('…%Z…')` or `%z` |
29
+ | `BH004-naive-datetime-compare` | low | a comparison against `datetime(2024, …)` or `strptime(…)` with no `tzinfo` / `timezone.utc` / `ZoneInfo` / `pytz` on the line |
30
+
31
+ A pinned constructor is never flagged: `new Date('2024-01-01T00:00:00Z')`,
32
+ `moment('2024-01-01')`, and `datetime(2024, 1, 1)` on its own are all fine.
33
+
34
+ ## Framework-conditioned suppression (the important part)
35
+
36
+ The single biggest failure mode of a temporal linter is firing on tests that
37
+ **already handle time correctly**. So when a file installs a frozen clock, every
38
+ clock-dependent rule (`BH001`, `BH002`, `BH004`) goes quiet for that file.
39
+ Markers:
40
+
41
+ `vi.useFakeTimers` · `vi.setSystemTime` · `jest.useFakeTimers` ·
42
+ `jest.setSystemTime` · `sinon.useFakeTimers` · `MockDate` · `freeze_time` /
43
+ `freezegun` · `time_machine`
44
+
45
+ Two deliberate choices inside that:
46
+
47
+ - **Suppression is file-wide, not block-scoped.** `vi.useFakeTimers()` inside a
48
+ `beforeEach` governs tests declared above it, and answering "is the clock
49
+ frozen _here_" accurately needs a real parser. Blackhawk errs toward silence.
50
+ - **`BH003` is never suppressed.** Freezing the clock pins _when_ a test runs,
51
+ never _where_. A frozen clock does not stop `toLocaleString()` from returning
52
+ a different string on a developer laptop than on a UTC runner.
53
+
54
+ ## Inline suppression (per-line escape hatch)
55
+
56
+ File-wide frozen-clock suppression is too coarse when one intentional real wait
57
+ sits among otherwise-deterministic tests. An inline pragma silences a single
58
+ finding:
59
+
60
+ ```js
61
+ // blackhawk-ignore BH002 -- real MongoDB claim race; deterministic wait impossible
62
+ await new Promise((r) => setTimeout(r, 25));
63
+ ```
64
+
65
+ - **Reason required** (the `-- reason` tail) — keeps suppressions honest and
66
+ greppable, like `eslint-disable-next-line` / `# noqa`.
67
+ - **Rule-scoped** (`BH002`, or the full `BH002-real-delay`) so it never
68
+ blanket-silences the line; a `BH001` on the same line still fires.
69
+ - **Placement:** the pragma may trail the offending line or sit on the line
70
+ directly above it. Comma-separate ids to suppress several (`BH002,BH004`).
71
+ - **Counted separately:** suppressed findings are reported as `N suppressed`,
72
+ out of the actionable total — so a genuinely clean suite can read zero
73
+ findings while the known-OK waits stay visible.
74
+
75
+ ## Fidelity limits (regex/AST-lite, on purpose)
76
+
77
+ Blackhawk is a line scanner with no TypeScript parser dependency, so it ships
78
+ anywhere `node` does. The cost, stated plainly:
79
+
80
+ - **Line-scoped.** A call split across lines (`setTimeout(\n fn,\n 500\n)`) is
81
+ missed, as is a `setTimeout` whose callback body contains a comma.
82
+ - **Comment-blind, one level deep.** Lines starting with `#`, `//`, `*`, `/*`,
83
+ or a docstring quote are skipped; a multi-line block comment whose inner lines
84
+ do not start with `*` is still scanned.
85
+ - **String-aware, one line deep.** A match that _starts_ inside a string literal
86
+ on its own line is rejected as fixture data (`pyFile('time.sleep(1)')` does
87
+ not fire), while an anchor whose pattern merely reaches into quotes
88
+ (`strftime('..%Z')`) still does; template `${...}` interpolation is code. A
89
+ string spanning lines is only seen on its opening line, so a fixture blob's
90
+ continuation lines are still scanned like code.
91
+ - **No type awareness.** `.toLocaleString()` on a `Number` reads the same as on
92
+ a `Date`.
93
+ - **Suppression is a substring match.** A mention of `freezegun` in a comment
94
+ silences the file. That direction is intentional: a missed finding costs less
95
+ than a false one.
96
+
97
+ ## Which files get scanned
98
+
99
+ A directory walk only visits **test** files — `*.test.*`, `*.spec.*`,
100
+ `test_*.py`, `*_test.py`, or any supported source under `tests/`, `test/`,
101
+ `__tests__/`, `e2e/`, `spec/`. A file named explicitly on the command line is
102
+ always scanned, test-looking or not. Supported suffixes: `.py`, `.js`, `.jsx`,
103
+ `.ts`, `.tsx`, `.mjs`, `.cjs`.
104
+
105
+ ## Invocation
106
+
107
+ ```bash
108
+ # Scan the repo's test files (advisory — always exits 0):
109
+ canary skills run canary-blackhawk
110
+
111
+ # Scan a specific suite:
112
+ canary skills run canary-blackhawk -- tests/e2e
113
+
114
+ # Machine-readable findings:
115
+ canary skills run canary-blackhawk -- tests --json
116
+
117
+ # Fail the step on any finding:
118
+ canary skills run canary-blackhawk -- tests --strict
119
+
120
+ # Usage, options, and the full rule list (exits 0):
121
+ canary skills run canary-blackhawk -- --help
122
+ ```
123
+
124
+ An unknown flag is rejected with `unrecognized arguments: <flag>` and exit 2;
125
+ use `--` to end option parsing when a path itself starts with a dash.
126
+
127
+ `--json` shape:
128
+
129
+ ```json
130
+ {
131
+ "schema_version": 1,
132
+ "findings": [
133
+ {
134
+ "file": "tests/clock.spec.ts",
135
+ "line": 12,
136
+ "rule_id": "BH001-wall-clock",
137
+ "severity": "high",
138
+ "snippet": "const t = Date.now();",
139
+ "why": "reads the wall clock, so the assertion depends on when the suite runs..."
140
+ }
141
+ ],
142
+ "summary": {
143
+ "files_scanned": 8,
144
+ "findings": 1,
145
+ "by_severity": { "high": 1 },
146
+ "suppressed": 0
147
+ }
148
+ }
149
+ ```
150
+
151
+ ## CI wiring (GitHub Actions)
152
+
153
+ Advisory first, then promote to blocking once the backlog is drained and the
154
+ signal is trusted — the same path every canary gate takes.
155
+
156
+ ```yaml
157
+ - name: Temporal-dependency lint (advisory)
158
+ run: canary skills run canary-blackhawk -- tests
159
+ # Once clean, add --strict to make new offenders fail the PR:
160
+ # run: canary skills run canary-blackhawk -- tests --strict
161
+ ```
162
+
163
+ ## Fixing what it finds
164
+
165
+ | Finding | Fix |
166
+ | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
167
+ | `BH001` | Freeze the clock (`vi.useFakeTimers()` + `vi.setSystemTime(...)`, `@freeze_time("2024-01-01")`) or inject a clock the test controls. |
168
+ | `BH002` | Advance a fake timer (`vi.advanceTimersByTime(500)`) or await the real condition instead of a duration. |
169
+ | `BH003` | Assert on a UTC representation (`toISOString()`, `strftime('%Y-%m-%dT%H:%M:%SZ')` on a UTC datetime), or pin the locale and timezone explicitly. |
170
+ | `BH004` | Attach a timezone: `datetime(2024, 1, 1, tzinfo=timezone.utc)`. |
@@ -0,0 +1,188 @@
1
+ #!/usr/bin/env node
2
+ // canary-blackhawk -- temporal-dependency linter for test files.
3
+ //
4
+ // Statically flags tests that depend on wall-clock time, a real delay, or the
5
+ // local timezone: the ones that pass all day and fail at midnight, across a DST
6
+ // boundary, or on Feb 29.
7
+ //
8
+ // <paths> files or directories to scan (default: the current directory).
9
+ // --json emit machine-readable findings instead of human text.
10
+ // --strict exit 1 when there are findings (default is advisory: exit 0).
11
+ //
12
+ // Tier-0 deterministic analysis -- no LLM, no network, no secrets, no
13
+ // dependency on any other skill.
14
+ //
15
+ // Invoked via `canary skills run canary-blackhawk -- [paths] [--json] [--strict]`.
16
+
17
+ import fs from 'node:fs';
18
+ import { scanPaths, toJson } from './scanner.mjs';
19
+ import { RULES } from './rules.mjs';
20
+ import {
21
+ createParser,
22
+ formatUsageError,
23
+ EXIT_USAGE,
24
+ } from '../../../lib/parse-args.mjs';
25
+
26
+ export const SCHEMA_VERSION = 1;
27
+
28
+ const PREFIX = 'canary-blackhawk:';
29
+
30
+ // --- no-silent-abstention (#508 D2, skill-CLI convention half) ---------------
31
+ //
32
+ // Skill CLIs are deliberately self-contained -- no engine import, no shared
33
+ // module -- so they cannot call `gateOutcome`. They honour the doctrine by
34
+ // CONVENTION instead, emitting the same greppable line the engine helper does.
35
+ // The skill-layer conformance registry (agents/skills/test/gate-conformance.
36
+ // test.ts) is what holds them to it: a row whose fixture collapses the
37
+ // denominator and asserts the loud outcome.
38
+ //
39
+ // U+26A0 / U+2014 are written as escapes so this source stays ASCII, matching
40
+ // ts/src/core/gate-result.ts.
41
+ const ABSTAINED_LINE =
42
+ '\u{26A0} Abstained \u{2014} verified zero items; this is not a pass.';
43
+
44
+ // The rules block is GENERATED from RULES, never hand-typed: a new rule shows
45
+ // up in --help the moment it is registered, so the help text cannot drift
46
+ // behind the linter as rules are added.
47
+ const USAGE =
48
+ 'usage: canary-blackhawk [-h] [--json] [--strict] [--] [path ...]\n' +
49
+ '\n' +
50
+ 'Temporal-dependency linter for test files: flags tests that depend on the\n' +
51
+ 'wall clock, a real delay, or the local timezone.\n' +
52
+ '\n' +
53
+ 'positional arguments:\n' +
54
+ ' path files or directories to scan (default: the current directory)\n' +
55
+ '\n' +
56
+ 'options:\n' +
57
+ ' -h, --help show this help message and exit\n' +
58
+ ' --json emit machine-readable findings instead of human text\n' +
59
+ ' --strict exit 1 when there are findings (default is advisory: exit 0)\n' +
60
+ '\n' +
61
+ 'rules:\n' +
62
+ RULES.map((r) => ` ${r.ruleId} (${r.severity})`).join('\n');
63
+
64
+ function summary(result) {
65
+ const bySeverity = {};
66
+ for (const f of result.findings) {
67
+ bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
68
+ }
69
+ return {
70
+ files_scanned: result.filesScanned,
71
+ findings: result.findings.length,
72
+ by_severity: bySeverity,
73
+ suppressed: result.suppressed ?? 0,
74
+ };
75
+ }
76
+
77
+ // A trailing "N suppressed" note keeps inline-ignored lines visible but out of
78
+ // the actionable total - the pattern the PR-guardian sticky comment uses.
79
+ function suppressedNote(result) {
80
+ const n = result.suppressed ?? 0;
81
+ return n ? `\n${n} suppressed (inline blackhawk-ignore).` : '';
82
+ }
83
+
84
+ function renderText(result) {
85
+ const count = result.findings.length;
86
+ const files = result.filesScanned;
87
+ const fp = files === 1 ? '' : 's';
88
+ // #508: zero findings over zero scanned files is an ABSENT result, not a
89
+ // clean one. Findings outrank abstention (a finding proves a file was read),
90
+ // so this is checked only on the no-findings path.
91
+ if (!count && !files) {
92
+ return (
93
+ `${ABSTAINED_LINE} No file matched the given paths, so there is ` +
94
+ 'nothing to report. Point at a directory that holds test files, or ' +
95
+ 'pass a file directly.'
96
+ );
97
+ }
98
+ if (!count) {
99
+ return (
100
+ `No temporal-dependency findings (${files} file${fp} scanned).` +
101
+ suppressedNote(result)
102
+ );
103
+ }
104
+ const sp = count === 1 ? '' : 's';
105
+ const lines = [
106
+ `${count} temporal-dependency finding${sp} in ${files} file${fp}:`,
107
+ '',
108
+ ];
109
+ for (const f of result.findings) {
110
+ lines.push(` ${f.file}:${f.line} [${f.severity}] ${f.ruleId}`);
111
+ lines.push(` ${f.snippet}`);
112
+ lines.push(` why: ${f.why}`);
113
+ }
114
+ lines.push('');
115
+ lines.push(
116
+ 'Advisory by default. Re-run with --strict to fail the step on findings.',
117
+ );
118
+ return lines.join('\n') + suppressedNote(result);
119
+ }
120
+
121
+ /**
122
+ * blackhawk takes paths, so it gets the `--` end-of-options terminator (a file
123
+ * literally named `--json` stays reachable) and treats a lone `-` as a
124
+ * positional, as argparse does. The four shared invariants live in the shared
125
+ * parser (#479).
126
+ */
127
+ export const CLI_SPEC = {
128
+ prog: 'canary-blackhawk',
129
+ booleans: { '--json': 'json', '--strict': 'strict' },
130
+ positionals: { key: 'paths', defaults: ['.'] },
131
+ };
132
+
133
+ const parseArgs = createParser(CLI_SPEC);
134
+
135
+ export function main(argv = []) {
136
+ const { positionals: paths, opts, help, error } = parseArgs(argv);
137
+
138
+ // Usage and parse errors resolve before any filesystem work, so `--help`
139
+ // never reports a missing path and a typo never half-runs a scan.
140
+ if (help) {
141
+ console.log(USAGE);
142
+ return 0;
143
+ }
144
+ if (error) {
145
+ console.error(formatUsageError(CLI_SPEC.prog, error));
146
+ return EXIT_USAGE;
147
+ }
148
+
149
+ for (const entry of paths) {
150
+ if (!fs.existsSync(entry)) {
151
+ console.error(`${PREFIX} path not found: ${entry}`);
152
+ return 1;
153
+ }
154
+ }
155
+
156
+ const result = scanPaths(paths);
157
+
158
+ if (opts.json) {
159
+ console.log(
160
+ JSON.stringify(
161
+ {
162
+ schema_version: SCHEMA_VERSION,
163
+ findings: result.findings.map(toJson),
164
+ summary: summary(result),
165
+ },
166
+ null,
167
+ 2,
168
+ ),
169
+ );
170
+ } else {
171
+ console.log(renderText(result));
172
+ }
173
+
174
+ // Advisory by default (D3). Under --strict the CLI carries an exit-code
175
+ // contract, so a collapsed denominator inherits EXIT_ABSTAINED (3) -- distinct
176
+ // from 1 ("found something real"), so CI can tell them apart.
177
+ if (opts.strict && !result.filesScanned) return 3;
178
+ return opts.strict && result.findings.length ? 1 : 0;
179
+ }
180
+
181
+ // Direct execution (the skill runner execs this file via its shebang).
182
+ //
183
+ // `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
184
+ // the pipe buffer, and `process.exit` tears the process down mid-write, leaving
185
+ // truncated JSON that still exits 0 (#791).
186
+ if (import.meta.url === `file://${process.argv[1]}`) {
187
+ process.exitCode = main(process.argv.slice(2));
188
+ }