canary-test-cli 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +308 -0
- package/agents/skills/canary:generate.md +49 -0
- package/agents/skills/canary:init.md +37 -0
- package/agents/skills/canary:migrate.md +66 -0
- package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
- package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
- package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
- package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
- package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
- package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
- package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
- package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
- package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
- package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
- package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
- package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
- package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
- package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
- package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
- package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
- package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
- package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
- package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
- package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
- package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
- package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
- package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
- package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
- package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
- package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
- package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
- package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
- package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
- package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
- package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
- package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
- package/agents/skills/lib/parse-args.mjs +214 -0
- package/dist/engine/cli-commands.js +3 -65
- package/dist/engine/core/quality-scorer.js +15 -2
- package/dist/engine/core/skill-dispatch.js +115 -0
- package/dist/engine/core/skill-examples.js +103 -3
- package/dist/engine/core/skill-registry.js +59 -4
- package/dist/engine/core/test-files.js +77 -0
- package/dist/engine/core/vacuity-scanner.js +184 -14
- package/dist/engine/guardian/analysis-emit.js +7 -2
- package/dist/engine/guardian/cli.js +121 -9
- package/dist/engine/guardian/pr-check.js +317 -18
- package/dist/engine/skills-cli.js +123 -19
- package/dist/reporters/testtracker.d.ts +1 -1
- package/dist/reporters/testtracker.js +1 -1
- package/package.json +3 -2
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: canary-add-framework
|
|
3
|
+
description: >
|
|
4
|
+
Add a new testing framework to Canary's registry end-to-end — aligns the
|
|
5
|
+
classifier↔registry contract, authors the
|
|
6
|
+
`ts/src/data/frameworks/registry.json` entry, validates the execution command,
|
|
7
|
+
and ensures every classifier `test_type` still resolves to a non-null
|
|
8
|
+
framework. Use for "add support for a new framework", "add k6", "support
|
|
9
|
+
cypress", "we need Locust", or when a classifier `test_type` exists with no
|
|
10
|
+
framework backing it (registry gap). Not for choosing between existing
|
|
11
|
+
frameworks at runtime (that's the recommender's job) or adding a one-off
|
|
12
|
+
framework name with no real CLI.
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
# Canary: Add Framework
|
|
16
|
+
|
|
17
|
+
> Add a new testing framework to Canary's registry end-to-end. Confirms the
|
|
18
|
+
> classifier↔registry contract, authors the registry entry, validates the
|
|
19
|
+
> execution command, and ensures every classifier `test_type` still resolves to
|
|
20
|
+
> a non-null framework.
|
|
21
|
+
|
|
22
|
+
## When to Use
|
|
23
|
+
|
|
24
|
+
- When the user asks to add support for a new framework ("add k6", "support
|
|
25
|
+
cypress", "we need Locust")
|
|
26
|
+
- When a classifier `test_type` exists but has no framework backing it (registry
|
|
27
|
+
gap)
|
|
28
|
+
- When migrating from a deprecated framework — adding the new entry before
|
|
29
|
+
removing the old one
|
|
30
|
+
- NOT for choosing between existing frameworks at runtime — that's the
|
|
31
|
+
recommender's job
|
|
32
|
+
- NOT for adding a one-off LLM-generated framework name that doesn't exist as a
|
|
33
|
+
real tool — frameworks must be real, runnable CLIs
|
|
34
|
+
|
|
35
|
+
## Process
|
|
36
|
+
|
|
37
|
+
### Phase 1: SCOPE — Confirm the Framework Belongs in the Registry
|
|
38
|
+
|
|
39
|
+
1. **Verify the framework is real and installable.** It needs a runnable CLI
|
|
40
|
+
invocation (the `execution_command` template). If you can't write a one-line
|
|
41
|
+
shell command that runs a single test file, it doesn't fit Canary's model.
|
|
42
|
+
2. **Identify the category.** Match to an existing `category` value (`e2e_ui`,
|
|
43
|
+
`python_unit`, `api`, `performance`, `frontend_unit`, etc.). New categories
|
|
44
|
+
require a coordinated classifier update — see Phase 2.
|
|
45
|
+
3. **Check for overlap.** Is there already a `preferred` framework in this
|
|
46
|
+
category? If yes, decide intentionally: is the new framework a replacement
|
|
47
|
+
(demote the old to `legacy`), a peer (add as `supported`), or the new
|
|
48
|
+
preference (demote the old, mark new as `preferred`)?
|
|
49
|
+
4. **Confirm the user wants this maintained.** Each registry entry is a
|
|
50
|
+
maintenance commitment. A framework added casually but never validated rots
|
|
51
|
+
into a routing trap.
|
|
52
|
+
|
|
53
|
+
### Phase 2: ALIGN — Classifier ↔ Registry Contract
|
|
54
|
+
|
|
55
|
+
1. **Find the classifier rule that emits this `test_type`.** Open
|
|
56
|
+
`ts/src/core/classifier.ts` and locate the heuristic block that produces the
|
|
57
|
+
target category.
|
|
58
|
+
2. **If no classifier rule exists for the category:** Add one _before_ the
|
|
59
|
+
registry entry. A framework with no routing path is a dead entry. Add a
|
|
60
|
+
heuristic that maps natural-language signals (keywords, phrases) to the new
|
|
61
|
+
`test_type`.
|
|
62
|
+
3. **If the category exists but the classifier never emits it confidently:**
|
|
63
|
+
Strengthen the rule's heuristics. Confidence below 0.7 should trigger a
|
|
64
|
+
clarification, not a generation.
|
|
65
|
+
4. **Run the contract tests:**
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pytest tests/unit/test_orchestrator.py
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Every `test_type` the classifier can emit must resolve to ≥1 framework via
|
|
72
|
+
`get_by_category` — the orchestrator tests assert non-null framework
|
|
73
|
+
resolution per category. This is the gate.
|
|
74
|
+
|
|
75
|
+
### Phase 3: AUTHOR — Write the Registry Entry
|
|
76
|
+
|
|
77
|
+
1. **Open `ts/src/data/frameworks/registry.json`.** Add a new object to
|
|
78
|
+
`frameworks[]`. Required fields:
|
|
79
|
+
- `name` (unique slug)
|
|
80
|
+
- `display_name`
|
|
81
|
+
- `category` (must match the classifier emission)
|
|
82
|
+
- `languages`
|
|
83
|
+
- `file_extensions`
|
|
84
|
+
- `execution_command` (with `{file}` placeholder)
|
|
85
|
+
- `status` (`preferred` / `supported` / `legacy`)
|
|
86
|
+
2. **Add recommended metadata.** `maturity`, `community_size`,
|
|
87
|
+
`recommended_for`, `strengths`, `avoid_when`, `ecosystems`. These show up in
|
|
88
|
+
the recommender's reasoning string — sparse entries produce sparse
|
|
89
|
+
recommendations.
|
|
90
|
+
3. **Validate the `execution_command`.** It must run a single test file when
|
|
91
|
+
`{file}` is substituted. Test manually:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
echo '<minimal test>' > /tmp/sample.<ext>
|
|
95
|
+
<execution_command with /tmp/sample.<ext> substituted>
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Confirm exit code 0 (or expected non-zero) and no missing-config errors.
|
|
99
|
+
|
|
100
|
+
4. **Confirm the `{file}` placeholder is the only substitution.** The executor
|
|
101
|
+
tokenizes the template with `shlex.split` before substituting `{file}`, so
|
|
102
|
+
the file path stays a single argv element; paths with spaces must remain
|
|
103
|
+
intact.
|
|
104
|
+
|
|
105
|
+
### Phase 4: VERIFY — Generate, Execute, Promote-Dry
|
|
106
|
+
|
|
107
|
+
1. **Generate a test that should route to the new framework.** Run the
|
|
108
|
+
`/canary-write-test` slash command in Claude Code with a prompt that should
|
|
109
|
+
hit the new `test_type`.
|
|
110
|
+
|
|
111
|
+
2. **Check the trace.** Classification matches the expected category,
|
|
112
|
+
recommendation picks the new framework (or the existing preferred — confirm
|
|
113
|
+
this matches your intent), output file has the right extension.
|
|
114
|
+
3. **Execute with `--execute`.** Confirm the framework's CLI actually runs the
|
|
115
|
+
generated file. A passing execution validates the `execution_command`
|
|
116
|
+
template under the real Canary invocation path.
|
|
117
|
+
4. **Don't promote.** This is a registry-validation run, not a real test
|
|
118
|
+
promotion. Leave the artifact in `tests/generated/`.
|
|
119
|
+
|
|
120
|
+
### Phase 5: DOCUMENT — Update Guides and State
|
|
121
|
+
|
|
122
|
+
1. **Update `docs/guides/framework-registry.md`** if you added a new category,
|
|
123
|
+
changed the contract surface, or introduced a new optional field schema.
|
|
124
|
+
2. **Update the LLM-providers guide** if the new framework only works with
|
|
125
|
+
specific provider output styles (rare; flag this as a smell if it's true).
|
|
126
|
+
3. **Log to `docs/CANARY_STATE.md`.** One line: framework name, category,
|
|
127
|
+
status, date added. This is the project ledger; future maintainers grep here
|
|
128
|
+
first.
|
|
129
|
+
4. **Open a PR.** Title: `feat(registry): add <framework-name> support`. Body
|
|
130
|
+
should include the validation steps you ran and the generated/executed
|
|
131
|
+
sample.
|
|
132
|
+
|
|
133
|
+
## Canary Integration
|
|
134
|
+
|
|
135
|
+
- **`ts/src/data/frameworks/registry.json`** — Source of truth for entries.
|
|
136
|
+
- **`ts/src/core/framework-registry.ts`** — Loader and lookup methods. Don't add
|
|
137
|
+
lookup helpers here unless the existing five (`get_all_frameworks`,
|
|
138
|
+
`get_by_category`, `get_preferred_by_category`, `find_by_name`,
|
|
139
|
+
`match_by_language`) genuinely don't fit.
|
|
140
|
+
- **`ts/src/core/classifier.ts`** — Routing rules. Coordinate changes with
|
|
141
|
+
registry entries.
|
|
142
|
+
- **`tests/unit/test_orchestrator.py`** — Contract enforcement. Non-null
|
|
143
|
+
resolution per `test_type` (api → pytest, e2e_ui → playwright, performance →
|
|
144
|
+
k6, etc.) is asserted here. `tests/unit/test_factory.py` separately enforces
|
|
145
|
+
the LLM provider matrix.
|
|
146
|
+
|
|
147
|
+
## Success Criteria
|
|
148
|
+
|
|
149
|
+
- New entry validates against the contract test (no orphaned `test_type` in
|
|
150
|
+
classifier, no null framework resolution in registry)
|
|
151
|
+
- The `/canary-write-test` slash command with a routing prompt resolves to the
|
|
152
|
+
new framework
|
|
153
|
+
- `--execute` runs the generated file successfully via the new
|
|
154
|
+
`execution_command`
|
|
155
|
+
- `framework-registry.md` documents any new category or schema field
|
|
156
|
+
- `CANARY_STATE.md` ledger entry is present
|
|
157
|
+
- PR includes the validation transcript
|
|
158
|
+
|
|
159
|
+
## Rationalizations to Reject
|
|
160
|
+
|
|
161
|
+
| Rationalization | Why It Is Wrong |
|
|
162
|
+
| ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
163
|
+
| "I'll add the registry entry now and update the classifier later" | Orphaned registry entries are routing traps. Either both land together or neither does. |
|
|
164
|
+
| "Marking it `preferred` is fine, no need to demote the existing entry" | Multiple `preferred` entries in one category means `get_preferred_by_category` picks by registry order, not by merit. Demote the loser explicitly. |
|
|
165
|
+
| "I tested the command with my path — Canary's substitution will work too" | Test the substituted command with a path that contains spaces. The executor's `shlex.split`-then-substitute order exists specifically to handle this case; a broken template will silently corrupt argv. |
|
|
166
|
+
| "It only needs to work for the happy-path prompt I tried" | If the classifier emits this `test_type` with confidence ≥0.7 for _any_ phrasing, the framework must handle the resulting generated file. Test at least two distinct prompts that route to the new entry. |
|
|
167
|
+
| "Sparse metadata is fine, we can fill it in later" | The recommender's reasoning string is read by humans and agents. Sparse metadata produces unhelpful recommendations — fill it in now while you have the context. |
|
|
168
|
+
|
|
169
|
+
## Examples
|
|
170
|
+
|
|
171
|
+
### Example: Adding Cypress as a peer to Playwright
|
|
172
|
+
|
|
173
|
+
**Scope:** Cypress already has community demand and a CI image. Category
|
|
174
|
+
`e2e_ui` already has Playwright as `preferred`. Decision: add Cypress as
|
|
175
|
+
`supported`, keep Playwright as `preferred`.
|
|
176
|
+
|
|
177
|
+
**Classifier:** Already emits `test_type=e2e_ui` on UI-test prompts. No
|
|
178
|
+
classifier change needed.
|
|
179
|
+
|
|
180
|
+
**Entry (abridged):**
|
|
181
|
+
|
|
182
|
+
```json
|
|
183
|
+
{
|
|
184
|
+
"name": "cypress",
|
|
185
|
+
"display_name": "Cypress",
|
|
186
|
+
"category": "e2e_ui",
|
|
187
|
+
"languages": ["javascript", "typescript"],
|
|
188
|
+
"file_extensions": ["cy.ts", "cy.js"],
|
|
189
|
+
"execution_command": "npx --yes cypress run --spec {file}",
|
|
190
|
+
"status": "supported",
|
|
191
|
+
"strengths": ["Time-travel debugging", "Strong DX for UI tests"],
|
|
192
|
+
"avoid_when": ["Cross-browser parity required (Playwright is stronger)"]
|
|
193
|
+
}
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
**Validation:** Generated a UI test prompt, confirmed recommender still picked
|
|
197
|
+
Playwright (correct — `preferred` wins). Verified Cypress is accessible by
|
|
198
|
+
routing a prompt with explicit Cypress mention through a future
|
|
199
|
+
explicit-framework override.
|
|
200
|
+
|
|
201
|
+
### Example: Adding k6 to a new performance category
|
|
202
|
+
|
|
203
|
+
**Scope:** Performance category doesn't exist in the registry yet. Classifier
|
|
204
|
+
emits `test_type=performance` on load/stress prompts, but the registry has no
|
|
205
|
+
matching entry — this is the orphan case.
|
|
206
|
+
|
|
207
|
+
**Phase 2 first:** Confirmed `ts/src/core/classifier.ts` already has the
|
|
208
|
+
performance rule (`if "performance" in p or "load test" in p ...`). No
|
|
209
|
+
classifier change needed.
|
|
210
|
+
|
|
211
|
+
**Phase 3:**
|
|
212
|
+
|
|
213
|
+
```json
|
|
214
|
+
{
|
|
215
|
+
"name": "k6",
|
|
216
|
+
"display_name": "k6",
|
|
217
|
+
"category": "performance",
|
|
218
|
+
"languages": ["javascript"],
|
|
219
|
+
"file_extensions": ["js"],
|
|
220
|
+
"execution_command": "k6 run {file}",
|
|
221
|
+
"status": "preferred",
|
|
222
|
+
"recommended_for": ["HTTP load testing", "Stress and spike tests"],
|
|
223
|
+
"avoid_when": ["Browser-level performance (use Playwright tracing)"]
|
|
224
|
+
}
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
**Validation:** Ran the `/canary-write-test` slash command with "Load test
|
|
228
|
+
/v1/search at 200 RPS for 5 minutes". Confirmed classifier emits
|
|
229
|
+
`test_type=performance` at 0.95 confidence, recommender picks k6, k6 CLI runs
|
|
230
|
+
the generated file without error.
|
|
231
|
+
|
|
232
|
+
## Escalation
|
|
233
|
+
|
|
234
|
+
- **When the framework requires a non-trivial config file (e.g.,
|
|
235
|
+
`playwright.config.ts`):** Document the setup in `avoid_when` or in
|
|
236
|
+
`recommended_for`. If Canary is expected to generate the config too, that's a
|
|
237
|
+
scaffolder change, not just a registry change.
|
|
238
|
+
- **When two frameworks legitimately should both be `preferred` for different
|
|
239
|
+
sub-cases:** Split the category. A single category should have a single
|
|
240
|
+
`preferred`. If the split isn't clean, surface this to the user as a design
|
|
241
|
+
decision rather than fudging the registry.
|
|
242
|
+
- **When the classifier doesn't reliably emit the target `test_type`:** Don't
|
|
243
|
+
add the registry entry yet. Strengthen the classifier first — an unreachable
|
|
244
|
+
entry is dead weight.
|
|
245
|
+
- **When the framework's CLI doesn't accept a single-file argument:** Canary's
|
|
246
|
+
model is one-test-per-file generation. Frameworks that only run by directory
|
|
247
|
+
or by tag don't fit cleanly; raise this with the user before forcing a
|
|
248
|
+
workaround.
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: canary-blackhawk
|
|
3
|
+
description:
|
|
4
|
+
Temporal-dependency linter for test files — statically flags tests that lean
|
|
5
|
+
on wall-clock time, a real delay, or the local timezone, the ones that pass
|
|
6
|
+
all day and fail at midnight, across a DST boundary, or on Feb 29. Suppresses
|
|
7
|
+
itself when the file already installs a frozen clock (fake timers, freezegun,
|
|
8
|
+
time_machine, MockDate). Self-contained, deterministic, advisory by default.
|
|
9
|
+
cli: scripts/cli.mjs
|
|
10
|
+
requires: [node>=20]
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# Canary Blackhawk
|
|
14
|
+
|
|
15
|
+
A test that reads the wall clock is a test with a scheduled outage. It passes
|
|
16
|
+
every run you watch, then fails at 23:59:59, on the Sunday the clocks move, or
|
|
17
|
+
on Feb 29. Blackhawk finds those lines before the calendar does.
|
|
18
|
+
|
|
19
|
+
Tier-0 deterministic analysis: no LLM, no network, no secrets, no dependency on
|
|
20
|
+
any other skill.
|
|
21
|
+
|
|
22
|
+
## Rules
|
|
23
|
+
|
|
24
|
+
| Rule | Severity | Fires on |
|
|
25
|
+
| ------------------------------ | -------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
26
|
+
| `BH001-wall-clock` | high | `Date.now()`, bare `new Date()`, `moment()`, `datetime.now()` / `.today()` / `.utcnow()`, `date.today()`, `time.time()`, `pd.Timestamp.now()` |
|
|
27
|
+
| `BH002-real-delay` | medium | `time.sleep(n)` and `setTimeout(fn, n)` with a literal `n > 0` |
|
|
28
|
+
| `BH003-local-timezone` | medium | `toLocaleString` / `toLocaleDateString` / `toLocaleTimeString`, `strftime('…%Z…')` or `%z` |
|
|
29
|
+
| `BH004-naive-datetime-compare` | low | a comparison against `datetime(2024, …)` or `strptime(…)` with no `tzinfo` / `timezone.utc` / `ZoneInfo` / `pytz` on the line |
|
|
30
|
+
|
|
31
|
+
A pinned constructor is never flagged: `new Date('2024-01-01T00:00:00Z')`,
|
|
32
|
+
`moment('2024-01-01')`, and `datetime(2024, 1, 1)` on its own are all fine.
|
|
33
|
+
|
|
34
|
+
## Framework-conditioned suppression (the important part)
|
|
35
|
+
|
|
36
|
+
The single biggest failure mode of a temporal linter is firing on tests that
|
|
37
|
+
**already handle time correctly**. So when a file installs a frozen clock, every
|
|
38
|
+
clock-dependent rule (`BH001`, `BH002`, `BH004`) goes quiet for that file.
|
|
39
|
+
Markers:
|
|
40
|
+
|
|
41
|
+
`vi.useFakeTimers` · `vi.setSystemTime` · `jest.useFakeTimers` ·
|
|
42
|
+
`jest.setSystemTime` · `sinon.useFakeTimers` · `MockDate` · `freeze_time` /
|
|
43
|
+
`freezegun` · `time_machine`
|
|
44
|
+
|
|
45
|
+
Two deliberate choices inside that:
|
|
46
|
+
|
|
47
|
+
- **Suppression is file-wide, not block-scoped.** `vi.useFakeTimers()` inside a
|
|
48
|
+
`beforeEach` governs tests declared above it, and answering "is the clock
|
|
49
|
+
frozen _here_" accurately needs a real parser. Blackhawk errs toward silence.
|
|
50
|
+
- **`BH003` is never suppressed.** Freezing the clock pins _when_ a test runs,
|
|
51
|
+
never _where_. A frozen clock does not stop `toLocaleString()` from returning
|
|
52
|
+
a different string on a developer laptop than on a UTC runner.
|
|
53
|
+
|
|
54
|
+
## Inline suppression (per-line escape hatch)
|
|
55
|
+
|
|
56
|
+
File-wide frozen-clock suppression is too coarse when one intentional real wait
|
|
57
|
+
sits among otherwise-deterministic tests. An inline pragma silences a single
|
|
58
|
+
finding:
|
|
59
|
+
|
|
60
|
+
```js
|
|
61
|
+
// blackhawk-ignore BH002 -- real MongoDB claim race; deterministic wait impossible
|
|
62
|
+
await new Promise((r) => setTimeout(r, 25));
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
- **Reason required** (the `-- reason` tail) — keeps suppressions honest and
|
|
66
|
+
greppable, like `eslint-disable-next-line` / `# noqa`.
|
|
67
|
+
- **Rule-scoped** (`BH002`, or the full `BH002-real-delay`) so it never
|
|
68
|
+
blanket-silences the line; a `BH001` on the same line still fires.
|
|
69
|
+
- **Placement:** the pragma may trail the offending line or sit on the line
|
|
70
|
+
directly above it. Comma-separate ids to suppress several (`BH002,BH004`).
|
|
71
|
+
- **Counted separately:** suppressed findings are reported as `N suppressed`,
|
|
72
|
+
out of the actionable total — so a genuinely clean suite can read zero
|
|
73
|
+
findings while the known-OK waits stay visible.
|
|
74
|
+
|
|
75
|
+
## Fidelity limits (regex/AST-lite, on purpose)
|
|
76
|
+
|
|
77
|
+
Blackhawk is a line scanner with no TypeScript parser dependency, so it ships
|
|
78
|
+
anywhere `node` does. The cost, stated plainly:
|
|
79
|
+
|
|
80
|
+
- **Line-scoped.** A call split across lines (`setTimeout(\n fn,\n 500\n)`) is
|
|
81
|
+
missed, as is a `setTimeout` whose callback body contains a comma.
|
|
82
|
+
- **Comment-blind, one level deep.** Lines starting with `#`, `//`, `*`, `/*`,
|
|
83
|
+
or a docstring quote are skipped; a multi-line block comment whose inner lines
|
|
84
|
+
do not start with `*` is still scanned.
|
|
85
|
+
- **String-aware, one line deep.** A match that _starts_ inside a string literal
|
|
86
|
+
on its own line is rejected as fixture data (`pyFile('time.sleep(1)')` does
|
|
87
|
+
not fire), while an anchor whose pattern merely reaches into quotes
|
|
88
|
+
(`strftime('..%Z')`) still does; template `${...}` interpolation is code. A
|
|
89
|
+
string spanning lines is only seen on its opening line, so a fixture blob's
|
|
90
|
+
continuation lines are still scanned like code.
|
|
91
|
+
- **No type awareness.** `.toLocaleString()` on a `Number` reads the same as on
|
|
92
|
+
a `Date`.
|
|
93
|
+
- **Suppression is a substring match.** A mention of `freezegun` in a comment
|
|
94
|
+
silences the file. That direction is intentional: a missed finding costs less
|
|
95
|
+
than a false one.
|
|
96
|
+
|
|
97
|
+
## Which files get scanned
|
|
98
|
+
|
|
99
|
+
A directory walk only visits **test** files — `*.test.*`, `*.spec.*`,
|
|
100
|
+
`test_*.py`, `*_test.py`, or any supported source under `tests/`, `test/`,
|
|
101
|
+
`__tests__/`, `e2e/`, `spec/`. A file named explicitly on the command line is
|
|
102
|
+
always scanned, test-looking or not. Supported suffixes: `.py`, `.js`, `.jsx`,
|
|
103
|
+
`.ts`, `.tsx`, `.mjs`, `.cjs`.
|
|
104
|
+
|
|
105
|
+
## Invocation
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
# Scan the repo's test files (advisory — always exits 0):
|
|
109
|
+
canary skills run canary-blackhawk
|
|
110
|
+
|
|
111
|
+
# Scan a specific suite:
|
|
112
|
+
canary skills run canary-blackhawk -- tests/e2e
|
|
113
|
+
|
|
114
|
+
# Machine-readable findings:
|
|
115
|
+
canary skills run canary-blackhawk -- tests --json
|
|
116
|
+
|
|
117
|
+
# Fail the step on any finding:
|
|
118
|
+
canary skills run canary-blackhawk -- tests --strict
|
|
119
|
+
|
|
120
|
+
# Usage, options, and the full rule list (exits 0):
|
|
121
|
+
canary skills run canary-blackhawk -- --help
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
An unknown flag is rejected with `unrecognized arguments: <flag>` and exit 2;
|
|
125
|
+
use `--` to end option parsing when a path itself starts with a dash.
|
|
126
|
+
|
|
127
|
+
`--json` shape:
|
|
128
|
+
|
|
129
|
+
```json
|
|
130
|
+
{
|
|
131
|
+
"schema_version": 1,
|
|
132
|
+
"findings": [
|
|
133
|
+
{
|
|
134
|
+
"file": "tests/clock.spec.ts",
|
|
135
|
+
"line": 12,
|
|
136
|
+
"rule_id": "BH001-wall-clock",
|
|
137
|
+
"severity": "high",
|
|
138
|
+
"snippet": "const t = Date.now();",
|
|
139
|
+
"why": "reads the wall clock, so the assertion depends on when the suite runs..."
|
|
140
|
+
}
|
|
141
|
+
],
|
|
142
|
+
"summary": {
|
|
143
|
+
"files_scanned": 8,
|
|
144
|
+
"findings": 1,
|
|
145
|
+
"by_severity": { "high": 1 },
|
|
146
|
+
"suppressed": 0
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## CI wiring (GitHub Actions)
|
|
152
|
+
|
|
153
|
+
Advisory first, then promote to blocking once the backlog is drained and the
|
|
154
|
+
signal is trusted — the same path every canary gate takes.
|
|
155
|
+
|
|
156
|
+
```yaml
|
|
157
|
+
- name: Temporal-dependency lint (advisory)
|
|
158
|
+
run: canary skills run canary-blackhawk -- tests
|
|
159
|
+
# Once clean, add --strict to make new offenders fail the PR:
|
|
160
|
+
# run: canary skills run canary-blackhawk -- tests --strict
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## Fixing what it finds
|
|
164
|
+
|
|
165
|
+
| Finding | Fix |
|
|
166
|
+
| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
167
|
+
| `BH001` | Freeze the clock (`vi.useFakeTimers()` + `vi.setSystemTime(...)`, `@freeze_time("2024-01-01")`) or inject a clock the test controls. |
|
|
168
|
+
| `BH002` | Advance a fake timer (`vi.advanceTimersByTime(500)`) or await the real condition instead of a duration. |
|
|
169
|
+
| `BH003` | Assert on a UTC representation (`toISOString()`, `strftime('%Y-%m-%dT%H:%M:%SZ')` on a UTC datetime), or pin the locale and timezone explicitly. |
|
|
170
|
+
| `BH004` | Attach a timezone: `datetime(2024, 1, 1, tzinfo=timezone.utc)`. |
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// canary-blackhawk -- temporal-dependency linter for test files.
|
|
3
|
+
//
|
|
4
|
+
// Statically flags tests that depend on wall-clock time, a real delay, or the
|
|
5
|
+
// local timezone: the ones that pass all day and fail at midnight, across a DST
|
|
6
|
+
// boundary, or on Feb 29.
|
|
7
|
+
//
|
|
8
|
+
// <paths> files or directories to scan (default: the current directory).
|
|
9
|
+
// --json emit machine-readable findings instead of human text.
|
|
10
|
+
// --strict exit 1 when there are findings (default is advisory: exit 0).
|
|
11
|
+
//
|
|
12
|
+
// Tier-0 deterministic analysis -- no LLM, no network, no secrets, no
|
|
13
|
+
// dependency on any other skill.
|
|
14
|
+
//
|
|
15
|
+
// Invoked via `canary skills run canary-blackhawk -- [paths] [--json] [--strict]`.
|
|
16
|
+
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import { scanPaths, toJson } from './scanner.mjs';
|
|
19
|
+
import { RULES } from './rules.mjs';
|
|
20
|
+
import {
|
|
21
|
+
createParser,
|
|
22
|
+
formatUsageError,
|
|
23
|
+
EXIT_USAGE,
|
|
24
|
+
} from '../../../lib/parse-args.mjs';
|
|
25
|
+
|
|
26
|
+
export const SCHEMA_VERSION = 1;
|
|
27
|
+
|
|
28
|
+
const PREFIX = 'canary-blackhawk:';
|
|
29
|
+
|
|
30
|
+
// --- no-silent-abstention (#508 D2, skill-CLI convention half) ---------------
|
|
31
|
+
//
|
|
32
|
+
// Skill CLIs are deliberately self-contained -- no engine import, no shared
|
|
33
|
+
// module -- so they cannot call `gateOutcome`. They honour the doctrine by
|
|
34
|
+
// CONVENTION instead, emitting the same greppable line the engine helper does.
|
|
35
|
+
// The skill-layer conformance registry (agents/skills/test/gate-conformance.
|
|
36
|
+
// test.ts) is what holds them to it: a row whose fixture collapses the
|
|
37
|
+
// denominator and asserts the loud outcome.
|
|
38
|
+
//
|
|
39
|
+
// U+26A0 / U+2014 are written as escapes so this source stays ASCII, matching
|
|
40
|
+
// ts/src/core/gate-result.ts.
|
|
41
|
+
const ABSTAINED_LINE =
|
|
42
|
+
'\u{26A0} Abstained \u{2014} verified zero items; this is not a pass.';
|
|
43
|
+
|
|
44
|
+
// The rules block is GENERATED from RULES, never hand-typed: a new rule shows
|
|
45
|
+
// up in --help the moment it is registered, so the help text cannot drift
|
|
46
|
+
// behind the linter as rules are added.
|
|
47
|
+
const USAGE =
|
|
48
|
+
'usage: canary-blackhawk [-h] [--json] [--strict] [--] [path ...]\n' +
|
|
49
|
+
'\n' +
|
|
50
|
+
'Temporal-dependency linter for test files: flags tests that depend on the\n' +
|
|
51
|
+
'wall clock, a real delay, or the local timezone.\n' +
|
|
52
|
+
'\n' +
|
|
53
|
+
'positional arguments:\n' +
|
|
54
|
+
' path files or directories to scan (default: the current directory)\n' +
|
|
55
|
+
'\n' +
|
|
56
|
+
'options:\n' +
|
|
57
|
+
' -h, --help show this help message and exit\n' +
|
|
58
|
+
' --json emit machine-readable findings instead of human text\n' +
|
|
59
|
+
' --strict exit 1 when there are findings (default is advisory: exit 0)\n' +
|
|
60
|
+
'\n' +
|
|
61
|
+
'rules:\n' +
|
|
62
|
+
RULES.map((r) => ` ${r.ruleId} (${r.severity})`).join('\n');
|
|
63
|
+
|
|
64
|
+
function summary(result) {
|
|
65
|
+
const bySeverity = {};
|
|
66
|
+
for (const f of result.findings) {
|
|
67
|
+
bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
files_scanned: result.filesScanned,
|
|
71
|
+
findings: result.findings.length,
|
|
72
|
+
by_severity: bySeverity,
|
|
73
|
+
suppressed: result.suppressed ?? 0,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// A trailing "N suppressed" note keeps inline-ignored lines visible but out of
|
|
78
|
+
// the actionable total - the pattern the PR-guardian sticky comment uses.
|
|
79
|
+
function suppressedNote(result) {
|
|
80
|
+
const n = result.suppressed ?? 0;
|
|
81
|
+
return n ? `\n${n} suppressed (inline blackhawk-ignore).` : '';
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function renderText(result) {
|
|
85
|
+
const count = result.findings.length;
|
|
86
|
+
const files = result.filesScanned;
|
|
87
|
+
const fp = files === 1 ? '' : 's';
|
|
88
|
+
// #508: zero findings over zero scanned files is an ABSENT result, not a
|
|
89
|
+
// clean one. Findings outrank abstention (a finding proves a file was read),
|
|
90
|
+
// so this is checked only on the no-findings path.
|
|
91
|
+
if (!count && !files) {
|
|
92
|
+
return (
|
|
93
|
+
`${ABSTAINED_LINE} No file matched the given paths, so there is ` +
|
|
94
|
+
'nothing to report. Point at a directory that holds test files, or ' +
|
|
95
|
+
'pass a file directly.'
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
if (!count) {
|
|
99
|
+
return (
|
|
100
|
+
`No temporal-dependency findings (${files} file${fp} scanned).` +
|
|
101
|
+
suppressedNote(result)
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
const sp = count === 1 ? '' : 's';
|
|
105
|
+
const lines = [
|
|
106
|
+
`${count} temporal-dependency finding${sp} in ${files} file${fp}:`,
|
|
107
|
+
'',
|
|
108
|
+
];
|
|
109
|
+
for (const f of result.findings) {
|
|
110
|
+
lines.push(` ${f.file}:${f.line} [${f.severity}] ${f.ruleId}`);
|
|
111
|
+
lines.push(` ${f.snippet}`);
|
|
112
|
+
lines.push(` why: ${f.why}`);
|
|
113
|
+
}
|
|
114
|
+
lines.push('');
|
|
115
|
+
lines.push(
|
|
116
|
+
'Advisory by default. Re-run with --strict to fail the step on findings.',
|
|
117
|
+
);
|
|
118
|
+
return lines.join('\n') + suppressedNote(result);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* blackhawk takes paths, so it gets the `--` end-of-options terminator (a file
|
|
123
|
+
* literally named `--json` stays reachable) and treats a lone `-` as a
|
|
124
|
+
* positional, as argparse does. The four shared invariants live in the shared
|
|
125
|
+
* parser (#479).
|
|
126
|
+
*/
|
|
127
|
+
export const CLI_SPEC = {
|
|
128
|
+
prog: 'canary-blackhawk',
|
|
129
|
+
booleans: { '--json': 'json', '--strict': 'strict' },
|
|
130
|
+
positionals: { key: 'paths', defaults: ['.'] },
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
const parseArgs = createParser(CLI_SPEC);
|
|
134
|
+
|
|
135
|
+
export function main(argv = []) {
|
|
136
|
+
const { positionals: paths, opts, help, error } = parseArgs(argv);
|
|
137
|
+
|
|
138
|
+
// Usage and parse errors resolve before any filesystem work, so `--help`
|
|
139
|
+
// never reports a missing path and a typo never half-runs a scan.
|
|
140
|
+
if (help) {
|
|
141
|
+
console.log(USAGE);
|
|
142
|
+
return 0;
|
|
143
|
+
}
|
|
144
|
+
if (error) {
|
|
145
|
+
console.error(formatUsageError(CLI_SPEC.prog, error));
|
|
146
|
+
return EXIT_USAGE;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
for (const entry of paths) {
|
|
150
|
+
if (!fs.existsSync(entry)) {
|
|
151
|
+
console.error(`${PREFIX} path not found: ${entry}`);
|
|
152
|
+
return 1;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const result = scanPaths(paths);
|
|
157
|
+
|
|
158
|
+
if (opts.json) {
|
|
159
|
+
console.log(
|
|
160
|
+
JSON.stringify(
|
|
161
|
+
{
|
|
162
|
+
schema_version: SCHEMA_VERSION,
|
|
163
|
+
findings: result.findings.map(toJson),
|
|
164
|
+
summary: summary(result),
|
|
165
|
+
},
|
|
166
|
+
null,
|
|
167
|
+
2,
|
|
168
|
+
),
|
|
169
|
+
);
|
|
170
|
+
} else {
|
|
171
|
+
console.log(renderText(result));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Advisory by default (D3). Under --strict the CLI carries an exit-code
|
|
175
|
+
// contract, so a collapsed denominator inherits EXIT_ABSTAINED (3) -- distinct
|
|
176
|
+
// from 1 ("found something real"), so CI can tell them apart.
|
|
177
|
+
if (opts.strict && !result.filesScanned) return 3;
|
|
178
|
+
return opts.strict && result.findings.length ? 1 : 0;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Direct execution (the skill runner execs this file via its shebang).
|
|
182
|
+
//
|
|
183
|
+
// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
|
|
184
|
+
// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
|
|
185
|
+
// truncated JSON that still exits 0 (#791).
|
|
186
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
187
|
+
process.exitCode = main(process.argv.slice(2));
|
|
188
|
+
}
|