canary-test-cli 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +308 -0
- package/agents/skills/canary:generate.md +49 -0
- package/agents/skills/canary:init.md +37 -0
- package/agents/skills/canary:migrate.md +66 -0
- package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
- package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
- package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
- package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
- package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
- package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
- package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
- package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
- package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
- package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
- package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
- package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
- package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
- package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
- package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
- package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
- package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
- package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
- package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
- package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
- package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
- package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
- package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
- package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
- package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
- package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
- package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
- package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
- package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
- package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
- package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
- package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
- package/agents/skills/lib/parse-args.mjs +214 -0
- package/dist/engine/analysis/cli.js +116 -54
- package/dist/engine/analysis/engine.js +34 -16
- package/dist/engine/analysis/reports.js +5 -4
- package/dist/engine/cli-commands.js +252 -106
- package/dist/engine/cli-common.js +15 -24
- package/dist/engine/cli.core.js +37 -11
- package/dist/engine/cli.js +2 -2
- package/dist/engine/company-knowledge-cli.js +2 -2
- package/dist/engine/core/adoption.js +408 -0
- package/dist/engine/core/framework-probes.js +7 -7
- package/dist/engine/core/fs-glob.js +2 -2
- package/dist/engine/core/gate-result.js +17 -0
- package/dist/engine/core/migrator.js +9 -17
- package/dist/engine/core/pattern-matcher.js +23 -5
- package/dist/engine/core/persona.js +421 -0
- package/dist/engine/core/promotion-verdict.js +261 -0
- package/dist/engine/core/quality-scorer.js +15 -2
- package/dist/engine/core/reporter.js +1 -9
- package/dist/engine/core/skill-dispatch.js +115 -0
- package/dist/engine/core/skill-examples.js +392 -0
- package/dist/engine/core/skill-registry.js +59 -4
- package/dist/engine/core/skill-surfaces.js +307 -0
- package/dist/engine/core/static-linter.js +310 -38
- package/dist/engine/core/test-files.js +77 -0
- package/dist/engine/core/ticket-updater.js +1 -7
- package/dist/engine/core/vacuity-scanner.js +726 -0
- package/dist/engine/core/workflow-discovery.js +2 -8
- package/dist/engine/core/workspace-detect.js +7 -6
- package/dist/engine/data/personas/registry.json +36 -0
- package/dist/engine/guardian/adjudication.js +5 -5
- package/dist/engine/guardian/analysis-emit.js +19 -28
- package/dist/engine/guardian/cli.js +150 -51
- package/dist/engine/guardian/coverage.js +1 -1
- package/dist/engine/guardian/diff-coverage/heuristic-tier.js +1 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +2 -2
- package/dist/engine/guardian/pr-check.js +322 -33
- package/dist/engine/guardian/pr-comment.js +4 -3
- package/dist/engine/history/cli.js +210 -6
- package/dist/engine/history/ndjson-store.js +9 -5
- package/dist/engine/history/record.js +34 -5
- package/dist/engine/history/run-recorder.js +165 -0
- package/dist/engine/history/schema.js +25 -7
- package/dist/engine/history/store.js +9 -0
- package/dist/engine/mcp-server.js +35 -13
- package/dist/engine/skills-cli.js +249 -23
- package/dist/engine/util/ensure-ascii.js +37 -0
- package/dist/engine/workflow-cli.js +6 -6
- package/dist/gate-result.d.ts +11 -0
- package/dist/gate-result.js +18 -0
- package/dist/reporters/testtracker.d.ts +1 -1
- package/dist/reporters/testtracker.js +1 -1
- package/dist/uninstall.js +12 -5
- package/package.json +3 -2
|
@@ -0,0 +1,726 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* canary-cassandra -- vacuous-test detection (#612).
|
|
3
|
+
*
|
|
4
|
+
* A vacuous test PASSES WITHOUT PROVING ANYTHING. It has assertions, it goes
|
|
5
|
+
* green, and it goes green identically against the bug it was written to catch,
|
|
6
|
+
* so every gate this repo owns reads it as healthy. Three shipped examples are
|
|
7
|
+
* recorded in #486 and all three cleared coverage, `review-test`, and CI:
|
|
8
|
+
*
|
|
9
|
+
* - an assertion whose expectation could not have been false (`toBe(true)`),
|
|
10
|
+
* - a test whose target was never actually invoked, and
|
|
11
|
+
* - a test whose only assertion was an ABSENCE, which the buggy code satisfied
|
|
12
|
+
* by crashing before it could do anything.
|
|
13
|
+
*
|
|
14
|
+
* This module is one implementation, consumed by both the `canary vacuity-check`
|
|
15
|
+
* CLI and the promotion gate (`promotion-verdict.ts`, #477). It is deliberately
|
|
16
|
+
* NOT a self-contained `.mjs` skill: #605's accepted risk was that
|
|
17
|
+
* `static_linter` and `quality_scorer` already overlap and a third
|
|
18
|
+
* half-enforcer would be the real defect. `agents/skills/claude-code/
|
|
19
|
+
* canary-cassandra/SKILL.md` drives this CLI rather than carrying a second copy
|
|
20
|
+
* of the detection.
|
|
21
|
+
*
|
|
22
|
+
* ## The fidelity ladder, and why VAC-002 needs one
|
|
23
|
+
*
|
|
24
|
+
* A test's "declared target" is declared nowhere. Inferring it from imports is
|
|
25
|
+
* exactly the heuristic tier STRATEGY.md distrusts, and the issue predicts the
|
|
26
|
+
* failure precisely: a correct integration test gets flagged when the call sits
|
|
27
|
+
* several frames deeper. So:
|
|
28
|
+
*
|
|
29
|
+
* - `annotated` -- the author wrote `@covers <symbol>`. The rule checks THAT
|
|
30
|
+
* symbol and says so.
|
|
31
|
+
* - `import-inferred` -- no annotation, so the target set is the symbols
|
|
32
|
+
* imported from relative paths, closed over local declarations to a fixpoint
|
|
33
|
+
* (one helper, or a chain of them, still counts as reaching the target).
|
|
34
|
+
* - neither -- the target cannot be resolved. That is "cannot verify", which is
|
|
35
|
+
* a finding about the SCAN, so it lands in `skipped` with its reason.
|
|
36
|
+
*
|
|
37
|
+
* The inference reads four binding forms, because #705 measured what happens
|
|
38
|
+
* when it reads one: 65 of 99 findings on canary's own tree were the namespace
|
|
39
|
+
* import, the dynamic import, and the subprocess launch -- tests invoking
|
|
40
|
+
* exactly what they claimed, through a construct the target set could not see.
|
|
41
|
+
* The issue rules out every quiet answer (a threshold, a mute, a widened blanket
|
|
42
|
+
* skip) on the grounds that a suppressed inference and a passing check must not
|
|
43
|
+
* look alike, so the fix is to WIDEN WHAT THE INFERENCE CAN SEE and leave the
|
|
44
|
+
* rule's authority untouched.
|
|
45
|
+
*
|
|
46
|
+
* A skip is PER RULE, not per test: `VAC-001` needs no target and always runs,
|
|
47
|
+
* so a test whose target is unresolvable is still genuinely `checked` and stays
|
|
48
|
+
* in the denominator. Saying otherwise would understate what was verified. What
|
|
49
|
+
* must not happen is a reader mistaking that for a full pass, which is why
|
|
50
|
+
* `promotion-verdict.ts` puts the skip count in its remedy rather than letting
|
|
51
|
+
* `promote` read as unqualified.
|
|
52
|
+
*
|
|
53
|
+
* ## The denominator
|
|
54
|
+
*
|
|
55
|
+
* `scanVacuity` returns a {@link GateResult}, so a file it could not read
|
|
56
|
+
* reports `checked: 0` and `gateOutcome` structurally refuses to print a pass.
|
|
57
|
+
* A vacuity detector that could itself go quiet and look clean would be the
|
|
58
|
+
* joke telling itself.
|
|
59
|
+
*/
|
|
60
|
+
import { readFileSync } from 'node:fs';
|
|
61
|
+
import { ASSERT_JS, ASSERT_PY, enumerateTests, frameworkForPath, } from './static-linter.js';
|
|
62
|
+
import { blankStringContent } from './string-literals.js';
|
|
63
|
+
/**
|
|
64
|
+
* `@covers <symbol>` -- the explicit rung of the ladder.
|
|
65
|
+
*
|
|
66
|
+
* Global, because {@link annotationFor} needs the LAST match in its window, not
|
|
67
|
+
* the first: `exec` returns the match nearest the start, which is the FARTHEST
|
|
68
|
+
* annotation above the declaration.
|
|
69
|
+
*/
|
|
70
|
+
const COVERS_PRAGMA = /@covers\s+([A-Za-z_$][\w$]*)/g;
|
|
71
|
+
/**
|
|
72
|
+
* An import whose specifier is relative: the local code a test can target.
|
|
73
|
+
*
|
|
74
|
+
* The namespace form (`import * as ns from './x.js'`) needs its own alternative
|
|
75
|
+
* rather than falling out of `(\w+)`: `*` is not a word character, so a file
|
|
76
|
+
* written entirely in namespace imports resolved to an EMPTY target set and
|
|
77
|
+
* every test in it drew a `VAC-002` (#705). On canary's own `agents/skills/test`
|
|
78
|
+
* tree that single omission was the largest share of the 65 findings the issue
|
|
79
|
+
* counted -- `import * as diffscan from '../.../diffscan.mjs'` is the house
|
|
80
|
+
* style there, and `diffscan.findDeletions(...)` is unmistakably an invocation
|
|
81
|
+
* of the target.
|
|
82
|
+
*/
|
|
83
|
+
const JS_RELATIVE_IMPORT = /import\s+(?:type\s+)?(?:\*\s+as\s+(\w+)|\{([^}]*)\}|(\w+))[^'"]*from\s*['"](\.[^'"]*)['"]/g;
|
|
84
|
+
const JS_RELATIVE_REQUIRE = /(?:const|let|var)\s+(?:\{([^}]*)\}|(\w+))\s*=\s*require\s*\(\s*['"](\.[^'"]*)['"]/g;
|
|
85
|
+
/**
|
|
86
|
+
* `const x = await import('./y.js')` / `const { a } = await import('./y.js')`.
|
|
87
|
+
*
|
|
88
|
+
* A dynamic import leaves no static import statement, so a suite that loads its
|
|
89
|
+
* subject this way -- to control module state per test, or to import a module
|
|
90
|
+
* only after an env var is set -- resolved to no target at all (#705).
|
|
91
|
+
*/
|
|
92
|
+
const JS_DYNAMIC_IMPORT = /(?:const|let|var)\s+(?:\{([^}]*)\}|(\w+))\s*=\s*(?:await\s+)?import\s*\(\s*['"](\.[^'"]*)['"]\s*\)/g;
|
|
93
|
+
/** A bare `await import('./y.js')` -- no binding, so it names no symbol. */
|
|
94
|
+
const JS_BARE_DYNAMIC_IMPORT = /(?<![\w$.])import\s*\(\s*['"]\.[^'"]*['"]\s*\)/;
|
|
95
|
+
/**
|
|
96
|
+
* A string literal naming a first-party script -- something a subprocess can be
|
|
97
|
+
* pointed at and that lives in this repo.
|
|
98
|
+
*
|
|
99
|
+
* The discriminator is deliberately the EXTENSION, not the path shape: it is
|
|
100
|
+
* what separates `spawnSync(cli, ...)` where `cli` is
|
|
101
|
+
* `path.join(SCRIPTS, 'cli.mjs')` from `spawnSync('git', args)`. A bare command
|
|
102
|
+
* name is not a repo path and must not make a test look covered.
|
|
103
|
+
*/
|
|
104
|
+
const SCRIPT_PATH_LITERAL = /['"`][^'"`\n]*[\w$)/.-]\.(?:mjs|cjs|jsx?|tsx?|py|sh)['"`]/;
|
|
105
|
+
/**
|
|
106
|
+
* A declaration binding one name to an expression -- the statement-bounded form
|
|
107
|
+
* used to spot a handle on a first-party script.
|
|
108
|
+
*/
|
|
109
|
+
const JS_SIMPLE_DECL = /(?:^|\n)\s*(?:const|let|var)\s+(\w+)\s*=\s*([^;\n]+)/g;
|
|
110
|
+
/** The child-process launchers whose first argument is an executable target. */
|
|
111
|
+
const SUBPROCESS_LAUNCH = /(?<![\w$.])(?:execFileSync|execSync|spawnSync|execFile|spawn|fork)\s*\(/;
|
|
112
|
+
/**
|
|
113
|
+
* Python has no `.`-prefix requirement for a first-party import, so `from x
|
|
114
|
+
* import y` counts. `import os` and the stdlib are excluded by name below --
|
|
115
|
+
* a heuristic, but the alternative is treating every pytest file as
|
|
116
|
+
* unresolvable.
|
|
117
|
+
*/
|
|
118
|
+
const PY_FROM_IMPORT = /^[ \t]*from\s+([\w.]+)\s+import\s+(\([^)]*\)|[^\n#]+)/gm;
|
|
119
|
+
const PY_STDLIB = new Set([
|
|
120
|
+
'os',
|
|
121
|
+
'sys',
|
|
122
|
+
'json',
|
|
123
|
+
're',
|
|
124
|
+
'time',
|
|
125
|
+
'math',
|
|
126
|
+
'pathlib',
|
|
127
|
+
'typing',
|
|
128
|
+
'datetime',
|
|
129
|
+
'unittest',
|
|
130
|
+
'pytest',
|
|
131
|
+
'collections',
|
|
132
|
+
'subprocess',
|
|
133
|
+
'tempfile',
|
|
134
|
+
'itertools',
|
|
135
|
+
'functools',
|
|
136
|
+
'socket',
|
|
137
|
+
'uuid',
|
|
138
|
+
'random',
|
|
139
|
+
]);
|
|
140
|
+
/**
|
|
141
|
+
* A local declaration whose body may reach the target set.
|
|
142
|
+
*
|
|
143
|
+
* Three shapes, and the third is not optional. Measured on canary's own suite,
|
|
144
|
+
* the largest single source of false positives was the testkit idiom
|
|
145
|
+
* `const { findings, write } = kitFor(dir)`: the imported target is `kitFor`,
|
|
146
|
+
* `findings()` reaches it, and a pattern that only understood `const x = ` saw
|
|
147
|
+
* none of it -- so every test in `doc-links.test.ts` read as touching nothing at
|
|
148
|
+
* all. An object pattern binds every name in it to the same reaching RHS.
|
|
149
|
+
*/
|
|
150
|
+
const JS_LOCAL_DECL = /(?:^|\n)\s*(?:export\s+)?(?:async\s+)?(?:function\s+(\w+)|(?:const|let|var)\s+(?:\{([^}]*)\}|\[([^\]]*)\]|(\w+))\s*=)/g;
|
|
151
|
+
const PY_LOCAL_DECL = /(?:^|\n)\s*def\s+(\w+)\s*\(/g;
|
|
152
|
+
/**
|
|
153
|
+
* A destructuring ASSIGNMENT with no declarator: `({ write, findings } =
|
|
154
|
+
* kitFor(root))`.
|
|
155
|
+
*
|
|
156
|
+
* The declare-then-assign-in-a-hook idiom -- `let findings: Kit['findings']` at
|
|
157
|
+
* module scope, bound inside `beforeEach`. `doc-links.test.ts` is written this
|
|
158
|
+
* way throughout, and because the binding line carries no `const`/`let`/`var`,
|
|
159
|
+
* a declaration-only pattern misses it and every test in the file reads as
|
|
160
|
+
* touching nothing at all.
|
|
161
|
+
*/
|
|
162
|
+
const JS_DESTRUCTURED_ASSIGN = /\(\s*\{([^}]*)\}\s*=\s*([^;\n]*)\)/g;
|
|
163
|
+
/** Every identifier bound by one declaration match (a pattern binds several). */
|
|
164
|
+
function boundNames(m, python) {
|
|
165
|
+
if (python)
|
|
166
|
+
return m[1] ? [m[1]] : [];
|
|
167
|
+
if (m[1])
|
|
168
|
+
return [m[1]];
|
|
169
|
+
if (m[4])
|
|
170
|
+
return [m[4]];
|
|
171
|
+
const pattern = m[2] ?? m[3] ?? '';
|
|
172
|
+
return pattern
|
|
173
|
+
.split(',')
|
|
174
|
+
.map((raw) => raw
|
|
175
|
+
.split(':')
|
|
176
|
+
.pop()
|
|
177
|
+
.trim()
|
|
178
|
+
.replace(/^\.\.\./, ''))
|
|
179
|
+
.filter((n) => /^[A-Za-z_$][\w$]*$/.test(n));
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Assertions whose expectation is an ABSENCE. A test built only from these
|
|
183
|
+
* passes identically when the code under test never ran at all -- the
|
|
184
|
+
* `canary-katana` case from #486, where a bare tmpdir exited before the write
|
|
185
|
+
* and `expect(existsSync(...)).toBe(false)` was free.
|
|
186
|
+
*/
|
|
187
|
+
const ABSENCE_ASSERTION = /\.toBe\s*\(\s*(?:false|null|undefined)\s*\)|\.toBeNull\s*\(|\.toBeUndefined\s*\(|\.toBeFalsy\s*\(|\.toHaveLength\s*\(\s*0\s*\)|\.toEqual\s*\(\s*(?:\[\s*\]|\{\s*\})\s*\)|\.not\s*\.\s*to\w+/;
|
|
188
|
+
const PY_ABSENCE_ASSERTION = /\bassert\s+not\b|\bis\s+None\b|==\s*(?:False|None)\b|==\s*(?:\[\s*\]|\{\s*\})|\bassert\s+len\s*\([^)]*\)\s*==\s*0\b/;
|
|
189
|
+
/**
|
|
190
|
+
* Any assertion at all -- imported from the linter rather than restated.
|
|
191
|
+
*
|
|
192
|
+
* A local `/\bexpect\s*\(|\bassert\s*[.(]/` was NOT the linter's vocabulary, and
|
|
193
|
+
* the comment claiming it was is how the gap survived: `ASSERT_JS` also knows
|
|
194
|
+
* `should`-style, `.should`, the `toThrow` family, and the
|
|
195
|
+
* `expectX()`/`assertX()` helper convention that the linter's own notes say
|
|
196
|
+
* accounted for 9 of 16 residual findings here. For a suite written in any of
|
|
197
|
+
* those styles the assertion list came out EMPTY, VAC-003's `length > 0` guard
|
|
198
|
+
* short-circuited, and the rule reported nothing while nothing said it could not
|
|
199
|
+
* look -- the silent zero this module exists to prevent, one layer inside it.
|
|
200
|
+
*/
|
|
201
|
+
const JS_ASSERTION = ASSERT_JS;
|
|
202
|
+
const PY_ASSERTION = new RegExp(`${ASSERT_PY.source}|\\bself\\.assert\\w+`);
|
|
203
|
+
function mk(file, line, rule, severity, test, message, suggestion, fidelity) {
|
|
204
|
+
const f = {
|
|
205
|
+
file,
|
|
206
|
+
line,
|
|
207
|
+
rule,
|
|
208
|
+
severity,
|
|
209
|
+
test,
|
|
210
|
+
message,
|
|
211
|
+
suggestion,
|
|
212
|
+
};
|
|
213
|
+
if (fidelity)
|
|
214
|
+
f.fidelity = fidelity;
|
|
215
|
+
return f;
|
|
216
|
+
}
|
|
217
|
+
function lineOf(code, offset) {
|
|
218
|
+
let n = 1;
|
|
219
|
+
for (let i = 0; i < offset && i < code.length; i += 1) {
|
|
220
|
+
if (code[i] === '\n')
|
|
221
|
+
n += 1;
|
|
222
|
+
}
|
|
223
|
+
return n;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* The identifiers a comma-separated import clause binds.
|
|
227
|
+
*
|
|
228
|
+
* `{ save as store }` binds `store`; `{ save }` binds `save`. Anything that is
|
|
229
|
+
* not a bare identifier after that (`type Kit`, a stray comment) is dropped --
|
|
230
|
+
* the filter is also what guarantees no name reaching {@link mentionsAny} can
|
|
231
|
+
* carry regex metacharacters.
|
|
232
|
+
*/
|
|
233
|
+
function clauseNames(list) {
|
|
234
|
+
// Parentheses and newlines stripped first, so the multi-line
|
|
235
|
+
// `from m import (\n a,\n b,\n)` form yields names rather than `(a` -- which
|
|
236
|
+
// the identifier filter below silently dropped, taking the whole file's target
|
|
237
|
+
// set with it.
|
|
238
|
+
return (list ?? '')
|
|
239
|
+
.replace(/[()\n]/g, ' ')
|
|
240
|
+
.split(',')
|
|
241
|
+
.map((raw) => raw
|
|
242
|
+
.trim()
|
|
243
|
+
.split(/\s+as\s+/)
|
|
244
|
+
.pop()
|
|
245
|
+
?.trim() ?? '')
|
|
246
|
+
.filter((name) => /^[A-Za-z_$][\w$]*$/.test(name));
|
|
247
|
+
}
|
|
248
|
+
/** Python first-party imports: `from x import y`, minus the stdlib by name. */
|
|
249
|
+
function pythonImportedTargets(code) {
|
|
250
|
+
const names = new Set();
|
|
251
|
+
for (const m of code.matchAll(PY_FROM_IMPORT)) {
|
|
252
|
+
const root = m[1].split('.')[0];
|
|
253
|
+
if (PY_STDLIB.has(root))
|
|
254
|
+
continue;
|
|
255
|
+
for (const n of clauseNames(m[2]))
|
|
256
|
+
names.add(n);
|
|
257
|
+
}
|
|
258
|
+
return names;
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* One binding form: which capture holds the `{...}` clause, and which the single
|
|
262
|
+
* name (default import, namespace alias, or `const x = ...`).
|
|
263
|
+
*/
|
|
264
|
+
const JS_BINDING_FORMS = [
|
|
265
|
+
{ re: JS_RELATIVE_IMPORT, clause: 2, singles: [1, 3] },
|
|
266
|
+
{ re: JS_RELATIVE_REQUIRE, clause: 1, singles: [2] },
|
|
267
|
+
{ re: JS_DYNAMIC_IMPORT, clause: 1, singles: [2] },
|
|
268
|
+
];
|
|
269
|
+
/** JS/TS first-party imports: any `import`/`require` with a relative specifier. */
|
|
270
|
+
function jsImportedTargets(code) {
|
|
271
|
+
const names = new Set();
|
|
272
|
+
for (const { re, clause, singles } of JS_BINDING_FORMS) {
|
|
273
|
+
// Reset explicitly: these are module-level `/g` patterns, so a leftover
|
|
274
|
+
// `lastIndex` from an earlier file would silently skip the head of this one.
|
|
275
|
+
re.lastIndex = 0;
|
|
276
|
+
for (const m of code.matchAll(re)) {
|
|
277
|
+
for (const n of clauseNames(m[clause]))
|
|
278
|
+
names.add(n);
|
|
279
|
+
for (const g of singles)
|
|
280
|
+
if (m[g])
|
|
281
|
+
names.add(m[g]);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
for (const n of subprocessScriptHandles(code))
|
|
285
|
+
names.add(n);
|
|
286
|
+
return names;
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Names bound to a first-party SCRIPT PATH -- the target of a subprocess test.
|
|
290
|
+
*
|
|
291
|
+
* `agents/skills/test` drives most skill CLIs the way a user does, by spawning
|
|
292
|
+
* them: `const cli = path.join(SCRIPTS, 'cli.mjs'); spawnSync(cli, ['--help'])`.
|
|
293
|
+
* No symbol crosses that boundary, so import-inferred fidelity saw a test that
|
|
294
|
+
* referenced none of its file's imports and reported `VAC-002` on a test that is
|
|
295
|
+
* in fact exercising exactly what it claims (#705).
|
|
296
|
+
*
|
|
297
|
+
* The handle -- `cli` -- is the symbol that stands in for the target, so binding
|
|
298
|
+
* it is what lets the existing machinery work unchanged, `closeOverLocals`
|
|
299
|
+
* included. A launch site must be present in the file: a path literal on its own
|
|
300
|
+
* is data (a fixture, an expected value), not an invocation.
|
|
301
|
+
*/
|
|
302
|
+
function subprocessScriptHandles(code) {
|
|
303
|
+
const names = new Set();
|
|
304
|
+
if (!SUBPROCESS_LAUNCH.test(code))
|
|
305
|
+
return names;
|
|
306
|
+
JS_SIMPLE_DECL.lastIndex = 0;
|
|
307
|
+
for (const m of code.matchAll(JS_SIMPLE_DECL)) {
|
|
308
|
+
if (SCRIPT_PATH_LITERAL.test(m[2] ?? ''))
|
|
309
|
+
names.add(m[1]);
|
|
310
|
+
}
|
|
311
|
+
return names;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Does this test body itself reach first-party code the target set cannot name?
|
|
315
|
+
*
|
|
316
|
+
* Two shapes, both of which leave no identifier to match: a subprocess launched
|
|
317
|
+
* at a script path written inline (`spawnSync(path.join(D, 'cli.mjs'), ...)`),
|
|
318
|
+
* and a bare `await import('./x.js')` whose result is never bound. Read from the
|
|
319
|
+
* ORIGINAL source rather than the blanked copy, because the evidence in both
|
|
320
|
+
* cases IS the string literal.
|
|
321
|
+
*
|
|
322
|
+
* `SUBPROCESS_LAUNCH` and `SCRIPT_PATH_LITERAL` are required together: a test
|
|
323
|
+
* that spawns `git` and separately mentions a `.py` fixture path is not covered
|
|
324
|
+
* by either half alone.
|
|
325
|
+
*/
|
|
326
|
+
function reachesOutOfBandTarget(rawBody) {
|
|
327
|
+
if (SUBPROCESS_LAUNCH.test(rawBody) && SCRIPT_PATH_LITERAL.test(rawBody))
|
|
328
|
+
return true;
|
|
329
|
+
return JS_BARE_DYNAMIC_IMPORT.test(rawBody);
|
|
330
|
+
}
|
|
331
|
+
/** Names imported from first-party (relative) modules. */
|
|
332
|
+
function importedTargets(code, python) {
|
|
333
|
+
return python ? pythonImportedTargets(code) : jsImportedTargets(code);
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Brace depth immediately BEFORE each character, over already-blanked code.
|
|
337
|
+
*
|
|
338
|
+
* Cheap and approximate on purpose: string content is blanked before this runs,
|
|
339
|
+
* so the only braces it can see are real ones (a `{` inside a comment is the
|
|
340
|
+
* residual inaccuracy, and it can only widen a body, never narrow one).
|
|
341
|
+
*/
|
|
342
|
+
function braceDepths(code) {
|
|
343
|
+
const depths = new Int32Array(code.length);
|
|
344
|
+
let d = 0;
|
|
345
|
+
for (let i = 0; i < code.length; i += 1) {
|
|
346
|
+
depths[i] = d;
|
|
347
|
+
const c = code[i];
|
|
348
|
+
if (c === '{')
|
|
349
|
+
d += 1;
|
|
350
|
+
else if (c === '}')
|
|
351
|
+
d -= 1;
|
|
352
|
+
}
|
|
353
|
+
return depths;
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* Where declaration `i`'s body ends.
|
|
357
|
+
*
|
|
358
|
+
* Bounding it at the NEXT declaration is wrong for any helper that declares
|
|
359
|
+
* something inside itself, and that is the common shape for the subprocess
|
|
360
|
+
* helper #705 is about:
|
|
361
|
+
*
|
|
362
|
+
* ```ts
|
|
363
|
+
* const SCRIPT = join(REPO_ROOT, 'scripts', 'entropy-ratchet.mjs');
|
|
364
|
+
* function run() {
|
|
365
|
+
* const r = spawnSync(process.execPath, [SCRIPT, ...]); // <- next decl
|
|
366
|
+
* ...
|
|
367
|
+
* }
|
|
368
|
+
* ```
|
|
369
|
+
*
|
|
370
|
+
* `run`'s body stopped at `const r`, so it never saw `SCRIPT`, so `run()` did
|
|
371
|
+
* not reach the target and every test calling it read as vacuous. Nesting is the
|
|
372
|
+
* discriminator: the body runs to the next declaration at the same or shallower
|
|
373
|
+
* brace depth, which is the first one that is genuinely a SIBLING.
|
|
374
|
+
*/
|
|
375
|
+
function declEnd(code, matches, i, depth) {
|
|
376
|
+
if (depth === null)
|
|
377
|
+
return matches[i + 1]?.index ?? code.length;
|
|
378
|
+
const own = depth[matches[i].index] ?? 0;
|
|
379
|
+
for (let j = i + 1; j < matches.length; j += 1) {
|
|
380
|
+
const at = matches[j].index;
|
|
381
|
+
if ((depth[at] ?? 0) <= own)
|
|
382
|
+
return at;
|
|
383
|
+
}
|
|
384
|
+
return code.length;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Grow `targets` with local declarations that themselves reach a target, to a
|
|
388
|
+
* fixpoint.
|
|
389
|
+
*
|
|
390
|
+
* This is the concession the issue asked for. Without it, a test that goes
|
|
391
|
+
* through a helper defined in the same file -- `roundTrip()` calling
|
|
392
|
+
* `load(save(v))` -- reads as never touching its target, and the rule
|
|
393
|
+
* confidently reports a correct test as vacuous. One hop covers the common
|
|
394
|
+
* case; the fixpoint covers a chain of them.
|
|
395
|
+
*/
|
|
396
|
+
function closeOverLocals(code, targets, python) {
|
|
397
|
+
const decls = [];
|
|
398
|
+
const re = python ? PY_LOCAL_DECL : JS_LOCAL_DECL;
|
|
399
|
+
re.lastIndex = 0;
|
|
400
|
+
const matches = [...code.matchAll(re)];
|
|
401
|
+
const depth = python ? null : braceDepths(code);
|
|
402
|
+
for (let i = 0; i < matches.length; i += 1) {
|
|
403
|
+
const m = matches[i];
|
|
404
|
+
const names = boundNames(m, python);
|
|
405
|
+
if (names.length === 0)
|
|
406
|
+
continue;
|
|
407
|
+
const start = m.index + m[0].length;
|
|
408
|
+
const end = declEnd(code, matches, i, depth);
|
|
409
|
+
decls.push({ names, body: code.slice(start, end) });
|
|
410
|
+
}
|
|
411
|
+
if (!python) {
|
|
412
|
+
JS_DESTRUCTURED_ASSIGN.lastIndex = 0;
|
|
413
|
+
for (const m of code.matchAll(JS_DESTRUCTURED_ASSIGN)) {
|
|
414
|
+
const names = (m[1] ?? '')
|
|
415
|
+
.split(',')
|
|
416
|
+
.map((raw) => raw.split(':').pop().trim())
|
|
417
|
+
.filter((n) => /^[A-Za-z_$][\w$]*$/.test(n));
|
|
418
|
+
// The RHS alone is the body here: unlike a declaration, an assignment
|
|
419
|
+
// does not own the text that follows it.
|
|
420
|
+
if (names.length > 0)
|
|
421
|
+
decls.push({ names, body: m[2] ?? '' });
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
const reaching = new Set(targets);
|
|
425
|
+
let grew = true;
|
|
426
|
+
while (grew) {
|
|
427
|
+
grew = false;
|
|
428
|
+
for (const d of decls) {
|
|
429
|
+
if (d.names.every((n) => reaching.has(n)))
|
|
430
|
+
continue;
|
|
431
|
+
const reaches = [...reaching].some((t) => identifierPattern(t).test(d.body));
|
|
432
|
+
if (!reaches)
|
|
433
|
+
continue;
|
|
434
|
+
for (const n of d.names)
|
|
435
|
+
reaching.add(n);
|
|
436
|
+
grew = true;
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
return reaching;
|
|
440
|
+
}
|
|
441
|
+
/** Body lines of a test, paired with their 1-based line numbers. */
|
|
442
|
+
function bodyLines(code, block) {
|
|
443
|
+
const first = lineOf(code, block.bodyStart);
|
|
444
|
+
return block.body.split('\n').map((text, i) => ({ text, line: first + i }));
|
|
445
|
+
}
|
|
446
|
+
function isComment(line) {
|
|
447
|
+
const s = line.trim();
|
|
448
|
+
return s.startsWith('#') || s.startsWith('//') || s.startsWith('*');
|
|
449
|
+
}
|
|
450
|
+
/** The text inside a balanced `expect(...)`, or null. */
|
|
451
|
+
function expectArgument(line) {
|
|
452
|
+
const open = line.indexOf('expect(');
|
|
453
|
+
if (open < 0)
|
|
454
|
+
return null;
|
|
455
|
+
let depth = 0;
|
|
456
|
+
for (let i = open + 'expect'.length; i < line.length; i += 1) {
|
|
457
|
+
if (line[i] === '(')
|
|
458
|
+
depth += 1;
|
|
459
|
+
else if (line[i] === ')') {
|
|
460
|
+
depth -= 1;
|
|
461
|
+
if (depth === 0)
|
|
462
|
+
return line.slice(open + 'expect('.length, i);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
return null;
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* The matcher following `expect(...)`: its argument, and whether it is negated.
|
|
469
|
+
*
|
|
470
|
+
* The negation is returned rather than swallowed. `expect(v).not.toBe(v)` has
|
|
471
|
+
* identical texts on both sides, so a comparison that ignored `.not` reported it
|
|
472
|
+
* as `VAC-001` -- "no implementation can fail it" -- about an assertion that can
|
|
473
|
+
* only ever FAIL. Inverting the rule's own claim is worse than missing the case,
|
|
474
|
+
* and it was a `critical` finding that BLOCKS promotion. `pyTautology` already
|
|
475
|
+
* guarded the analogous `assert False`; the JS path had no equivalent.
|
|
476
|
+
*/
|
|
477
|
+
function matcherOf(line) {
|
|
478
|
+
const m = /\.\s*(not\s*\.\s*)?to\w+\s*\(([^()]*)\)/.exec(line);
|
|
479
|
+
return m ? { argument: m[2], negated: m[1] !== undefined } : null;
|
|
480
|
+
}
|
|
481
|
+
function normalize(expr) {
|
|
482
|
+
return expr.replace(/\s+/g, '');
|
|
483
|
+
}
|
|
484
|
+
/** VAC-001 for one JS/TS line. */
|
|
485
|
+
function jsTautology(line) {
|
|
486
|
+
const actual = expectArgument(line);
|
|
487
|
+
const matcher = matcherOf(line);
|
|
488
|
+
if (actual === null || matcher === null || matcher.negated)
|
|
489
|
+
return false;
|
|
490
|
+
const a = normalize(actual);
|
|
491
|
+
const e = normalize(matcher.argument);
|
|
492
|
+
if (a === '' || e === '')
|
|
493
|
+
return false;
|
|
494
|
+
return a === e;
|
|
495
|
+
}
|
|
496
|
+
/** VAC-001 for one pytest line. */
|
|
497
|
+
function pyTautology(line) {
|
|
498
|
+
const t = line.trim();
|
|
499
|
+
// `assert False` is a deliberate unreachable marker -- it can only ever fail,
|
|
500
|
+
// so it is the opposite of vacuous and must never be flagged.
|
|
501
|
+
if (/^assert\s+True\s*(?:,|$)/.test(t))
|
|
502
|
+
return true;
|
|
503
|
+
// Same reason `.not` is excluded above: `assert x != x` can only ever fail.
|
|
504
|
+
const cmp = /^assert\s+(.+?)\s*==\s*(.+?)\s*(?:,|$)/.exec(t);
|
|
505
|
+
if (!cmp)
|
|
506
|
+
return false;
|
|
507
|
+
return normalize(cmp[1]) === normalize(cmp[2]);
|
|
508
|
+
}
|
|
509
|
+
function scanBlock(code, block, file, python, reaching, annotated, skipped, outOfBand) {
|
|
510
|
+
const lines = bodyLines(code, block).filter((l) => !isComment(l.text));
|
|
511
|
+
const targets = annotated !== null ? new Set([annotated]) : reaching;
|
|
512
|
+
return [
|
|
513
|
+
...tautologies(lines, block, file, python),
|
|
514
|
+
...targetNeverInvoked(block, file, reaching, annotated, outOfBand),
|
|
515
|
+
...absenceOnly(lines, block, file, python, targets, skipped),
|
|
516
|
+
];
|
|
517
|
+
}
|
|
518
|
+
/**
|
|
519
|
+
* A pattern matching `name` as a whole identifier.
|
|
520
|
+
*
|
|
521
|
+
* NOT `\b${name}\b`, which is wrong for the `$` that JS identifiers allow and
|
|
522
|
+
* `\w` does not. `\b` sits between a `\w` and a non-`\w`, so `\b$fetch\b` can
|
|
523
|
+
* only match after a word character -- a `$`-prefixed import never matched at
|
|
524
|
+
* all, and `\bfoo$bar\b` can never match. That silently shrank the target set,
|
|
525
|
+
* producing a `VAC-002` false positive on a test invoking its target on the only
|
|
526
|
+
* line it had. Lookarounds over `[\w$]` give the boundary JS actually has.
|
|
527
|
+
*
|
|
528
|
+
* `name` is escaped as well: every call site filters to `[A-Za-z_$][\w$]*`
|
|
529
|
+
* today, so nothing can currently smuggle a metacharacter through, but the
|
|
530
|
+
* escape means a widened filter cannot turn into a silent semantic change.
|
|
531
|
+
*/
|
|
532
|
+
function identifierPattern(name) {
|
|
533
|
+
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
534
|
+
return new RegExp(`(?<![\\w$])${escaped}(?![\\w$])`);
|
|
535
|
+
}
|
|
536
|
+
/** Does `text` name any symbol in `targets`? */
|
|
537
|
+
function mentionsAny(text, targets) {
|
|
538
|
+
return (targets !== null &&
|
|
539
|
+
[...targets].some((t) => identifierPattern(t).test(text)));
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* VAC-001 -- deterministic, hence `critical`: an expectation identical to the
|
|
543
|
+
* value it checks cannot fail for any implementation.
|
|
544
|
+
*/
|
|
545
|
+
function tautologies(lines, block, file, python) {
|
|
546
|
+
return lines
|
|
547
|
+
.filter((l) => (python ? pyTautology(l.text) : jsTautology(l.text)))
|
|
548
|
+
.map((l) => mk(file, l.line, 'VAC-001', 'critical', block.name, 'Assertion compares a value with itself; no implementation can fail it.', 'Assert the value the code under test should have produced, not the input.'));
|
|
549
|
+
}
|
|
550
|
+
/** VAC-002 -- the target is never referenced anywhere in the body. */
|
|
551
|
+
function targetNeverInvoked(block, file, reaching, annotated, outOfBand) {
|
|
552
|
+
if (annotated !== null) {
|
|
553
|
+
if (mentionsAny(block.body, new Set([annotated])))
|
|
554
|
+
return [];
|
|
555
|
+
return [
|
|
556
|
+
mk(file, block.line, 'VAC-002', 'warning', block.name, `Declared target \`${annotated}\` is never referenced in this test.`, 'Invoke the target, or correct the @covers annotation to name what the test actually exercises.', 'annotated'),
|
|
557
|
+
];
|
|
558
|
+
}
|
|
559
|
+
// The subprocess / bare-dynamic-import shapes reach first-party code without
|
|
560
|
+
// naming a symbol, so no target set can ever match them (#705).
|
|
561
|
+
if (outOfBand)
|
|
562
|
+
return [];
|
|
563
|
+
if (reaching === null || mentionsAny(block.body, reaching))
|
|
564
|
+
return [];
|
|
565
|
+
return [
|
|
566
|
+
mk(file, block.line, 'VAC-002', 'warning', block.name, 'This test references none of the symbols the file imports from first-party modules.', 'If the target is reached indirectly, add `// @covers <symbol>` so the check verifies the real target instead of inferring one.', 'import-inferred'),
|
|
567
|
+
];
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* VAC-003 -- every assertion is an absence, AND none of them observes the
|
|
571
|
+
* target.
|
|
572
|
+
*
|
|
573
|
+
* That second clause is not a refinement, it is the rule. The first cut omitted
|
|
574
|
+
* it and reported 254 findings across canary's 2154 tests, nearly all of the
|
|
575
|
+
* form `expect(isCI()).toBe(false)` -- a perfectly good negative test, because
|
|
576
|
+
* the assertion invokes the target, so the target provably ran and the `false`
|
|
577
|
+
* is load-bearing. The #486 katana defect is the other shape:
|
|
578
|
+
* `expect(existsSync(ledger)).toBe(false)` after a bare call, where the absence
|
|
579
|
+
* is observed on a BYSTANDER and the buggy code satisfied it by exiting before
|
|
580
|
+
* the write. Reported at the first absence assertion, the line an author adds a
|
|
581
|
+
* precondition next to.
|
|
582
|
+
*/
|
|
583
|
+
function absenceOnly(lines, block, file, python, targets, skipped) {
|
|
584
|
+
if (targets === null)
|
|
585
|
+
return [];
|
|
586
|
+
const anyAssertion = python ? PY_ASSERTION : JS_ASSERTION;
|
|
587
|
+
const absence = python ? PY_ABSENCE_ASSERTION : ABSENCE_ASSERTION;
|
|
588
|
+
const assertions = lines.filter((l) => anyAssertion.test(l.text));
|
|
589
|
+
if (assertions.length === 0) {
|
|
590
|
+
// Zero recognised assertions is unanswerable, not clean: either the test
|
|
591
|
+
// asserts nothing (which is `LINT-006`'s finding, not this rule's) or its
|
|
592
|
+
// assertion style is one the vocabulary does not know. Both are "cannot
|
|
593
|
+
// verify", so both are recorded rather than passed over in silence.
|
|
594
|
+
skipped.push({
|
|
595
|
+
name: `VAC-003 (${block.name})`,
|
|
596
|
+
reason: 'no recognised assertion, so absence-only could not be judged -- the test may assert nothing (LINT-006) or use an unrecognised assertion style',
|
|
597
|
+
});
|
|
598
|
+
return [];
|
|
599
|
+
}
|
|
600
|
+
if (!assertions.every((l) => absence.test(l.text)))
|
|
601
|
+
return [];
|
|
602
|
+
if (assertions.some((l) => mentionsAny(l.text, targets)))
|
|
603
|
+
return [];
|
|
604
|
+
return [
|
|
605
|
+
mk(file, assertions[0].line, 'VAC-003', 'warning', block.name, 'Every assertion in this test asserts an absence, and none of them observes the target.', 'Add one assertion proving the operation actually ran (exit code, returned value, a positive existence) -- otherwise the test passes identically when the code crashed before doing anything.'),
|
|
606
|
+
];
|
|
607
|
+
}
|
|
608
|
+
/** A zero-denominator result that names why it could not measure. */
|
|
609
|
+
function unreadable(path, reason) {
|
|
610
|
+
return { checked: 0, findings: [], skipped: [{ name: path, reason }] };
|
|
611
|
+
}
|
|
612
|
+
function readSource(path) {
|
|
613
|
+
try {
|
|
614
|
+
return { ok: true, source: readFileSync(path, 'utf-8') };
|
|
615
|
+
}
|
|
616
|
+
catch (e) {
|
|
617
|
+
const code = e.code ?? 'unknown error';
|
|
618
|
+
return { ok: false, reason: `could not be read (${code})` };
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* The target set for the `import-inferred` rung, or `null` when there is none.
|
|
623
|
+
*
|
|
624
|
+
* Imports are read from the ORIGINAL source, not the blanked copy: blanking
|
|
625
|
+
* replaces literal CONTENT with spaces, so `from './store.js'` becomes
|
|
626
|
+
* `from ' '` and the leading `.` that marks a first-party module is
|
|
627
|
+
* gone -- which silently collapsed every JS/TS file to "target unresolvable".
|
|
628
|
+
*
|
|
629
|
+
* The cost is that an import written inside a fixture string is read as real.
|
|
630
|
+
* That only ever ADDS names to the target set, which makes VAC-002 quieter,
|
|
631
|
+
* never noisier -- the safe direction for a heuristic-tier rule.
|
|
632
|
+
*/
|
|
633
|
+
function resolveTargets(source, code, python) {
|
|
634
|
+
const imported = importedTargets(source, python);
|
|
635
|
+
return imported.size > 0 ? closeOverLocals(code, imported, python) : null;
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* The `@covers` symbol declared above `block`, or `null`.
|
|
639
|
+
*
|
|
640
|
+
* Two bugs lived in the naive version, and both produced a FALSE BLOCK, which is
|
|
641
|
+
* the worst outcome available here: `annotated` is the one vacuity fidelity
|
|
642
|
+
* allowed to block a promotion, so a stray annotation failed a correct test.
|
|
643
|
+
*
|
|
644
|
+
* - The window was a blind 400-character look-back, so it reached over the
|
|
645
|
+
* PREVIOUS test and its annotation. It is now floored at `floor` -- the end of
|
|
646
|
+
* the previous test's body -- so only text genuinely between the two
|
|
647
|
+
* declarations can be read.
|
|
648
|
+
* - `exec` returns the match nearest the START of the window, i.e. the FARTHEST
|
|
649
|
+
* annotation above the declaration. It now takes the last, which is the
|
|
650
|
+
* nearest.
|
|
651
|
+
*/
|
|
652
|
+
function annotationFor(code, block, floor) {
|
|
653
|
+
const from = Math.max(floor, block.bodyStart - 400);
|
|
654
|
+
const window = code.slice(from, block.bodyStart);
|
|
655
|
+
const matches = [...window.matchAll(COVERS_PRAGMA)];
|
|
656
|
+
return matches.at(-1)?.[1] ?? null;
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Scan one test file for vacuous tests.
|
|
660
|
+
*
|
|
661
|
+
* `checked` counts the tests actually analysed. A file no ruleset can parse
|
|
662
|
+
* yields `checked: 0` plus a skip entry, never an empty finding list that reads
|
|
663
|
+
* as clean.
|
|
664
|
+
*/
|
|
665
|
+
export function scanVacuity(path) {
|
|
666
|
+
const framework = frameworkForPath(path);
|
|
667
|
+
if (framework === null) {
|
|
668
|
+
return unreadable(path, 'no ruleset parses this extension, so a clean result would be meaningless');
|
|
669
|
+
}
|
|
670
|
+
const python = framework === 'pytest';
|
|
671
|
+
const read = readSource(path);
|
|
672
|
+
if (!read.ok)
|
|
673
|
+
return unreadable(path, read.reason);
|
|
674
|
+
const source = read.source;
|
|
675
|
+
// Whole-source blanking, offset-preserving: a `expect(true).toBe(true)`
|
|
676
|
+
// carried as fixture DATA is not a vacuous test, and a `it(...)` inside a
|
|
677
|
+
// string must not be able to truncate a real test's body (#590).
|
|
678
|
+
const code = blankStringContent(source, { python });
|
|
679
|
+
const blocks = enumerateTests(code, source, python);
|
|
680
|
+
const reaching = resolveTargets(source, code, python);
|
|
681
|
+
const skipped = [];
|
|
682
|
+
const findings = scanAllBlocks({ code, source, path, python, reaching }, blocks, skipped);
|
|
683
|
+
const result = {
|
|
684
|
+
checked: blocks.length,
|
|
685
|
+
findings,
|
|
686
|
+
};
|
|
687
|
+
if (skipped.length > 0)
|
|
688
|
+
result.skipped = skipped;
|
|
689
|
+
return result;
|
|
690
|
+
}
|
|
691
|
+
function scanAllBlocks(ctx, blocks, skipped) {
|
|
692
|
+
const findings = [];
|
|
693
|
+
for (let i = 0; i < blocks.length; i += 1) {
|
|
694
|
+
const block = blocks[i];
|
|
695
|
+
// An annotation may only be read from the gap between the previous test's
|
|
696
|
+
// end and this declaration -- see `annotationFor`.
|
|
697
|
+
const prev = blocks[i - 1];
|
|
698
|
+
const floor = prev ? prev.bodyStart + prev.body.length : 0;
|
|
699
|
+
const annotated = annotationFor(ctx.code, block, floor);
|
|
700
|
+
const outOfBand = !ctx.python &&
|
|
701
|
+
annotated === null &&
|
|
702
|
+
reachesOutOfBandTarget(ctx.source.slice(block.bodyStart, block.bodyStart + block.body.length));
|
|
703
|
+
if (annotated === null && ctx.reaching === null) {
|
|
704
|
+
// Both target-dependent rules go dark together, and both say so. VAC-003
|
|
705
|
+
// asks "does any assertion observe the target", which is unanswerable
|
|
706
|
+
// without a target -- so it abstains rather than falling back to the
|
|
707
|
+
// 254-false-positive version of itself.
|
|
708
|
+
//
|
|
709
|
+
// An out-of-band reach answers VAC-002 (the test DOES invoke first-party
|
|
710
|
+
// code) but not VAC-003, which needs a SYMBOL to ask "did an assertion
|
|
711
|
+
// observe it". So the skip narrows rather than disappearing: reporting
|
|
712
|
+
// both as dark would overstate the gap, dropping it entirely would hide a
|
|
713
|
+
// real one, and #705 is explicit that a suppressed inference and a passing
|
|
714
|
+
// check must not look alike.
|
|
715
|
+
skipped.push({
|
|
716
|
+
name: `${outOfBand ? 'VAC-003' : 'VAC-002/VAC-003'} (${block.name})`,
|
|
717
|
+
reason: outOfBand
|
|
718
|
+
? 'target reached out of band (subprocess or bare dynamic import), so VAC-002 is answered but no symbol exists for absence-only to observe'
|
|
719
|
+
: 'target unresolvable: no @covers annotation and no first-party relative import to infer from',
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
findings.push(...scanBlock(ctx.code, block, ctx.path, ctx.python, ctx.reaching, annotated, skipped, outOfBand));
|
|
723
|
+
}
|
|
724
|
+
return findings;
|
|
725
|
+
}
|
|
726
|
+
//# sourceMappingURL=vacuity-scanner.js.map
|