canary-test-cli 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +308 -0
- package/agents/skills/canary:generate.md +49 -0
- package/agents/skills/canary:init.md +37 -0
- package/agents/skills/canary:migrate.md +66 -0
- package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
- package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
- package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
- package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
- package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
- package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
- package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
- package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
- package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
- package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
- package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
- package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
- package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
- package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
- package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
- package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
- package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
- package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
- package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
- package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
- package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
- package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
- package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
- package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
- package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
- package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
- package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
- package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
- package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
- package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
- package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
- package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
- package/agents/skills/lib/parse-args.mjs +214 -0
- package/dist/engine/analysis/cli.js +116 -54
- package/dist/engine/analysis/engine.js +34 -16
- package/dist/engine/analysis/reports.js +5 -4
- package/dist/engine/cli-commands.js +252 -106
- package/dist/engine/cli-common.js +15 -24
- package/dist/engine/cli.core.js +37 -11
- package/dist/engine/cli.js +2 -2
- package/dist/engine/company-knowledge-cli.js +2 -2
- package/dist/engine/core/adoption.js +408 -0
- package/dist/engine/core/framework-probes.js +7 -7
- package/dist/engine/core/fs-glob.js +2 -2
- package/dist/engine/core/gate-result.js +17 -0
- package/dist/engine/core/migrator.js +9 -17
- package/dist/engine/core/pattern-matcher.js +23 -5
- package/dist/engine/core/persona.js +421 -0
- package/dist/engine/core/promotion-verdict.js +261 -0
- package/dist/engine/core/quality-scorer.js +15 -2
- package/dist/engine/core/reporter.js +1 -9
- package/dist/engine/core/skill-dispatch.js +115 -0
- package/dist/engine/core/skill-examples.js +392 -0
- package/dist/engine/core/skill-registry.js +59 -4
- package/dist/engine/core/skill-surfaces.js +307 -0
- package/dist/engine/core/static-linter.js +310 -38
- package/dist/engine/core/test-files.js +77 -0
- package/dist/engine/core/ticket-updater.js +1 -7
- package/dist/engine/core/vacuity-scanner.js +726 -0
- package/dist/engine/core/workflow-discovery.js +2 -8
- package/dist/engine/core/workspace-detect.js +7 -6
- package/dist/engine/data/personas/registry.json +36 -0
- package/dist/engine/guardian/adjudication.js +5 -5
- package/dist/engine/guardian/analysis-emit.js +19 -28
- package/dist/engine/guardian/cli.js +150 -51
- package/dist/engine/guardian/coverage.js +1 -1
- package/dist/engine/guardian/diff-coverage/heuristic-tier.js +1 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +2 -2
- package/dist/engine/guardian/pr-check.js +322 -33
- package/dist/engine/guardian/pr-comment.js +4 -3
- package/dist/engine/history/cli.js +210 -6
- package/dist/engine/history/ndjson-store.js +9 -5
- package/dist/engine/history/record.js +34 -5
- package/dist/engine/history/run-recorder.js +165 -0
- package/dist/engine/history/schema.js +25 -7
- package/dist/engine/history/store.js +9 -0
- package/dist/engine/mcp-server.js +35 -13
- package/dist/engine/skills-cli.js +249 -23
- package/dist/engine/util/ensure-ascii.js +37 -0
- package/dist/engine/workflow-cli.js +6 -6
- package/dist/gate-result.d.ts +11 -0
- package/dist/gate-result.js +18 -0
- package/dist/reporters/testtracker.d.ts +1 -1
- package/dist/reporters/testtracker.js +1 -1
- package/dist/uninstall.js +12 -5
- package/package.json +3 -2
|
@@ -37,7 +37,7 @@ const TEST_FN_PY = /^([ \t]*)def (test_\w+)\s*\(/gm;
|
|
|
37
37
|
// been blanked away; blanking is length-preserving precisely so these offsets
|
|
38
38
|
// still address the untouched text (#590).
|
|
39
39
|
const TEST_FN_JS = /(?:^|\s)(?:it|test)\s*\(\s*['"]([^'"]*)['"]/dgm;
|
|
40
|
-
const ASSERT_PY = /\bassert\b|\bpytest\.raises\b/;
|
|
40
|
+
export const ASSERT_PY = /\bassert\b|\bpytest\.raises\b/;
|
|
41
41
|
// Assertion styles a JS/TS test may use. `expect()` (jest/vitest/playwright)
|
|
42
42
|
// was the only one recognized until canary was pointed at its own suites and
|
|
43
43
|
// reported 216 assertion-free tests of which 13 were real -- the other 200 were
|
|
@@ -49,9 +49,42 @@ const ASSERT_PY = /\bassert\b|\bpytest\.raises\b/;
|
|
|
49
49
|
// own 16 residual findings. A regex linter cannot follow the call, so the NAME
|
|
50
50
|
// carries the signal; the `[A-Z]` keeps it to the convention rather than
|
|
51
51
|
// excusing any call that merely starts with those letters.
|
|
52
|
-
const ASSERT_JS = /\bexpect\s*\(|\bto(?:Be|Equal|Contain|Have|Match|Throw|Raise)\b|\bassert\s*\.\s*\w+\s*\(|\bassert\s*\(|\bshould\s*\.|\.should\b|\b(?:expect|assert)[A-Z]\w*\s*\(/;
|
|
52
|
+
export const ASSERT_JS = /\bexpect\s*\(|\bto(?:Be|Equal|Contain|Have|Match|Throw|Raise)\b|\bassert\s*\.\s*\w+\s*\(|\bassert\s*\(|\bshould\s*\.|\.should\b|\b(?:expect|assert)[A-Z]\w*\s*\(/;
|
|
53
53
|
// Strippers
|
|
54
|
-
|
|
54
|
+
/**
|
|
55
|
+
* A single-line quoted string literal.
|
|
56
|
+
*
|
|
57
|
+
* Written as one unambiguous alternation per quote style rather than the
|
|
58
|
+
* backreferenced `/(['"])(?:\\.|(?!\1).)*?\1/`, because that shape is
|
|
59
|
+
* CATASTROPHICALLY BACKTRACKING: `\\.` and `(?!\1).` both match a backslash, so
|
|
60
|
+
* every backslash doubles the parses the engine must try, and an UNTERMINATED
|
|
61
|
+
* literal makes it try all of them. Measured on this file's own scanners: a line
|
|
62
|
+
* `const p = 'C:` followed by 42 backslashes and no closing quote took **5.9
|
|
63
|
+
* seconds**, at 36 backslashes 142ms, and the growth is exponential -- ~55 is
|
|
64
|
+
* minutes. A linter that hangs on a Windows-path fixture is not advisory, it is
|
|
65
|
+
* down, and the input reaches here routinely because `blankMultilineStrings`
|
|
66
|
+
* blanks only the lines BETWEEN delimiters, so a multi-line literal's opening
|
|
67
|
+
* line always arrives intact.
|
|
68
|
+
*
|
|
69
|
+
* Each branch is `[^quote\\\n]` or `\\[\s\S]` -- disjoint by construction, so
|
|
70
|
+
* there is exactly one way to parse any input and the match is linear. `\n` is
|
|
71
|
+
* excluded so an unterminated literal cannot swallow the following lines.
|
|
72
|
+
*/
|
|
73
|
+
const STRING_LITERAL = /'(?:[^'\\\n]|\\[\s\S])*'|"(?:[^"\\\n]|\\[\s\S])*"/g;
|
|
74
|
+
/**
|
|
75
|
+
* A backtick template literal that opens and closes on ONE line.
|
|
76
|
+
*
|
|
77
|
+
* `blankStrings` knew only `'` and `"`, so a single-line template literal was
|
|
78
|
+
* read as live code by every rule. In a TS suite that is the common case for
|
|
79
|
+
* fixture data -- pointing the soundness rules at canary's own tests reported 6
|
|
80
|
+
* findings inside the fixtures of the file testing those very rules. The
|
|
81
|
+
* multi-line stripper below never covered it either: it only blanks the
|
|
82
|
+
* INTERIOR of a run whose delimiter count is odd, so a balanced one-line
|
|
83
|
+
* literal falls through both.
|
|
84
|
+
*/
|
|
85
|
+
const TEMPLATE_LITERAL = /`(?:[^`\\]|\\[\s\S])*`/g;
|
|
86
|
+
/** A `${...}` substitution: string syntax wrapping genuinely live code. */
|
|
87
|
+
const TEMPLATE_SUBSTITUTION = /\$\{[^{}]*\}/g;
|
|
55
88
|
// Magic numbers -- scoped to TIMING values only.
|
|
56
89
|
//
|
|
57
90
|
// "Extract the magic number to a named constant" is a production-code
|
|
@@ -103,6 +136,21 @@ function isComment(line) {
|
|
|
103
136
|
const s = line.trim();
|
|
104
137
|
return s.startsWith('#') || s.startsWith('//') || s.startsWith('*');
|
|
105
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* Drop a TRAILING comment from a line whose strings are already blanked.
|
|
141
|
+
*
|
|
142
|
+
* `isComment` only recognises a line that is entirely a comment, so a trailing
|
|
143
|
+
* `// seeded from process.pid earlier` was read as live code -- and because
|
|
144
|
+
* SOUND-001 gates promotion, a comment mentioning `process.pid` or `Date.now()`
|
|
145
|
+
* was enough to block a correct test. Safe to apply after `blankStrings`,
|
|
146
|
+
* because a `//` inside a string literal is already gone by then.
|
|
147
|
+
*
|
|
148
|
+
* Applied only by `scanSoundness`: the older rules were written against
|
|
149
|
+
* comment-bearing lines and changing what they see is not this change's business.
|
|
150
|
+
*/
|
|
151
|
+
function stripComments(line) {
|
|
152
|
+
return line.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$|#.*$/, '');
|
|
153
|
+
}
|
|
106
154
|
function mk(file, line, rule, severity, message, suggestion) {
|
|
107
155
|
return { file, line, rule, severity, message, suggestion };
|
|
108
156
|
}
|
|
@@ -156,7 +204,19 @@ const FLAKINESS_RULES = [
|
|
|
156
204
|
* would delete those rules outright.
|
|
157
205
|
*/
|
|
158
206
|
function blankStrings(line) {
|
|
159
|
-
|
|
207
|
+
// Template literals first: their interior can contain `'`/`"` that would
|
|
208
|
+
// otherwise pair up across the boundary and blank the wrong span.
|
|
209
|
+
//
|
|
210
|
+
// `${...}` substitutions are preserved, because they are the one part of a
|
|
211
|
+
// template literal that IS executed -- blanking them would take a real
|
|
212
|
+
// `Date.now()` dark, which is the abstention shape rather than a false
|
|
213
|
+
// positive. So the literal is replaced by the concatenation of its
|
|
214
|
+
// substitutions, and the quoted text around them disappears.
|
|
215
|
+
const detemplated = line.replace(TEMPLATE_LITERAL, (lit) => {
|
|
216
|
+
const subs = lit.match(TEMPLATE_SUBSTITUTION) ?? [];
|
|
217
|
+
return subs.length === 0 ? '""' : `""+${subs.join('+')}+""`;
|
|
218
|
+
});
|
|
219
|
+
return detemplated.replace(STRING_LITERAL, '""');
|
|
160
220
|
}
|
|
161
221
|
function scanFlakiness(lines, file) {
|
|
162
222
|
const out = [];
|
|
@@ -266,6 +326,192 @@ function scanMagicNumbers(lines, file) {
|
|
|
266
326
|
});
|
|
267
327
|
return out;
|
|
268
328
|
}
|
|
329
|
+
// --- Soundness (#605) -------------------------------------------------------
|
|
330
|
+
//
|
|
331
|
+
// A test can assert, pass, and still prove nothing about the implementation --
|
|
332
|
+
// because the value it pins is one no correct implementation is OBLIGED to
|
|
333
|
+
// produce. A generated test that pins a UUID, a pid, a temp-dir path, or a
|
|
334
|
+
// float it compares with exact equality goes green on the machine that made it
|
|
335
|
+
// and is a scheduled failure everywhere else. Nothing in this linter saw that
|
|
336
|
+
// class: the assertion exists, so LINT-006 is satisfied, and the assertion line
|
|
337
|
+
// itself is often clean, so FLAKE-003/004 never fire.
|
|
338
|
+
//
|
|
339
|
+
// The three rules below all key off the EXPECTATION position rather than the
|
|
340
|
+
// line, which is what keeps them actionable. A random temp dir used as test
|
|
341
|
+
// INPUT is fine and extremely common; the same value used as the expected
|
|
342
|
+
// RESULT is the defect. A rule that could not tell those apart would fire on
|
|
343
|
+
// every fixture-using test in the repo and be ignored within a week -- the
|
|
344
|
+
// LINT-005 outcome (0-for-157 actionable), re-run.
|
|
345
|
+
/**
|
|
346
|
+
* Non-deterministic sources FLAKE-003/004 do not already name. Kept disjoint
|
|
347
|
+
* from them so one defect never yields two findings the author must dismiss
|
|
348
|
+
* separately.
|
|
349
|
+
*/
|
|
350
|
+
const NONDET_EXTRA = /\b(?:crypto\.)?randomUUID\s*\(|\buuid4\s*\(|\bnanoid\s*\(|\bprocess\.pid\b|\bprocess\.hrtime\b|\bos\.getpid\s*\(|\bos\.hostname\s*\(|\bsocket\.gethostname\s*\(|\bperformance\.now\s*\(|\btime\.monotonic\s*\(|\bmkdtempSync\s*\(|\btempfile\.mkdtemp\s*\(|\bos\.tmpdir\s*\(/;
|
|
351
|
+
/**
|
|
352
|
+
* Sources whose value a test may legitimately pin as an EXPECTATION, and so the
|
|
353
|
+
* only ones worth tracking through a variable.
|
|
354
|
+
*
|
|
355
|
+
* Deliberately excludes the path family (`mkdtempSync`, `tmpdir`, `mkdtemp`).
|
|
356
|
+
* Measured against canary's own suite, taint through a temp directory was
|
|
357
|
+
* 0-for-3 actionable: a temp path is non-deterministic by design, and the
|
|
358
|
+
* assertion built from it (`expect(globDirs(tmp)).toEqual([join(tmp, 'apps')])`)
|
|
359
|
+
* is about the relationship between input and output, not about the path. The
|
|
360
|
+
* path family stays a direct-mode source, where `toBe(mkdtempSync(...))` --
|
|
361
|
+
* pinning a directory created in the assertion itself -- is still nonsense.
|
|
362
|
+
*/
|
|
363
|
+
const NONDET_TAINT_SOURCE = new RegExp(`${RANDOM.source}|${TIMESTAMP.source}|` +
|
|
364
|
+
`\\b(?:crypto\\.)?randomUUID\\s*\\(|\\buuid4\\s*\\(|\\bnanoid\\s*\\(|` +
|
|
365
|
+
`\\bprocess\\.pid\\b|\\bprocess\\.hrtime\\b|\\bos\\.getpid\\s*\\(|` +
|
|
366
|
+
`\\bos\\.hostname\\s*\\(|\\bsocket\\.gethostname\\s*\\(|` +
|
|
367
|
+
`\\bperformance\\.now\\s*\\(|\\btime\\.monotonic\\s*\\(`);
|
|
368
|
+
/**
|
|
369
|
+
* Where a JS/TS assertion's EXPECTED value begins.
|
|
370
|
+
*
|
|
371
|
+
* `toBeCloseTo` deliberately does not match: `\s*\(` must follow the matcher
|
|
372
|
+
* name, and `CloseTo(` does not. That single fact is what exempts the CORRECT
|
|
373
|
+
* float comparison from SOUND-002 and the pinned-fraction contract from
|
|
374
|
+
* SOUND-003, without either rule naming the fix.
|
|
375
|
+
*/
|
|
376
|
+
const JS_MATCHER_OPEN = /\.to(?:Be|Equal|StrictEqual|MatchObject|Contain|ContainEqual|HaveProperty|HaveLength|HaveValue)\s*\(/;
|
|
377
|
+
/** The Python analogue: `==` in an assert, or `assertEqual(`'s argument list. */
|
|
378
|
+
const PY_MATCHER_OPEN = /==|\bassertEqual\s*\(/;
|
|
379
|
+
/**
|
|
380
|
+
* A `const|let|var x = <rhs>` (JS) or `x = <rhs>` (Python) binding.
|
|
381
|
+
*
|
|
382
|
+
* The RHS stops at `;` rather than running to end-of-line, and both are `/g` so
|
|
383
|
+
* every binding on a line is seen. The old `(.*)$` meant one non-deterministic
|
|
384
|
+
* call anywhere later on the line tainted every name bound before it --
|
|
385
|
+
* `const expected = 3; const now = Date.now();` tainted `expected` -- and the
|
|
386
|
+
* "source" quoted back in the message was the rest of the statement, which is
|
|
387
|
+
* what gave the bug away.
|
|
388
|
+
*/
|
|
389
|
+
const JS_BINDING = /\b(?:const|let|var)\s+(\w+)\s*=\s*([^;\n]*)/g;
|
|
390
|
+
const PY_BINDING = /^[ \t]*(\w+)\s*=\s*([^=;\n][^;\n]*)/g;
|
|
391
|
+
const LEADING_FRACTION = /^\s*-?\d+\.\d+(?![\d.])/;
|
|
392
|
+
const LEADING_INTEGER = /^\s*-?\d+(?![\d.])/;
|
|
393
|
+
/** A division between two operands, e.g. `total / count`. */
|
|
394
|
+
const DIVISION = /[\w)\]]\s*\/\s*[\w(]/;
|
|
395
|
+
/**
|
|
396
|
+
* True iff a decimal literal has an exact binary-float representation.
|
|
397
|
+
*
|
|
398
|
+
* The discrimination that makes SOUND-002 usable rather than noise. `0.5` and
|
|
399
|
+
* `1.125` are exact, so `toBe(0.5)` states a contract an implementation can
|
|
400
|
+
* actually honour; `0.1` and `0.3` are not, so exact equality on them is a bet
|
|
401
|
+
* on the specific arithmetic path that produced the value. Flagging the exact
|
|
402
|
+
* ones too would train readers to dismiss the rule.
|
|
403
|
+
*
|
|
404
|
+
* `0.d1..dn` is `N / 10^n = N / (2^n * 5^n)`, so its reduced denominator is a
|
|
405
|
+
* power of two exactly when `5^n` divides `N`. Computed in BigInt because the
|
|
406
|
+
* float version was wrong twice over past 10 digits: `10 ** n` exceeds 2^31 so
|
|
407
|
+
* `den & (den - 1)` coerced through ToInt32 -- for a 32-digit fraction that is
|
|
408
|
+
* `0 & -1 === 0`, i.e. "exactly representable", and the rule went silent -- and
|
|
409
|
+
* `Number(frac)` loses precision past 15 digits, making the reduction
|
|
410
|
+
* meaningless before that. Both failures were in the permissive direction.
|
|
411
|
+
*/
|
|
412
|
+
function isBinaryExact(literal) {
|
|
413
|
+
const frac = literal.split('.')[1] ?? '';
|
|
414
|
+
if (frac === '')
|
|
415
|
+
return true;
|
|
416
|
+
return BigInt(frac) % 5n ** BigInt(frac.length) === 0n;
|
|
417
|
+
}
|
|
418
|
+
/** The expected-value text of an assertion line, or null if there is none. */
|
|
419
|
+
function expectationOf(line, python) {
|
|
420
|
+
const open = python ? PY_MATCHER_OPEN : JS_MATCHER_OPEN;
|
|
421
|
+
const m = open.exec(line);
|
|
422
|
+
if (m === null)
|
|
423
|
+
return null;
|
|
424
|
+
// To end-of-line rather than to a matching paren: a nested call in the
|
|
425
|
+
// expectation (`toBe(mkdtempSync(dir))`) has unbalanced parens under any
|
|
426
|
+
// regex, and the actual-value side is already excluded by starting after the
|
|
427
|
+
// matcher.
|
|
428
|
+
return line.slice(m.index + m[0].length);
|
|
429
|
+
}
|
|
430
|
+
/** The actual-value text of an assertion line (everything before the matcher). */
|
|
431
|
+
function actualOf(line, python) {
|
|
432
|
+
const open = python ? PY_MATCHER_OPEN : JS_MATCHER_OPEN;
|
|
433
|
+
const m = open.exec(line);
|
|
434
|
+
return m === null ? null : line.slice(0, m.index);
|
|
435
|
+
}
|
|
436
|
+
function scanSoundness(lines, file, python) {
|
|
437
|
+
const out = [];
|
|
438
|
+
// Variables bound to a non-deterministic source, name -> the source text.
|
|
439
|
+
//
|
|
440
|
+
// File-scoped, not test-scoped, and deliberately so: bounding taint by test
|
|
441
|
+
// body needs the same next-declaration machinery the assertion scanners use,
|
|
442
|
+
// and a name reused across tests in the same file with one tainted binding is
|
|
443
|
+
// a finding worth showing either way. Same trade blackhawk documents for
|
|
444
|
+
// file-wide frozen-clock suppression.
|
|
445
|
+
const tainted = new Map();
|
|
446
|
+
lines.forEach((raw, idx) => {
|
|
447
|
+
if (isComment(raw))
|
|
448
|
+
return;
|
|
449
|
+
// Data must never read as code: a fixture string carrying `Date.now()` is
|
|
450
|
+
// the linter's own test suite, not a defect in it. Comments are stripped for
|
|
451
|
+
// the same reason and it is not cosmetic -- `isComment` only skips a WHOLE
|
|
452
|
+
// comment line, so a trailing `// seeded from process.pid earlier` was read
|
|
453
|
+
// as an expectation and blocked the promotion of a correct test.
|
|
454
|
+
const line = stripComments(blankStrings(raw));
|
|
455
|
+
const lineNo = idx + 1;
|
|
456
|
+
// EVERY binding on the line, and a deterministic one CLEARS the taint.
|
|
457
|
+
//
|
|
458
|
+
// Two bugs lived in the single-`exec` version, both producing a false
|
|
459
|
+
// block. The RHS capture ran to end-of-line, so `const expected = 3; const
|
|
460
|
+
// now = Date.now();` tainted `expected` -- and the reported "source" was the
|
|
461
|
+
// rest of the statement, which is how it was noticed. And the map was
|
|
462
|
+
// write-only, so a name re-bound to a constant in a later test stayed
|
|
463
|
+
// tainted for the whole file.
|
|
464
|
+
for (const b of line.matchAll(python ? PY_BINDING : JS_BINDING)) {
|
|
465
|
+
const name = b[1];
|
|
466
|
+
const rhs = b[2].trim();
|
|
467
|
+
if (NONDET_TAINT_SOURCE.test(rhs))
|
|
468
|
+
tainted.set(name, rhs);
|
|
469
|
+
else
|
|
470
|
+
tainted.delete(name);
|
|
471
|
+
}
|
|
472
|
+
const isAssertion = (python ? ASSERT_PY : ASSERT_JS).test(line);
|
|
473
|
+
if (!isAssertion)
|
|
474
|
+
return;
|
|
475
|
+
const expected = expectationOf(line, python);
|
|
476
|
+
if (expected === null)
|
|
477
|
+
return;
|
|
478
|
+
// SOUND-001a: a non-deterministic source called straight into the
|
|
479
|
+
// expectation.
|
|
480
|
+
if (NONDET_EXTRA.test(expected)) {
|
|
481
|
+
out.push(mk(file, lineNo, 'SOUND-001', 'warning', 'Assertion pins a non-deterministic value.', 'Pin a fixture value, or assert the SHAPE (matcher/regex/type) instead of the exact value.'));
|
|
482
|
+
}
|
|
483
|
+
else {
|
|
484
|
+
// SOUND-001b: the mode a line-based rule cannot see -- the value became
|
|
485
|
+
// non-deterministic on an earlier line and arrives here through a
|
|
486
|
+
// variable, so this line reads as perfectly clean.
|
|
487
|
+
for (const [name, source] of tainted) {
|
|
488
|
+
// `[\w$]` lookarounds, not `\b`: `$` is legal in a JS identifier and is
|
|
489
|
+
// not a word character, so `\b$stamp\b` could only match after a word
|
|
490
|
+
// char and a `$`-prefixed name was invisible.
|
|
491
|
+
if (!new RegExp(`(?<![\\w$])${name}(?![\\w$])`).test(expected))
|
|
492
|
+
continue;
|
|
493
|
+
out.push(mk(file, lineNo, 'SOUND-001', 'warning', `Assertion pins \`${name}\`, which came from \`${source}\`.`, 'A value the implementation did not have to produce proves nothing. Freeze the source (fake timers / fixed seed) or assert the shape.'));
|
|
494
|
+
break; // one finding per assertion line
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
// SOUND-002: exact equality against a fraction with no exact binary form.
|
|
498
|
+
const frac = LEADING_FRACTION.exec(expected);
|
|
499
|
+
if (frac && !isBinaryExact(frac[0].trim())) {
|
|
500
|
+
out.push(mk(file, lineNo, 'SOUND-002', 'warning', `Exact equality against ${frac[0].trim()}, which has no exact binary representation.`, python
|
|
501
|
+
? 'Use pytest.approx() -- exact float equality depends on the arithmetic path, not the contract.'
|
|
502
|
+
: 'Use toBeCloseTo() -- exact float equality depends on the arithmetic path, not the contract.'));
|
|
503
|
+
}
|
|
504
|
+
// SOUND-003: a ratio pinned to an integer never states whether the
|
|
505
|
+
// operation truncates. This is the realworld S4 integer/fractional
|
|
506
|
+
// precondition, one layer down: the test exercises the input shape the
|
|
507
|
+
// author had in mind and leaves the other one unspecified.
|
|
508
|
+
const actual = actualOf(line, python);
|
|
509
|
+
if (actual && DIVISION.test(actual) && LEADING_INTEGER.test(expected)) {
|
|
510
|
+
out.push(mk(file, lineNo, 'SOUND-003', 'warning', 'A ratio is pinned to an integer, leaving the integer/fractional contract unpinned.', 'Add a case whose inputs divide to a fractional result, so the test says whether the operation truncates.'));
|
|
511
|
+
}
|
|
512
|
+
});
|
|
513
|
+
return out;
|
|
514
|
+
}
|
|
269
515
|
function lineOf(code, offset) {
|
|
270
516
|
let n = 1;
|
|
271
517
|
for (let i = 0; i < offset && i < code.length; i++) {
|
|
@@ -274,50 +520,75 @@ function lineOf(code, offset) {
|
|
|
274
520
|
}
|
|
275
521
|
return n;
|
|
276
522
|
}
|
|
277
|
-
|
|
523
|
+
/**
|
|
524
|
+
* Every test declaration in `code`, with its body bounded by the NEXT one.
|
|
525
|
+
*
|
|
526
|
+
* Extracted so the vacuity scanner (#612) shares one notion of "where does this
|
|
527
|
+
* test end" with LINT-006 rather than growing a second, subtly different one.
|
|
528
|
+
* The boundary logic is the part with a bug history — a fixed 2000-character
|
|
529
|
+
* lookahead was wrong in both directions (#590), the `(?:^|\s)` prefix put the
|
|
530
|
+
* reported line one early (#633) — so a divergent copy would inherit none of
|
|
531
|
+
* those fixes.
|
|
532
|
+
*
|
|
533
|
+
* `code` must be the string-blanked source (see `blankStringContent`) so a
|
|
534
|
+
* declaration inside a fixture cannot truncate a real test's body; `source` is
|
|
535
|
+
* the untouched text, read only to recover names that blanking erased.
|
|
536
|
+
*/
|
|
537
|
+
export function enumerateTests(code, source, python) {
|
|
278
538
|
const out = [];
|
|
279
|
-
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
out.push(
|
|
539
|
+
if (python) {
|
|
540
|
+
for (const m of code.matchAll(TEST_FN_PY)) {
|
|
541
|
+
const indent = m[1].length;
|
|
542
|
+
const start = m.index;
|
|
543
|
+
const bodyStart = start + m[0].length;
|
|
544
|
+
const rest = code.slice(bodyStart);
|
|
545
|
+
const nextFn = rest.match(new RegExp(`^[ \\t]{${indent}}def `, 'm'));
|
|
546
|
+
out.push({
|
|
547
|
+
name: m[2],
|
|
548
|
+
line: lineOf(code, start),
|
|
549
|
+
body: nextFn ? rest.slice(0, nextFn.index) : rest,
|
|
550
|
+
bodyStart,
|
|
551
|
+
});
|
|
287
552
|
}
|
|
553
|
+
return out;
|
|
288
554
|
}
|
|
289
|
-
return out;
|
|
290
|
-
}
|
|
291
|
-
function scanAssertionFreeJs(code, file, source) {
|
|
292
|
-
const out = [];
|
|
293
|
-
// The test declarations, in source order, so each body can be bounded by the
|
|
294
|
-
// NEXT one -- the JS analogue of what the pytest scanner already does with
|
|
295
|
-
// "next `def` at the same indent".
|
|
296
|
-
//
|
|
297
|
-
// This replaces a fixed 2000-character lookahead that was wrong in BOTH
|
|
298
|
-
// directions: a long test whose first assertion fell past the window was
|
|
299
|
-
// flagged (false positive), and a short empty test could borrow the next
|
|
300
|
-
// test's assertion from inside the window (false negative). Neither failure
|
|
301
|
-
// is visible without a real codebase to run it against, which is why
|
|
302
|
-
// dogfooding found them and the unit tests did not.
|
|
303
555
|
const decls = [...code.matchAll(TEST_FN_JS)];
|
|
304
556
|
for (let i = 0; i < decls.length; i += 1) {
|
|
305
557
|
const m = decls[i];
|
|
306
558
|
const start = m.index;
|
|
307
559
|
const bodyStart = start + m[0].length;
|
|
308
560
|
const bodyEnd = decls[i + 1]?.index ?? code.length;
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
561
|
+
// The coordinate comes from the NAME's offset, not the match's: TEST_FN_JS
|
|
562
|
+
// opens with `(?:^|\s)`, which CONSUMES the character before `it`/`test` --
|
|
563
|
+
// for any test not on line 1 that is the previous line's newline (#633).
|
|
564
|
+
const nameStart = m.indices?.[1]?.[0] ?? start;
|
|
565
|
+
out.push({
|
|
566
|
+
name: m.indices?.[1]
|
|
312
567
|
? source.slice(m.indices[1][0], m.indices[1][1])
|
|
313
|
-
: m[1]
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
568
|
+
: m[1],
|
|
569
|
+
line: lineOf(code, nameStart),
|
|
570
|
+
body: code.slice(bodyStart, bodyEnd),
|
|
571
|
+
bodyStart,
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
return out;
|
|
575
|
+
}
|
|
576
|
+
function scanAssertionFreePy(code, file) {
|
|
577
|
+
const out = [];
|
|
578
|
+
for (const t of enumerateTests(code, code, true)) {
|
|
579
|
+
if (!ASSERT_PY.test(t.body)) {
|
|
580
|
+
out.push(mk(file, t.line, 'LINT-006', 'warning', `\`${t.name}\` contains no assertions.`, 'Add at least one assert statement; a test that never fails proves nothing.'));
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
return out;
|
|
584
|
+
}
|
|
585
|
+
function scanAssertionFreeJs(code, file, source) {
|
|
586
|
+
const out = [];
|
|
587
|
+
// Bodies are bounded by the NEXT declaration -- see `enumerateTests`, which
|
|
588
|
+
// now owns that logic and its bug history (#590, #633) for both languages.
|
|
589
|
+
for (const t of enumerateTests(code, source, false)) {
|
|
590
|
+
if (!ASSERT_JS.test(t.body)) {
|
|
591
|
+
out.push(mk(file, t.line, 'LINT-006', 'warning', `Test "${t.name}" contains no assertions.`, 'Add an expect() call; a test that never asserts always passes.'));
|
|
321
592
|
}
|
|
322
593
|
}
|
|
323
594
|
return out;
|
|
@@ -392,6 +663,7 @@ export class StaticLinter {
|
|
|
392
663
|
...scanSelectors(lines, path),
|
|
393
664
|
...scanMissingAwait(lines, path),
|
|
394
665
|
...scanMagicNumbers(lines, path),
|
|
666
|
+
...scanSoundness(lines, path, fw === 'pytest'),
|
|
395
667
|
...(fw === 'pytest'
|
|
396
668
|
? scanAssertionFreePy(scanned, path)
|
|
397
669
|
: scanAssertionFreeJs(scanned, path, code)),
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one answer to "which files in this tree are tests?" (#755).
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `cli-commands.ts`, where it was private, because
|
|
5
|
+
* `canary-cassandra`'s skill CLI needs the SAME answer as `canary
|
|
6
|
+
* vacuity-check`. The four Tier-0 detectors are meant to be mergeable by a
|
|
7
|
+
* single consumer, and two collectors disagreeing about the denominator is the
|
|
8
|
+
* quietest way for that to stop being true: the same run would report a
|
|
9
|
+
* different `checked` depending on which door it came through.
|
|
10
|
+
*
|
|
11
|
+
* The walk's ignore set is load-bearing (#566): a dependency's own test suite
|
|
12
|
+
* is not the consumer's to fix. One downstream run before that fix produced 254
|
|
13
|
+
* of 256 findings inside `node_modules`, with the only `critical` in vendored
|
|
14
|
+
* code.
|
|
15
|
+
*/
|
|
16
|
+
import { readdirSync, statSync } from 'node:fs';
|
|
17
|
+
import { basename, join } from 'node:path';
|
|
18
|
+
import { JS_TEST_EXTENSIONS } from './static-linter.js';
|
|
19
|
+
/** True when `p` is a readable directory. A missing path is not one. */
|
|
20
|
+
export function isDir(p) {
|
|
21
|
+
try {
|
|
22
|
+
return statSync(p).isDirectory();
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** Directories never worth walking; see the module docstring for why. */
|
|
29
|
+
const IGNORED_DIRS = new Set([
|
|
30
|
+
'node_modules',
|
|
31
|
+
'.git',
|
|
32
|
+
'__pycache__',
|
|
33
|
+
'.venv',
|
|
34
|
+
'venv',
|
|
35
|
+
'dist',
|
|
36
|
+
'build',
|
|
37
|
+
'.next',
|
|
38
|
+
'.nuxt',
|
|
39
|
+
]);
|
|
40
|
+
function walkFiles(dir) {
|
|
41
|
+
const out = [];
|
|
42
|
+
let entries;
|
|
43
|
+
try {
|
|
44
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
for (const e of entries) {
|
|
50
|
+
const full = join(dir, e.name);
|
|
51
|
+
if (e.isDirectory()) {
|
|
52
|
+
if (!IGNORED_DIRS.has(e.name))
|
|
53
|
+
out.push(...walkFiles(full));
|
|
54
|
+
}
|
|
55
|
+
else if (e.isFile())
|
|
56
|
+
out.push(full);
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* `test_*.py` plus `*.test.*` / `*.spec.*` over every extension the scanners
|
|
62
|
+
* can actually read -- `.mjs` and `.cjs` included, which is the half of #566
|
|
63
|
+
* that made a directory of ESM tests collect zero files.
|
|
64
|
+
*/
|
|
65
|
+
const JS_TEST_FILE_RE = new RegExp(`\\.(test|spec)\\.(${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join('|')})$`);
|
|
66
|
+
/** Recursive test-file glob matching Python's `rglob` union, sorted by path. */
|
|
67
|
+
export function collectTestFiles(dir) {
|
|
68
|
+
return walkFiles(dir)
|
|
69
|
+
.filter((p) => {
|
|
70
|
+
const b = basename(p);
|
|
71
|
+
return ((b.startsWith('test_') && b.endsWith('.py')) || JS_TEST_FILE_RE.test(b));
|
|
72
|
+
})
|
|
73
|
+
.sort();
|
|
74
|
+
}
|
|
75
|
+
/** Human-readable list of what {@link collectTestFiles} looks for. */
|
|
76
|
+
export const SCANNABLE_DESC = `test_*.py, *.test|spec.{${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join(',')}}`;
|
|
77
|
+
//# sourceMappingURL=test-files.js.map
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
* int/float distinction). Non-integral values match.
|
|
31
31
|
*/
|
|
32
32
|
import { atlassianUrlFor, CommandNotFoundError, defaultHttpClient, defaultSubprocess, resolveRole, SubprocessTimeoutError, } from './workflow-discovery.js';
|
|
33
|
+
import { ensureAscii } from '../util/ensure-ascii.js';
|
|
33
34
|
// ---------------------------------------------------------------------------
|
|
34
35
|
// Python-compatibility helpers (copied locally per-module, matching reporter.ts)
|
|
35
36
|
// ---------------------------------------------------------------------------
|
|
@@ -52,13 +53,6 @@ function pyTruthy(value) {
|
|
|
52
53
|
function pyGet(obj, key, fallback) {
|
|
53
54
|
return Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : fallback;
|
|
54
55
|
}
|
|
55
|
-
/**
|
|
56
|
-
* Reproduce Python's `json.dumps(..., ensure_ascii=True)` (the library default)
|
|
57
|
-
* on `JSON.stringify` output: escape every code point >= 0x80 as `\uXXXX`.
|
|
58
|
-
*/
|
|
59
|
-
function ensureAscii(json) {
|
|
60
|
-
return json.replace(/[\u0080-\uffff]/g, (ch) => '\\u' + ch.charCodeAt(0).toString(16).padStart(4, '0'));
|
|
61
|
-
}
|
|
62
56
|
/**
|
|
63
57
|
* Python `json.dumps(obj)` (no indent) with library-default separators
|
|
64
58
|
* `(', ', ': ')` -- a space after every ',' and ':'. JS `JSON.stringify` emits
|