canary-test-cli 6.5.0 → 6.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/canary.js +69 -1
- package/dist/doctor-manifest.js +6 -1
- package/dist/doctor.js +7 -4
- package/dist/engine/cli-commands.js +84 -25
- package/dist/engine/cli.core.js +1 -1
- package/dist/engine/core/framework-probes.js +218 -0
- package/dist/engine/core/framework-registry.js +4 -1
- package/dist/engine/core/fs-glob.js +185 -0
- package/dist/engine/core/gate-result.js +27 -4
- package/dist/engine/core/migrator.js +240 -289
- package/dist/engine/core/scaffold-templates.js +123 -0
- package/dist/engine/core/scaffolder.js +4 -105
- package/dist/engine/core/static-linter.js +169 -17
- package/dist/engine/core/workspace-detect.js +0 -0
- package/dist/engine/guardian/adjudication.js +34 -30
- package/dist/engine/guardian/analysis-emit.js +13 -4
- package/dist/engine/guardian/cli.js +113 -16
- package/dist/engine/guardian/coverage.js +291 -9
- package/dist/engine/guardian/github-paging.js +97 -0
- package/dist/engine/guardian/pr-check.js +285 -26
- package/dist/engine/guardian/pr-comment.js +29 -15
- package/dist/engine/history/async-store.js +20 -0
- package/dist/gate-result.js +27 -4
- package/dist/overlay-commands.js +31 -3
- package/package.json +2 -2
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scaffold templates - the config-file bodies canary writes for each supported
|
|
3
|
+
* framework, and the derived set of frameworks it can scaffold.
|
|
4
|
+
*
|
|
5
|
+
* Extracted from `scaffolder.ts` to break a circular dependency (#543):
|
|
6
|
+
* `framework-registry` needs {@link scaffoldableFrameworks} to answer the
|
|
7
|
+
* scaffold capability question, while `scaffolder` needs the registry to
|
|
8
|
+
* degrade loudly on an unknown framework. With the data here, both depend on a
|
|
9
|
+
* leaf module and neither depends on the other. `scaffolder.ts` re-exports
|
|
10
|
+
* both names, so existing importers (the migrator, the parity suite) are
|
|
11
|
+
* unaffected.
|
|
12
|
+
*
|
|
13
|
+
* The template strings remain a **byte-exact contract** with the Python
|
|
14
|
+
* originals - see the note in `scaffolder.ts`. They were moved by line slice,
|
|
15
|
+
* not retyped, and `core-parity.test.ts` compares TEMPLATES against the golden
|
|
16
|
+
* fixture.
|
|
17
|
+
*/
|
|
18
|
+
// Exported so the migrator port can compute would-create / already-present sets
|
|
19
|
+
// in its dry-run path (Python: `from agent.core.scaffolder import TEMPLATES`).
|
|
20
|
+
export const TEMPLATES = {
|
|
21
|
+
playwright: {
|
|
22
|
+
files: {
|
|
23
|
+
'playwright.config.ts': `import { defineConfig, devices } from '@playwright/test';
|
|
24
|
+
|
|
25
|
+
export default defineConfig({
|
|
26
|
+
testDir: './tests/e2e',
|
|
27
|
+
fullyParallel: true,
|
|
28
|
+
forbidOnly: !!process.env.CI,
|
|
29
|
+
retries: process.env.CI ? 2 : 0,
|
|
30
|
+
workers: process.env.CI ? 1 : undefined,
|
|
31
|
+
reporter: 'html',
|
|
32
|
+
use: {
|
|
33
|
+
trace: 'on-first-retry',
|
|
34
|
+
},
|
|
35
|
+
projects: [
|
|
36
|
+
{
|
|
37
|
+
name: 'chromium',
|
|
38
|
+
use: { ...devices['Desktop Chrome'] },
|
|
39
|
+
},
|
|
40
|
+
],
|
|
41
|
+
});
|
|
42
|
+
`,
|
|
43
|
+
},
|
|
44
|
+
dirs: ['tests/e2e'],
|
|
45
|
+
},
|
|
46
|
+
vitest: {
|
|
47
|
+
files: {
|
|
48
|
+
'vitest.config.ts': `import { defineConfig } from 'vitest/config';
|
|
49
|
+
|
|
50
|
+
export default defineConfig({
|
|
51
|
+
test: {
|
|
52
|
+
environment: 'node',
|
|
53
|
+
include: ['tests/unit/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
`,
|
|
57
|
+
},
|
|
58
|
+
dirs: ['tests/unit'],
|
|
59
|
+
},
|
|
60
|
+
pytest: {
|
|
61
|
+
files: {
|
|
62
|
+
'pytest.ini': `[pytest]
|
|
63
|
+
testpaths = tests
|
|
64
|
+
python_files = test_*.py *_test.py
|
|
65
|
+
python_classes = Test*
|
|
66
|
+
python_functions = test_*
|
|
67
|
+
`,
|
|
68
|
+
},
|
|
69
|
+
dirs: ['tests'],
|
|
70
|
+
},
|
|
71
|
+
k6: {
|
|
72
|
+
files: {
|
|
73
|
+
'k6.config.js': `export const options = {
|
|
74
|
+
vus: 10,
|
|
75
|
+
duration: '30s',
|
|
76
|
+
};
|
|
77
|
+
`,
|
|
78
|
+
},
|
|
79
|
+
dirs: ['tests/performance'],
|
|
80
|
+
},
|
|
81
|
+
wdio: {
|
|
82
|
+
files: {
|
|
83
|
+
'wdio.conf.ts': `import type { Options } from "@wdio/types";
|
|
84
|
+
|
|
85
|
+
// Appium + WebdriverIO config. Fill in the capabilities stub below for the
|
|
86
|
+
// device/platform under test (Android shown; add an iOS entry as needed).
|
|
87
|
+
export const config: Options.Testrunner = {
|
|
88
|
+
runner: "local",
|
|
89
|
+
specs: ["./tests/**/*.spec.ts"],
|
|
90
|
+
maxInstances: 1,
|
|
91
|
+
// Appium capabilities stub \u{2014} replace deviceName / app / versions to match
|
|
92
|
+
// your emulator or real device.
|
|
93
|
+
capabilities: [
|
|
94
|
+
{
|
|
95
|
+
platformName: "Android",
|
|
96
|
+
"appium:automationName": "UiAutomator2",
|
|
97
|
+
"appium:deviceName": "Android Emulator",
|
|
98
|
+
"appium:app": "./app/build/outputs/apk/debug/app-debug.apk",
|
|
99
|
+
},
|
|
100
|
+
],
|
|
101
|
+
framework: "mocha",
|
|
102
|
+
mochaOpts: {
|
|
103
|
+
ui: "bdd",
|
|
104
|
+
timeout: 60000,
|
|
105
|
+
},
|
|
106
|
+
reporters: ["spec"],
|
|
107
|
+
// Requires the Appium service: \`npm i -D @wdio/appium-service appium\`.
|
|
108
|
+
services: ["appium"],
|
|
109
|
+
};
|
|
110
|
+
`,
|
|
111
|
+
},
|
|
112
|
+
dirs: ['tests'],
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
/**
|
|
116
|
+
* Frameworks canary can scaffold - the single source of truth for the
|
|
117
|
+
* `scaffold` capability, derived from the templates that actually exist
|
|
118
|
+
* (Python: `scaffoldable_frameworks`).
|
|
119
|
+
*/
|
|
120
|
+
export function scaffoldableFrameworks() {
|
|
121
|
+
return new Set(Object.keys(TEMPLATES));
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=scaffold-templates.js.map
|
|
@@ -23,6 +23,10 @@
|
|
|
23
23
|
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
24
24
|
import { join, resolve } from 'node:path';
|
|
25
25
|
import { FrameworkRegistry } from './framework-registry.js';
|
|
26
|
+
import { TEMPLATES } from './scaffold-templates.js';
|
|
27
|
+
// Re-exported so existing importers keep one entry point for the scaffold
|
|
28
|
+
// surface (Python: `from agent.core.scaffolder import TEMPLATES`).
|
|
29
|
+
export { TEMPLATES, scaffoldableFrameworks } from './scaffold-templates.js';
|
|
26
30
|
// ---------------------------------------------------------------------------
|
|
27
31
|
// Python-compatibility helper (copied locally per-module, matching reporter.ts)
|
|
28
32
|
// ---------------------------------------------------------------------------
|
|
@@ -41,103 +45,6 @@ function pyTruthy(value) {
|
|
|
41
45
|
return Object.keys(value).length > 0;
|
|
42
46
|
return Boolean(value);
|
|
43
47
|
}
|
|
44
|
-
// Exported so the migrator port can compute would-create / already-present sets
|
|
45
|
-
// in its dry-run path (Python: `from agent.core.scaffolder import TEMPLATES`).
|
|
46
|
-
export const TEMPLATES = {
|
|
47
|
-
playwright: {
|
|
48
|
-
files: {
|
|
49
|
-
'playwright.config.ts': `import { defineConfig, devices } from '@playwright/test';
|
|
50
|
-
|
|
51
|
-
export default defineConfig({
|
|
52
|
-
testDir: './tests/e2e',
|
|
53
|
-
fullyParallel: true,
|
|
54
|
-
forbidOnly: !!process.env.CI,
|
|
55
|
-
retries: process.env.CI ? 2 : 0,
|
|
56
|
-
workers: process.env.CI ? 1 : undefined,
|
|
57
|
-
reporter: 'html',
|
|
58
|
-
use: {
|
|
59
|
-
trace: 'on-first-retry',
|
|
60
|
-
},
|
|
61
|
-
projects: [
|
|
62
|
-
{
|
|
63
|
-
name: 'chromium',
|
|
64
|
-
use: { ...devices['Desktop Chrome'] },
|
|
65
|
-
},
|
|
66
|
-
],
|
|
67
|
-
});
|
|
68
|
-
`,
|
|
69
|
-
},
|
|
70
|
-
dirs: ['tests/e2e'],
|
|
71
|
-
},
|
|
72
|
-
vitest: {
|
|
73
|
-
files: {
|
|
74
|
-
'vitest.config.ts': `import { defineConfig } from 'vitest/config';
|
|
75
|
-
|
|
76
|
-
export default defineConfig({
|
|
77
|
-
test: {
|
|
78
|
-
environment: 'node',
|
|
79
|
-
include: ['tests/unit/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
|
|
80
|
-
},
|
|
81
|
-
});
|
|
82
|
-
`,
|
|
83
|
-
},
|
|
84
|
-
dirs: ['tests/unit'],
|
|
85
|
-
},
|
|
86
|
-
pytest: {
|
|
87
|
-
files: {
|
|
88
|
-
'pytest.ini': `[pytest]
|
|
89
|
-
testpaths = tests
|
|
90
|
-
python_files = test_*.py *_test.py
|
|
91
|
-
python_classes = Test*
|
|
92
|
-
python_functions = test_*
|
|
93
|
-
`,
|
|
94
|
-
},
|
|
95
|
-
dirs: ['tests'],
|
|
96
|
-
},
|
|
97
|
-
k6: {
|
|
98
|
-
files: {
|
|
99
|
-
'k6.config.js': `export const options = {
|
|
100
|
-
vus: 10,
|
|
101
|
-
duration: '30s',
|
|
102
|
-
};
|
|
103
|
-
`,
|
|
104
|
-
},
|
|
105
|
-
dirs: ['tests/performance'],
|
|
106
|
-
},
|
|
107
|
-
wdio: {
|
|
108
|
-
files: {
|
|
109
|
-
'wdio.conf.ts': `import type { Options } from "@wdio/types";
|
|
110
|
-
|
|
111
|
-
// Appium + WebdriverIO config. Fill in the capabilities stub below for the
|
|
112
|
-
// device/platform under test (Android shown; add an iOS entry as needed).
|
|
113
|
-
export const config: Options.Testrunner = {
|
|
114
|
-
runner: "local",
|
|
115
|
-
specs: ["./tests/**/*.spec.ts"],
|
|
116
|
-
maxInstances: 1,
|
|
117
|
-
// Appium capabilities stub \u{2014} replace deviceName / app / versions to match
|
|
118
|
-
// your emulator or real device.
|
|
119
|
-
capabilities: [
|
|
120
|
-
{
|
|
121
|
-
platformName: "Android",
|
|
122
|
-
"appium:automationName": "UiAutomator2",
|
|
123
|
-
"appium:deviceName": "Android Emulator",
|
|
124
|
-
"appium:app": "./app/build/outputs/apk/debug/app-debug.apk",
|
|
125
|
-
},
|
|
126
|
-
],
|
|
127
|
-
framework: "mocha",
|
|
128
|
-
mochaOpts: {
|
|
129
|
-
ui: "bdd",
|
|
130
|
-
timeout: 60000,
|
|
131
|
-
},
|
|
132
|
-
reporters: ["spec"],
|
|
133
|
-
// Requires the Appium service: \`npm i -D @wdio/appium-service appium\`.
|
|
134
|
-
services: ["appium"],
|
|
135
|
-
};
|
|
136
|
-
`,
|
|
137
|
-
},
|
|
138
|
-
dirs: ['tests'],
|
|
139
|
-
},
|
|
140
|
-
};
|
|
141
48
|
/**
|
|
142
49
|
* Handles initialization and scaffolding of test suites.
|
|
143
50
|
*
|
|
@@ -225,12 +132,4 @@ export class Scaffolder {
|
|
|
225
132
|
};
|
|
226
133
|
}
|
|
227
134
|
}
|
|
228
|
-
/**
|
|
229
|
-
* Frameworks canary can scaffold - the single source of truth for the
|
|
230
|
-
* `scaffold` capability, derived from the templates that actually exist
|
|
231
|
-
* (Python: `scaffoldable_frameworks`).
|
|
232
|
-
*/
|
|
233
|
-
export function scaffoldableFrameworks() {
|
|
234
|
-
return new Set(Object.keys(TEMPLATES));
|
|
235
|
-
}
|
|
236
135
|
//# sourceMappingURL=scaffolder.js.map
|
|
@@ -26,10 +26,36 @@ const BARE_PLAYWRIGHT_CALL = /(?<!await\s)(?<!return\s)(?<!\w)(?:page|frame|loca
|
|
|
26
26
|
const TEST_FN_PY = /^(\s*)def (test_\w+)\s*\(/gm;
|
|
27
27
|
const TEST_FN_JS = /(?:^|\s)(?:it|test)\s*\(\s*['"]([^'"]*)['"]/gm;
|
|
28
28
|
const ASSERT_PY = /\bassert\b|\bpytest\.raises\b/;
|
|
29
|
-
|
|
29
|
+
// Assertion styles a JS/TS test may use. `expect()` (jest/vitest/playwright)
|
|
30
|
+
// was the only one recognized until canary was pointed at its own suites and
|
|
31
|
+
// reported 216 assertion-free tests of which 13 were real -- the other 200 were
|
|
32
|
+
// `node:test` + `node:assert`, a whole framework the linter could not see.
|
|
33
|
+
// Kept as a union of shapes rather than an import-aware parse: a static linter
|
|
34
|
+
// that needs to resolve imports to judge one line is the wrong trade.
|
|
35
|
+
// The `expectX()/assertX()` alternative covers a test that delegates its
|
|
36
|
+
// assertion to a named helper (`expectAuthoringAllowed(res)`) -- 9 of canary's
|
|
37
|
+
// own 16 residual findings. A regex linter cannot follow the call, so the NAME
|
|
38
|
+
// carries the signal; the `[A-Z]` keeps it to the convention rather than
|
|
39
|
+
// excusing any call that merely starts with those letters.
|
|
40
|
+
const ASSERT_JS = /\bexpect\s*\(|\bto(?:Be|Equal|Contain|Have|Match|Throw|Raise)\b|\bassert\s*\.\s*\w+\s*\(|\bassert\s*\(|\bshould\s*\.|\.should\b|\b(?:expect|assert)[A-Z]\w*\s*\(/;
|
|
30
41
|
// Strippers
|
|
31
42
|
const STRING_LITERAL = /(['"])(?:\\.|(?!\1).)*?\1/g;
|
|
32
|
-
// Magic numbers
|
|
43
|
+
// Magic numbers -- scoped to TIMING values only.
|
|
44
|
+
//
|
|
45
|
+
// "Extract the magic number to a named constant" is a production-code
|
|
46
|
+
// principle, and it inverts in a test: the literal IS the specification.
|
|
47
|
+
// `expect(notes.length).toBe(2048)` states the contract that
|
|
48
|
+
// `expect(notes.length).toBe(MAX_NOTES)` hides behind a name the reader now has
|
|
49
|
+
// to go look up. Since this linter only ever reads TEST files, the unscoped
|
|
50
|
+
// rule was misapplied across its entire domain -- measured at 0-for-157
|
|
51
|
+
// actionable when canary was first pointed at its own suites.
|
|
52
|
+
//
|
|
53
|
+
// A timing value is the one case that survives: a bare `5000` in a
|
|
54
|
+
// setTimeout/retry/interval position is a duration whose units and intent are
|
|
55
|
+
// genuinely unclear, and naming it genuinely helps. (Hardcoded sleeps are
|
|
56
|
+
// separately flagged at CRITICAL by FLAKE-001/002; this is the softer signal
|
|
57
|
+
// for the non-sleep timing values those rules do not cover.)
|
|
58
|
+
const TIMING_CONTEXT = /\b(?:setTimeout|setInterval|waitForTimeout|sleep|delay|timeout|interval|retryDelay|retries|backoff|pollInterval|debounce|throttle)\b/i;
|
|
33
59
|
const NUMERIC_LITERAL = /(?<![\w.])-?\d+(?:\.\d+)?(?![\w.])/g;
|
|
34
60
|
const ALLOWED_NUMBERS = new Set(['0', '1', '2', '-1', '10', '100']);
|
|
35
61
|
const HTTP_STATUS = new Set([
|
|
@@ -99,11 +125,33 @@ const FLAKINESS_RULES = [
|
|
|
99
125
|
suggestion: 'Mock Date.now()/datetime.now() or use a fixed reference date.',
|
|
100
126
|
},
|
|
101
127
|
];
|
|
128
|
+
/**
|
|
129
|
+
* Blank single-line string literals so a rule matching CODE cannot fire on test
|
|
130
|
+
* DATA.
|
|
131
|
+
*
|
|
132
|
+
* `scanMagicNumbers` has always done this; the flakiness and missing-await
|
|
133
|
+
* rules never did, so `const src = 'const t = Date.now();'` -- a fixture string
|
|
134
|
+
* feeding a linter test -- was reported as a real timestamp dependency. Any
|
|
135
|
+
* suite that carries the patterns it tests as string data hits this, and
|
|
136
|
+
* canary's own linter tests are the worst case.
|
|
137
|
+
*
|
|
138
|
+
* The same defect shipped in `canary-blackhawk`'s pragma parser (#499) and was
|
|
139
|
+
* guarded in `canary-savant` (#495/#498): data must never act as code, nor as
|
|
140
|
+
* directive.
|
|
141
|
+
*
|
|
142
|
+
* NOT applied to the selector rules: LINT-001/002/003 match `'.btn'` / `'#id'`
|
|
143
|
+
* inside quotes by construction, because a selector IS a string. Stripping
|
|
144
|
+
* would delete those rules outright.
|
|
145
|
+
*/
|
|
146
|
+
function blankStrings(line) {
|
|
147
|
+
return line.replace(STRING_LITERAL, '""');
|
|
148
|
+
}
|
|
102
149
|
function scanFlakiness(lines, file) {
|
|
103
150
|
const out = [];
|
|
104
|
-
lines.forEach((
|
|
105
|
-
if (isComment(
|
|
151
|
+
lines.forEach((raw, idx) => {
|
|
152
|
+
if (isComment(raw))
|
|
106
153
|
return;
|
|
154
|
+
const line = blankStrings(raw);
|
|
107
155
|
for (const r of FLAKINESS_RULES) {
|
|
108
156
|
if (r.re.test(line) && (!r.guard || r.guard(line))) {
|
|
109
157
|
out.push(mk(file, idx + 1, r.rule, r.severity, r.message, r.suggestion));
|
|
@@ -137,25 +185,70 @@ function scanSelectors(lines, file) {
|
|
|
137
185
|
}
|
|
138
186
|
function scanMissingAwait(lines, file) {
|
|
139
187
|
const out = [];
|
|
140
|
-
lines.forEach((
|
|
141
|
-
if (isComment(
|
|
188
|
+
lines.forEach((raw, idx) => {
|
|
189
|
+
if (isComment(raw))
|
|
142
190
|
return;
|
|
191
|
+
// A `page.click(...)` inside a string is fixture data, not a missing await.
|
|
192
|
+
const line = blankStrings(raw);
|
|
143
193
|
if (BARE_PLAYWRIGHT_CALL.test(line) && !line.includes('await')) {
|
|
144
194
|
out.push(mk(file, idx + 1, 'LINT-004', 'critical', 'Playwright action called without await.', 'Add `await` before the call to ensure it completes before the next step.'));
|
|
145
195
|
}
|
|
146
196
|
});
|
|
147
197
|
return out;
|
|
148
198
|
}
|
|
199
|
+
/** Multi-line string delimiters: JS template literal, Python triple quotes. */
|
|
200
|
+
const MULTILINE_DELIMS = ['`', '"""', "'''"];
|
|
201
|
+
/**
|
|
202
|
+
* Blank the INTERIOR of multi-line strings, preserving line count and numbering.
|
|
203
|
+
*
|
|
204
|
+
* Every per-line rule (magic numbers, selectors, flakiness) sees one line at a
|
|
205
|
+
* time, so a line inside a multi-line template literal or a Python
|
|
206
|
+
* triple-quoted block reads as bare code. Canary's own diff fixtures are
|
|
207
|
+
* template literals, so `100644` -- a git file mode sitting in test DATA -- was
|
|
208
|
+
* reported as a magic number 30 times. Any consumer with a multi-line SQL,
|
|
209
|
+
* JSON, HTML, or diff fixture has the same defect.
|
|
210
|
+
*
|
|
211
|
+
* Deliberately conservative about an UNBALANCED delimiter (a stray backtick in
|
|
212
|
+
* a comment, say): blanking to end-of-file would silently disable these rules
|
|
213
|
+
* from that point down -- the abstention shape, one layer inside the linter. An
|
|
214
|
+
* unclosed run is therefore discarded rather than applied.
|
|
215
|
+
*/
|
|
216
|
+
function blankMultilineStrings(lines) {
|
|
217
|
+
const out = [...lines];
|
|
218
|
+
for (const delim of MULTILINE_DELIMS) {
|
|
219
|
+
let openAt = null;
|
|
220
|
+
for (let i = 0; i < out.length; i += 1) {
|
|
221
|
+
const hits = out[i].split(delim).length - 1;
|
|
222
|
+
// An even count opens and closes on the same line, which the single-line
|
|
223
|
+
// stripper already handles; only an odd count toggles the state.
|
|
224
|
+
if (hits === 0 || hits % 2 === 0)
|
|
225
|
+
continue;
|
|
226
|
+
if (openAt === null) {
|
|
227
|
+
openAt = i;
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
for (let j = openAt + 1; j < i; j += 1)
|
|
231
|
+
out[j] = '';
|
|
232
|
+
openAt = null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
149
238
|
function scanMagicNumbers(lines, file) {
|
|
150
239
|
const out = [];
|
|
151
240
|
lines.forEach((raw, idx) => {
|
|
152
241
|
if (isComment(raw))
|
|
153
242
|
return;
|
|
154
243
|
const scrubbed = raw.replace(STRING_LITERAL, '""');
|
|
244
|
+
// Only timing positions: everywhere else in a test file the literal is the
|
|
245
|
+
// specification, not a smell. See TIMING_CONTEXT above.
|
|
246
|
+
if (!TIMING_CONTEXT.test(scrubbed))
|
|
247
|
+
return;
|
|
155
248
|
for (const m of scrubbed.matchAll(NUMERIC_LITERAL)) {
|
|
156
249
|
if (isAllowedNumber(m[0]))
|
|
157
250
|
continue;
|
|
158
|
-
out.push(mk(file, idx + 1, 'LINT-005', 'info', `Magic
|
|
251
|
+
out.push(mk(file, idx + 1, 'LINT-005', 'info', `Magic timing value ${m[0]}.`, 'Name the duration (e.g. RETRY_DELAY_MS) so its units and intent are readable.'));
|
|
159
252
|
break; // one finding per line
|
|
160
253
|
}
|
|
161
254
|
});
|
|
@@ -185,40 +278,99 @@ function scanAssertionFreePy(code, file) {
|
|
|
185
278
|
}
|
|
186
279
|
function scanAssertionFreeJs(code, file) {
|
|
187
280
|
const out = [];
|
|
188
|
-
|
|
281
|
+
// The test declarations, in source order, so each body can be bounded by the
|
|
282
|
+
// NEXT one -- the JS analogue of what the pytest scanner already does with
|
|
283
|
+
// "next `def` at the same indent".
|
|
284
|
+
//
|
|
285
|
+
// This replaces a fixed 2000-character lookahead that was wrong in BOTH
|
|
286
|
+
// directions: a long test whose first assertion fell past the window was
|
|
287
|
+
// flagged (false positive), and a short empty test could borrow the next
|
|
288
|
+
// test's assertion from inside the window (false negative). Neither failure
|
|
289
|
+
// is visible without a real codebase to run it against, which is why
|
|
290
|
+
// dogfooding found them and the unit tests did not.
|
|
291
|
+
const decls = [...code.matchAll(TEST_FN_JS)];
|
|
292
|
+
for (let i = 0; i < decls.length; i += 1) {
|
|
293
|
+
const m = decls[i];
|
|
189
294
|
const start = m.index;
|
|
190
|
-
const
|
|
295
|
+
const bodyStart = start + m[0].length;
|
|
296
|
+
const bodyEnd = decls[i + 1]?.index ?? code.length;
|
|
297
|
+
const rest = code.slice(bodyStart, bodyEnd);
|
|
191
298
|
if (!ASSERT_JS.test(rest)) {
|
|
192
299
|
out.push(mk(file, lineOf(code, start), 'LINT-006', 'warning', `Test "${m[1]}" contains no assertions.`, 'Add an expect() call; a test that never asserts always passes.'));
|
|
193
300
|
}
|
|
194
301
|
}
|
|
195
302
|
return out;
|
|
196
303
|
}
|
|
197
|
-
|
|
304
|
+
/**
|
|
305
|
+
* Extensions whose contents the JS/TS scanners can actually read. ESM (`.mjs`)
|
|
306
|
+
* and CJS (`.cjs`) belong here as much as `.js` does -- omitting them is what
|
|
307
|
+
* made #566 possible.
|
|
308
|
+
*/
|
|
309
|
+
export const JS_TEST_EXTENSIONS = [
|
|
310
|
+
'.ts',
|
|
311
|
+
'.js',
|
|
312
|
+
'.mjs',
|
|
313
|
+
'.cjs',
|
|
314
|
+
'.mts',
|
|
315
|
+
'.cts',
|
|
316
|
+
];
|
|
317
|
+
const JS_EXT_SET = new Set(JS_TEST_EXTENSIONS);
|
|
318
|
+
/**
|
|
319
|
+
* The framework whose scanners can parse `path`, or `null` when no scanner
|
|
320
|
+
* understands the extension.
|
|
321
|
+
*
|
|
322
|
+
* This deliberately has no default. The previous `return 'pytest'` fallback
|
|
323
|
+
* meant an unrecognised extension was silently handed to the Python assertion
|
|
324
|
+
* scanners: over ESM JavaScript they match nothing, so a `.mjs` file with real
|
|
325
|
+
* defects linted to zero findings and rendered a green all-clear (#566). A
|
|
326
|
+
* guess that cannot be distinguished from a clean result is a false green;
|
|
327
|
+
* `null` forces the caller to abstain instead.
|
|
328
|
+
*/
|
|
329
|
+
export function lintableFramework(path) {
|
|
198
330
|
const suffix = extname(path).toLowerCase();
|
|
199
331
|
const name = basename(path).toLowerCase();
|
|
200
332
|
if (suffix === '.py')
|
|
201
333
|
return 'pytest';
|
|
202
334
|
if (name.includes('playwright'))
|
|
203
335
|
return 'playwright';
|
|
204
|
-
if (
|
|
336
|
+
if (JS_EXT_SET.has(suffix))
|
|
205
337
|
return 'vitest';
|
|
206
|
-
return
|
|
338
|
+
return null;
|
|
339
|
+
}
|
|
340
|
+
/** Thrown when a scanner is asked for a file no ruleset can parse. */
|
|
341
|
+
export class UnsupportedTestFileError extends Error {
|
|
342
|
+
path;
|
|
343
|
+
constructor(path) {
|
|
344
|
+
super(`No linter ruleset can parse ${extname(path) || basename(path)}`);
|
|
345
|
+
this.path = path;
|
|
346
|
+
this.name = 'UnsupportedTestFileError';
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
function requireFramework(path, framework) {
|
|
350
|
+
const fw = framework || lintableFramework(path);
|
|
351
|
+
if (fw === null)
|
|
352
|
+
throw new UnsupportedTestFileError(path);
|
|
353
|
+
return fw;
|
|
207
354
|
}
|
|
208
355
|
export class StaticLinter {
|
|
209
356
|
/** Full quality audit — all rules. */
|
|
210
357
|
lint(path, framework) {
|
|
211
358
|
const code = readFileSync(path, 'utf-8');
|
|
212
|
-
|
|
213
|
-
|
|
359
|
+
// No rule may read the interior of a multi-line string as code. Blanking
|
|
360
|
+
// preserves line count, so `scanned` re-joins to the same line numbers the
|
|
361
|
+
// per-line scanners report -- both halves must use it, or the assertion
|
|
362
|
+
// scanners go on mining `it(...)` declarations out of diff fixtures.
|
|
363
|
+
const lines = blankMultilineStrings(code.split('\n'));
|
|
364
|
+
const scanned = lines.join('\n');
|
|
365
|
+
const fw = requireFramework(path, framework);
|
|
214
366
|
const findings = [
|
|
215
367
|
...scanFlakiness(lines, path),
|
|
216
368
|
...scanSelectors(lines, path),
|
|
217
369
|
...scanMissingAwait(lines, path),
|
|
218
370
|
...scanMagicNumbers(lines, path),
|
|
219
371
|
...(fw === 'pytest'
|
|
220
|
-
? scanAssertionFreePy(
|
|
221
|
-
: scanAssertionFreeJs(
|
|
372
|
+
? scanAssertionFreePy(scanned, path)
|
|
373
|
+
: scanAssertionFreeJs(scanned, path)),
|
|
222
374
|
];
|
|
223
375
|
findings.sort((a, b) => a.line - b.line || cmp(a.rule, b.rule));
|
|
224
376
|
return findings;
|
|
@@ -226,7 +378,7 @@ export class StaticLinter {
|
|
|
226
378
|
/** Flakiness-only subset. */
|
|
227
379
|
flakeCheck(path) {
|
|
228
380
|
const code = readFileSync(path, 'utf-8');
|
|
229
|
-
const findings = scanFlakiness(code.split('\n'), path);
|
|
381
|
+
const findings = scanFlakiness(blankMultilineStrings(code.split('\n')), path);
|
|
230
382
|
findings.sort((a, b) => a.line - b.line);
|
|
231
383
|
return findings;
|
|
232
384
|
}
|
|
Binary file
|
|
@@ -33,14 +33,21 @@ import { mkdirSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync,
|
|
|
33
33
|
import { randomBytes } from 'node:crypto';
|
|
34
34
|
import { dirname, join } from 'node:path';
|
|
35
35
|
import { STICKY_MARKER, findSticky } from './pr-comment.js';
|
|
36
|
+
import { readAllPages, restPageReader } from './github-paging.js';
|
|
36
37
|
/** Schema tag for adjudication records (independent of the findings schema). */
|
|
37
38
|
export const ADJUDICATION_SCHEMA_VERSION = '1.0';
|
|
38
39
|
/**
|
|
39
40
|
* Record `source` + filename prefix. Deliberately namespaced UNDER the
|
|
40
|
-
* `canary-pr-guardian-` prefix
|
|
41
|
-
* `*.json` in `.harness/analyses
|
|
42
|
-
*
|
|
43
|
-
*
|
|
41
|
+
* `canary-pr-guardian-` prefix, because harness's `AnalysisArchive` reads every
|
|
42
|
+
* `*.json` in `.harness/analyses/`.
|
|
43
|
+
*
|
|
44
|
+
* The filenames are not provably distinct, though: a branch named
|
|
45
|
+
* `adjudication/pr-42` sanitizes through `analysisFilename` to exactly
|
|
46
|
+
* `canary-pr-guardian-adjudication-pr-42.json`, colliding with this prefix.
|
|
47
|
+
* What actually keeps the precision summary honest is the `source` field —
|
|
48
|
+
* `loadAdjudicationRecords` requires `source === ADJUDICATION_SOURCE` plus
|
|
49
|
+
* numeric `tp`/`fp`, so a findings record landing on that name is skipped, not
|
|
50
|
+
* mis-tallied. Read the field, never the filename.
|
|
44
51
|
*/
|
|
45
52
|
export const ADJUDICATION_SOURCE = 'canary-pr-guardian-adjudication';
|
|
46
53
|
/** GitHub reaction contents that carry an adjudication verdict. */
|
|
@@ -68,44 +75,35 @@ export class FakeReactionsClient {
|
|
|
68
75
|
}
|
|
69
76
|
/**
|
|
70
77
|
* Thin real {@link ReactionsClient} over the GitHub REST API (`fetch`).
|
|
71
|
-
* Network lives ONLY
|
|
72
|
-
*
|
|
78
|
+
* Network lives ONLY in the default {@link restPageReader}; both endpoints are
|
|
79
|
+
* reads, so a fork's read-only token is sufficient. The `read` seam exists so
|
|
80
|
+
* the #528 paging wiring is testable without a socket — production callers
|
|
81
|
+
* construct this with three arguments and get the real reader.
|
|
73
82
|
*/
|
|
74
83
|
export class RestReactionsClient {
|
|
75
84
|
repo;
|
|
76
85
|
prNumber;
|
|
77
|
-
token;
|
|
78
86
|
static API = 'https://api.github.com';
|
|
79
|
-
|
|
87
|
+
read;
|
|
88
|
+
constructor(repo, prNumber, token, read) {
|
|
80
89
|
this.repo = repo;
|
|
81
90
|
this.prNumber = prNumber;
|
|
82
|
-
this.
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
'X-GitHub-Api-Version': '2022-11-28',
|
|
91
|
-
'User-Agent': 'canary-pr-guardian',
|
|
92
|
-
},
|
|
93
|
-
});
|
|
94
|
-
if (!resp.ok) {
|
|
95
|
-
throw new Error(`GitHub API ${resp.status}: ${url}`);
|
|
96
|
-
}
|
|
97
|
-
return resp.json();
|
|
91
|
+
this.read =
|
|
92
|
+
read ??
|
|
93
|
+
restPageReader({
|
|
94
|
+
Authorization: `Bearer ${token}`,
|
|
95
|
+
Accept: 'application/vnd.github+json',
|
|
96
|
+
'X-GitHub-Api-Version': '2022-11-28',
|
|
97
|
+
'User-Agent': 'canary-pr-guardian',
|
|
98
|
+
}, (status, url) => new Error(`GitHub API ${status}: ${url}`));
|
|
98
99
|
}
|
|
99
100
|
async listComments() {
|
|
100
101
|
const url = `${RestReactionsClient.API}/repos/${this.repo}/issues/${this.prNumber}/comments`;
|
|
101
|
-
|
|
102
|
-
return Array.isArray(result) ? result : [];
|
|
102
|
+
return (await readAllPages(url, this.read));
|
|
103
103
|
}
|
|
104
104
|
async listReactions(commentId) {
|
|
105
105
|
const url = `${RestReactionsClient.API}/repos/${this.repo}/issues/comments/${commentId}/reactions`;
|
|
106
|
-
const result = await this.
|
|
107
|
-
if (!Array.isArray(result))
|
|
108
|
-
return [];
|
|
106
|
+
const result = await readAllPages(url, this.read);
|
|
109
107
|
const rows = [];
|
|
110
108
|
for (const raw of result) {
|
|
111
109
|
if (typeof raw !== 'object' || raw === null)
|
|
@@ -155,7 +153,13 @@ export function tallyAdjudications(reactions) {
|
|
|
155
153
|
// neither of which starts with a backtick, so anchoring on the second cell's
|
|
156
154
|
// leading backtick selects exactly the finding rows. Paths never contain `|`
|
|
157
155
|
// or backticks (see `fileLabel` in pr-check.ts), so the naive anchor is safe.
|
|
158
|
-
|
|
156
|
+
//
|
|
157
|
+
// The optional `[` accommodates the permalinked cell — `fileLabel` wraps the
|
|
158
|
+
// path as `[`path`](<blob url>)` whenever a blob base is resolvable, which is
|
|
159
|
+
// the normal case in CI. Without it this regex matched nothing on every posted
|
|
160
|
+
// comment and `activeFindingPaths` returned `[]`, zeroing the precision
|
|
161
|
+
// denominator silently instead of failing (#490, #508).
|
|
162
|
+
const FINDING_ROW_RE = /^\|[^|]*\|\s*\[?`([^`]+)`/;
|
|
159
163
|
/**
|
|
160
164
|
* Extract the file paths of the ACTIVE findings shown in a sticky-comment body
|
|
161
165
|
* (PURE). Reads the rendered table `render(fmt='comment')` emitted — this is
|
|
@@ -34,8 +34,10 @@
|
|
|
34
34
|
import { createHash, randomBytes } from 'node:crypto';
|
|
35
35
|
import { mkdirSync, renameSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
36
36
|
import { dirname, join } from 'node:path';
|
|
37
|
-
import {
|
|
38
|
-
|
|
37
|
+
import { coverageDegradedNotice, coverageStatus, } from './coverage.js';
|
|
38
|
+
import { combineNotices, render } from './pr-check.js';
|
|
39
|
+
// 1.1 adds the additive `coverage` block (#554); readers of 1.0 are unaffected.
|
|
40
|
+
export const SCHEMA_VERSION = '1.1';
|
|
39
41
|
export const SOURCE = 'canary-pr-guardian';
|
|
40
42
|
const REF_SAFE = /[^A-Za-z0-9._-]/g;
|
|
41
43
|
const REF_MAX = 100; // cap the sanitized ref so a long branch never hits ENAMETOOLONG
|
|
@@ -94,7 +96,11 @@ function isoUtcNow() {
|
|
|
94
96
|
*/
|
|
95
97
|
export function buildAnalysisRecord(findings, args) {
|
|
96
98
|
const { ref, gate, effective_tier, degraded_notice, exit_code } = args;
|
|
97
|
-
|
|
99
|
+
// #554: the record states BOTH degradations — the tier's and the coverage
|
|
100
|
+
// input's — so "no findings" can never be read as "coverage said so".
|
|
101
|
+
const coverage = args.coverage ?? null;
|
|
102
|
+
const notice = combineNotices(degraded_notice, coverage ? coverageDegradedNotice(coverage) : null);
|
|
103
|
+
const inner = JSON.parse(render(findings, 'json', effective_tier, notice));
|
|
98
104
|
const active = findings.filter((f) => !f.suppressed);
|
|
99
105
|
const suppressed = findings.filter((f) => f.suppressed);
|
|
100
106
|
const byFidelity = {};
|
|
@@ -110,7 +116,10 @@ export function buildAnalysisRecord(findings, args) {
|
|
|
110
116
|
checked: args.checked ?? 0,
|
|
111
117
|
abstained: args.abstained ?? false,
|
|
112
118
|
tier: effective_tier,
|
|
113
|
-
degradedNotice:
|
|
119
|
+
degradedNotice: notice,
|
|
120
|
+
coverage: coverage === null
|
|
121
|
+
? null
|
|
122
|
+
: { status: coverageStatus(coverage), ...coverage },
|
|
114
123
|
summary: {
|
|
115
124
|
total: findings.length,
|
|
116
125
|
unaddressed: active.length,
|