canary-test-cli 6.4.0 → 6.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/doctor.d.ts +51 -3
- package/dist/doctor.js +76 -9
- package/dist/engine/analysis/cli.js +69 -6
- package/dist/engine/cli-commands.js +33 -0
- package/dist/engine/core/framework-registry.js +4 -1
- package/dist/engine/core/gate-result.js +9 -2
- package/dist/engine/core/scaffold-templates.js +123 -0
- package/dist/engine/core/scaffolder.js +4 -105
- package/dist/engine/core/static-linter.js +125 -13
- package/dist/engine/guardian/adjudication.js +364 -0
- package/dist/engine/guardian/analysis-emit.js +2 -0
- package/dist/engine/guardian/cli.js +282 -15
- package/dist/engine/guardian/hard-gate.js +15 -2
- package/dist/engine/guardian/pr-check.js +5 -12
- package/dist/engine/history/async-store.js +20 -0
- package/dist/engine/history/cli.js +67 -0
- package/dist/engine/history/ndjson-store.js +4 -0
- package/dist/engine/history/store.js +3 -0
- package/dist/gate-result.d.ts +67 -0
- package/dist/gate-result.js +73 -0
- package/dist/overlay-commands.js +17 -1
- package/package.json +2 -1
|
@@ -26,10 +26,36 @@ const BARE_PLAYWRIGHT_CALL = /(?<!await\s)(?<!return\s)(?<!\w)(?:page|frame|loca
|
|
|
26
26
|
const TEST_FN_PY = /^(\s*)def (test_\w+)\s*\(/gm;
|
|
27
27
|
const TEST_FN_JS = /(?:^|\s)(?:it|test)\s*\(\s*['"]([^'"]*)['"]/gm;
|
|
28
28
|
const ASSERT_PY = /\bassert\b|\bpytest\.raises\b/;
|
|
29
|
-
|
|
29
|
+
// Assertion styles a JS/TS test may use. `expect()` (jest/vitest/playwright)
|
|
30
|
+
// was the only one recognized until canary was pointed at its own suites and
|
|
31
|
+
// reported 216 assertion-free tests of which 13 were real -- the other 200 were
|
|
32
|
+
// `node:test` + `node:assert`, a whole framework the linter could not see.
|
|
33
|
+
// Kept as a union of shapes rather than an import-aware parse: a static linter
|
|
34
|
+
// that needs to resolve imports to judge one line is the wrong trade.
|
|
35
|
+
// The `expectX()/assertX()` alternative covers a test that delegates its
|
|
36
|
+
// assertion to a named helper (`expectAuthoringAllowed(res)`) -- 9 of canary's
|
|
37
|
+
// own 16 residual findings. A regex linter cannot follow the call, so the NAME
|
|
38
|
+
// carries the signal; the `[A-Z]` keeps it to the convention rather than
|
|
39
|
+
// excusing any call that merely starts with those letters.
|
|
40
|
+
const ASSERT_JS = /\bexpect\s*\(|\bto(?:Be|Equal|Contain|Have|Match|Throw|Raise)\b|\bassert\s*\.\s*\w+\s*\(|\bassert\s*\(|\bshould\s*\.|\.should\b|\b(?:expect|assert)[A-Z]\w*\s*\(/;
|
|
30
41
|
// Strippers
|
|
31
42
|
const STRING_LITERAL = /(['"])(?:\\.|(?!\1).)*?\1/g;
|
|
32
|
-
// Magic numbers
|
|
43
|
+
// Magic numbers -- scoped to TIMING values only.
|
|
44
|
+
//
|
|
45
|
+
// "Extract the magic number to a named constant" is a production-code
|
|
46
|
+
// principle, and it inverts in a test: the literal IS the specification.
|
|
47
|
+
// `expect(notes.length).toBe(2048)` states the contract that
|
|
48
|
+
// `expect(notes.length).toBe(MAX_NOTES)` hides behind a name the reader now has
|
|
49
|
+
// to go look up. Since this linter only ever reads TEST files, the unscoped
|
|
50
|
+
// rule was misapplied across its entire domain -- measured at 0-for-157
|
|
51
|
+
// actionable when canary was first pointed at its own suites.
|
|
52
|
+
//
|
|
53
|
+
// A timing value is the one case that survives: a bare `5000` in a
|
|
54
|
+
// setTimeout/retry/interval position is a duration whose units and intent are
|
|
55
|
+
// genuinely unclear, and naming it genuinely helps. (Hardcoded sleeps are
|
|
56
|
+
// separately flagged at CRITICAL by FLAKE-001/002; this is the softer signal
|
|
57
|
+
// for the non-sleep timing values those rules do not cover.)
|
|
58
|
+
const TIMING_CONTEXT = /\b(?:setTimeout|setInterval|waitForTimeout|sleep|delay|timeout|interval|retryDelay|retries|backoff|pollInterval|debounce|throttle)\b/i;
|
|
33
59
|
const NUMERIC_LITERAL = /(?<![\w.])-?\d+(?:\.\d+)?(?![\w.])/g;
|
|
34
60
|
const ALLOWED_NUMBERS = new Set(['0', '1', '2', '-1', '10', '100']);
|
|
35
61
|
const HTTP_STATUS = new Set([
|
|
@@ -99,11 +125,33 @@ const FLAKINESS_RULES = [
|
|
|
99
125
|
suggestion: 'Mock Date.now()/datetime.now() or use a fixed reference date.',
|
|
100
126
|
},
|
|
101
127
|
];
|
|
128
|
+
/**
|
|
129
|
+
* Blank single-line string literals so a rule matching CODE cannot fire on test
|
|
130
|
+
* DATA.
|
|
131
|
+
*
|
|
132
|
+
* `scanMagicNumbers` has always done this; the flakiness and missing-await
|
|
133
|
+
* rules never did, so `const src = 'const t = Date.now();'` -- a fixture string
|
|
134
|
+
* feeding a linter test -- was reported as a real timestamp dependency. Any
|
|
135
|
+
* suite that carries the patterns it tests as string data hits this, and
|
|
136
|
+
* canary's own linter tests are the worst case.
|
|
137
|
+
*
|
|
138
|
+
* The same defect shipped in `canary-blackhawk`'s pragma parser (#499) and was
|
|
139
|
+
* guarded in `canary-savant` (#495/#498): data must never act as code, nor as
|
|
140
|
+
* directive.
|
|
141
|
+
*
|
|
142
|
+
* NOT applied to the selector rules: LINT-001/002/003 match `'.btn'` / `'#id'`
|
|
143
|
+
* inside quotes by construction, because a selector IS a string. Stripping
|
|
144
|
+
* would delete those rules outright.
|
|
145
|
+
*/
|
|
146
|
+
function blankStrings(line) {
|
|
147
|
+
return line.replace(STRING_LITERAL, '""');
|
|
148
|
+
}
|
|
102
149
|
function scanFlakiness(lines, file) {
|
|
103
150
|
const out = [];
|
|
104
|
-
lines.forEach((
|
|
105
|
-
if (isComment(
|
|
151
|
+
lines.forEach((raw, idx) => {
|
|
152
|
+
if (isComment(raw))
|
|
106
153
|
return;
|
|
154
|
+
const line = blankStrings(raw);
|
|
107
155
|
for (const r of FLAKINESS_RULES) {
|
|
108
156
|
if (r.re.test(line) && (!r.guard || r.guard(line))) {
|
|
109
157
|
out.push(mk(file, idx + 1, r.rule, r.severity, r.message, r.suggestion));
|
|
@@ -137,25 +185,70 @@ function scanSelectors(lines, file) {
|
|
|
137
185
|
}
|
|
138
186
|
function scanMissingAwait(lines, file) {
|
|
139
187
|
const out = [];
|
|
140
|
-
lines.forEach((
|
|
141
|
-
if (isComment(
|
|
188
|
+
lines.forEach((raw, idx) => {
|
|
189
|
+
if (isComment(raw))
|
|
142
190
|
return;
|
|
191
|
+
// A `page.click(...)` inside a string is fixture data, not a missing await.
|
|
192
|
+
const line = blankStrings(raw);
|
|
143
193
|
if (BARE_PLAYWRIGHT_CALL.test(line) && !line.includes('await')) {
|
|
144
194
|
out.push(mk(file, idx + 1, 'LINT-004', 'critical', 'Playwright action called without await.', 'Add `await` before the call to ensure it completes before the next step.'));
|
|
145
195
|
}
|
|
146
196
|
});
|
|
147
197
|
return out;
|
|
148
198
|
}
|
|
199
|
+
/** Multi-line string delimiters: JS template literal, Python triple quotes. */
|
|
200
|
+
const MULTILINE_DELIMS = ['`', '"""', "'''"];
|
|
201
|
+
/**
|
|
202
|
+
* Blank the INTERIOR of multi-line strings, preserving line count and numbering.
|
|
203
|
+
*
|
|
204
|
+
* Every per-line rule (magic numbers, selectors, flakiness) sees one line at a
|
|
205
|
+
* time, so a line inside a multi-line template literal or a Python
|
|
206
|
+
* triple-quoted block reads as bare code. Canary's own diff fixtures are
|
|
207
|
+
* template literals, so `100644` -- a git file mode sitting in test DATA -- was
|
|
208
|
+
* reported as a magic number 30 times. Any consumer with a multi-line SQL,
|
|
209
|
+
* JSON, HTML, or diff fixture has the same defect.
|
|
210
|
+
*
|
|
211
|
+
* Deliberately conservative about an UNBALANCED delimiter (a stray backtick in
|
|
212
|
+
* a comment, say): blanking to end-of-file would silently disable these rules
|
|
213
|
+
* from that point down -- the abstention shape, one layer inside the linter. An
|
|
214
|
+
* unclosed run is therefore discarded rather than applied.
|
|
215
|
+
*/
|
|
216
|
+
function blankMultilineStrings(lines) {
|
|
217
|
+
const out = [...lines];
|
|
218
|
+
for (const delim of MULTILINE_DELIMS) {
|
|
219
|
+
let openAt = null;
|
|
220
|
+
for (let i = 0; i < out.length; i += 1) {
|
|
221
|
+
const hits = out[i].split(delim).length - 1;
|
|
222
|
+
// An even count opens and closes on the same line, which the single-line
|
|
223
|
+
// stripper already handles; only an odd count toggles the state.
|
|
224
|
+
if (hits === 0 || hits % 2 === 0)
|
|
225
|
+
continue;
|
|
226
|
+
if (openAt === null) {
|
|
227
|
+
openAt = i;
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
for (let j = openAt + 1; j < i; j += 1)
|
|
231
|
+
out[j] = '';
|
|
232
|
+
openAt = null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
149
238
|
function scanMagicNumbers(lines, file) {
|
|
150
239
|
const out = [];
|
|
151
240
|
lines.forEach((raw, idx) => {
|
|
152
241
|
if (isComment(raw))
|
|
153
242
|
return;
|
|
154
243
|
const scrubbed = raw.replace(STRING_LITERAL, '""');
|
|
244
|
+
// Only timing positions: everywhere else in a test file the literal is the
|
|
245
|
+
// specification, not a smell. See TIMING_CONTEXT above.
|
|
246
|
+
if (!TIMING_CONTEXT.test(scrubbed))
|
|
247
|
+
return;
|
|
155
248
|
for (const m of scrubbed.matchAll(NUMERIC_LITERAL)) {
|
|
156
249
|
if (isAllowedNumber(m[0]))
|
|
157
250
|
continue;
|
|
158
|
-
out.push(mk(file, idx + 1, 'LINT-005', 'info', `Magic
|
|
251
|
+
out.push(mk(file, idx + 1, 'LINT-005', 'info', `Magic timing value ${m[0]}.`, 'Name the duration (e.g. RETRY_DELAY_MS) so its units and intent are readable.'));
|
|
159
252
|
break; // one finding per line
|
|
160
253
|
}
|
|
161
254
|
});
|
|
@@ -185,9 +278,23 @@ function scanAssertionFreePy(code, file) {
|
|
|
185
278
|
}
|
|
186
279
|
function scanAssertionFreeJs(code, file) {
|
|
187
280
|
const out = [];
|
|
188
|
-
|
|
281
|
+
// The test declarations, in source order, so each body can be bounded by the
|
|
282
|
+
// NEXT one -- the JS analogue of what the pytest scanner already does with
|
|
283
|
+
// "next `def` at the same indent".
|
|
284
|
+
//
|
|
285
|
+
// This replaces a fixed 2000-character lookahead that was wrong in BOTH
|
|
286
|
+
// directions: a long test whose first assertion fell past the window was
|
|
287
|
+
// flagged (false positive), and a short empty test could borrow the next
|
|
288
|
+
// test's assertion from inside the window (false negative). Neither failure
|
|
289
|
+
// is visible without a real codebase to run it against, which is why
|
|
290
|
+
// dogfooding found them and the unit tests did not.
|
|
291
|
+
const decls = [...code.matchAll(TEST_FN_JS)];
|
|
292
|
+
for (let i = 0; i < decls.length; i += 1) {
|
|
293
|
+
const m = decls[i];
|
|
189
294
|
const start = m.index;
|
|
190
|
-
const
|
|
295
|
+
const bodyStart = start + m[0].length;
|
|
296
|
+
const bodyEnd = decls[i + 1]?.index ?? code.length;
|
|
297
|
+
const rest = code.slice(bodyStart, bodyEnd);
|
|
191
298
|
if (!ASSERT_JS.test(rest)) {
|
|
192
299
|
out.push(mk(file, lineOf(code, start), 'LINT-006', 'warning', `Test "${m[1]}" contains no assertions.`, 'Add an expect() call; a test that never asserts always passes.'));
|
|
193
300
|
}
|
|
@@ -209,7 +316,12 @@ export class StaticLinter {
|
|
|
209
316
|
/** Full quality audit — all rules. */
|
|
210
317
|
lint(path, framework) {
|
|
211
318
|
const code = readFileSync(path, 'utf-8');
|
|
212
|
-
|
|
319
|
+
// No rule may read the interior of a multi-line string as code. Blanking
|
|
320
|
+
// preserves line count, so `scanned` re-joins to the same line numbers the
|
|
321
|
+
// per-line scanners report -- both halves must use it, or the assertion
|
|
322
|
+
// scanners go on mining `it(...)` declarations out of diff fixtures.
|
|
323
|
+
const lines = blankMultilineStrings(code.split('\n'));
|
|
324
|
+
const scanned = lines.join('\n');
|
|
213
325
|
const fw = framework || detectFramework(path);
|
|
214
326
|
const findings = [
|
|
215
327
|
...scanFlakiness(lines, path),
|
|
@@ -217,8 +329,8 @@ export class StaticLinter {
|
|
|
217
329
|
...scanMissingAwait(lines, path),
|
|
218
330
|
...scanMagicNumbers(lines, path),
|
|
219
331
|
...(fw === 'pytest'
|
|
220
|
-
? scanAssertionFreePy(
|
|
221
|
-
: scanAssertionFreeJs(
|
|
332
|
+
? scanAssertionFreePy(scanned, path)
|
|
333
|
+
: scanAssertionFreeJs(scanned, path)),
|
|
222
334
|
];
|
|
223
335
|
findings.sort((a, b) => a.line - b.line || cmp(a.rule, b.rule));
|
|
224
336
|
return findings;
|
|
@@ -226,7 +338,7 @@ export class StaticLinter {
|
|
|
226
338
|
/** Flakiness-only subset. */
|
|
227
339
|
flakeCheck(path) {
|
|
228
340
|
const code = readFileSync(path, 'utf-8');
|
|
229
|
-
const findings = scanFlakiness(code.split('\n'), path);
|
|
341
|
+
const findings = scanFlakiness(blankMultilineStrings(code.split('\n')), path);
|
|
230
342
|
findings.sort((a, b) => a.line - b.line);
|
|
231
343
|
return findings;
|
|
232
344
|
}
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finding adjudication collection — the precision the hard gate depends on
|
|
3
|
+
* (#490).
|
|
4
|
+
*
|
|
5
|
+
* `pr-check.ts` documents the soft→hard promotion contract as
|
|
6
|
+
* `precision = TP / (TP + FP)` fed by reviewer adjudication — but until this
|
|
7
|
+
* module nothing collected adjudications, so no repo could ever earn the hard
|
|
8
|
+
* gate. Reviewers already give the lowest-friction feedback available: a 👍
|
|
9
|
+
* (true positive) or 👎 (false positive) reaction on the guardian's sticky
|
|
10
|
+
* comment. This module reads those reactions back off the comment the guardian
|
|
11
|
+
* already upserts by marker, and persists a per-PR adjudication record to the
|
|
12
|
+
* existing `.harness/analyses/` channel (no new store — see
|
|
13
|
+
* {@link module:./analysis-emit}).
|
|
14
|
+
*
|
|
15
|
+
* Granularity (per the #490 design sketch): **whole-comment first**. One sticky
|
|
16
|
+
* comment carries N findings, so a reaction adjudicates the *run*, not one
|
|
17
|
+
* finding — except when the comment shows exactly one active finding, in which
|
|
18
|
+
* case the reaction is attributable to that finding's path. Per-finding
|
|
19
|
+
* comments were rejected as a worse artifact (N comments per PR).
|
|
20
|
+
*
|
|
21
|
+
* Zero-denominator discipline: a precision computed over 0 adjudicated
|
|
22
|
+
* findings is **unknown**, never 100%. {@link summarizePrecision} returns
|
|
23
|
+
* `precision: null` and {@link renderPrecision} says so in words. Most
|
|
24
|
+
* reviewers react to neither — the sample is small and self-selected, and every
|
|
25
|
+
* rendered surface states the sample size rather than presenting the number as
|
|
26
|
+
* ground truth.
|
|
27
|
+
*
|
|
28
|
+
* SC-11 boundary: deterministic HTTP/filesystem behind seams — no agent/LLM
|
|
29
|
+
* import. Network lives ONLY in {@link RestReactionsClient}; every unit test
|
|
30
|
+
* uses {@link FakeReactionsClient}.
|
|
31
|
+
*/
|
|
32
|
+
import { mkdirSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
33
|
+
import { randomBytes } from 'node:crypto';
|
|
34
|
+
import { dirname, join } from 'node:path';
|
|
35
|
+
import { STICKY_MARKER, findSticky } from './pr-comment.js';
|
|
36
|
+
/** Schema tag for adjudication records (independent of the findings schema). */
|
|
37
|
+
export const ADJUDICATION_SCHEMA_VERSION = '1.0';
|
|
38
|
+
/**
|
|
39
|
+
* Record `source` + filename prefix. Deliberately namespaced UNDER the
|
|
40
|
+
* `canary-pr-guardian-` prefix (harness's `AnalysisArchive` reads every
|
|
41
|
+
* `*.json` in `.harness/analyses/`) while never colliding with a pr-check
|
|
42
|
+
* findings record: those are `canary-pr-guardian-<sanitized-ref>.json` and a
|
|
43
|
+
* ref is sanitized from a git ref / `pr-<n>`, never `adjudication-pr-<n>`.
|
|
44
|
+
*/
|
|
45
|
+
export const ADJUDICATION_SOURCE = 'canary-pr-guardian-adjudication';
|
|
46
|
+
/** GitHub reaction contents that carry an adjudication verdict. */
|
|
47
|
+
const THUMBS_UP = '+1';
|
|
48
|
+
const THUMBS_DOWN = '-1';
|
|
49
|
+
// Loud notices carry an em-dash as output data; escaped per the ASCII-source rule.
|
|
50
|
+
const EM_DASH = '\u{2014}';
|
|
51
|
+
/** In-memory {@link ReactionsClient} for unit tests — no network. */
|
|
52
|
+
export class FakeReactionsClient {
|
|
53
|
+
comments;
|
|
54
|
+
reactionsByComment;
|
|
55
|
+
constructor(init = {}) {
|
|
56
|
+
this.comments = init.comments ?? [];
|
|
57
|
+
this.reactionsByComment = new Map(Object.entries(init.reactions ?? {}).map(([id, rows]) => [
|
|
58
|
+
Number(id),
|
|
59
|
+
rows,
|
|
60
|
+
]));
|
|
61
|
+
}
|
|
62
|
+
async listComments() {
|
|
63
|
+
return this.comments;
|
|
64
|
+
}
|
|
65
|
+
async listReactions(commentId) {
|
|
66
|
+
return this.reactionsByComment.get(commentId) ?? [];
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Thin real {@link ReactionsClient} over the GitHub REST API (`fetch`).
|
|
71
|
+
* Network lives ONLY here; no unit test exercises this class. Both endpoints
|
|
72
|
+
* are reads, so a fork's read-only token is sufficient.
|
|
73
|
+
*/
|
|
74
|
+
export class RestReactionsClient {
|
|
75
|
+
repo;
|
|
76
|
+
prNumber;
|
|
77
|
+
token;
|
|
78
|
+
static API = 'https://api.github.com';
|
|
79
|
+
constructor(repo, prNumber, token) {
|
|
80
|
+
this.repo = repo;
|
|
81
|
+
this.prNumber = prNumber;
|
|
82
|
+
this.token = token;
|
|
83
|
+
}
|
|
84
|
+
async get(url) {
|
|
85
|
+
const resp = await fetch(url, {
|
|
86
|
+
method: 'GET',
|
|
87
|
+
headers: {
|
|
88
|
+
Authorization: `Bearer ${this.token}`,
|
|
89
|
+
Accept: 'application/vnd.github+json',
|
|
90
|
+
'X-GitHub-Api-Version': '2022-11-28',
|
|
91
|
+
'User-Agent': 'canary-pr-guardian',
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
if (!resp.ok) {
|
|
95
|
+
throw new Error(`GitHub API ${resp.status}: ${url}`);
|
|
96
|
+
}
|
|
97
|
+
return resp.json();
|
|
98
|
+
}
|
|
99
|
+
async listComments() {
|
|
100
|
+
const url = `${RestReactionsClient.API}/repos/${this.repo}/issues/${this.prNumber}/comments`;
|
|
101
|
+
const result = await this.get(url);
|
|
102
|
+
return Array.isArray(result) ? result : [];
|
|
103
|
+
}
|
|
104
|
+
async listReactions(commentId) {
|
|
105
|
+
const url = `${RestReactionsClient.API}/repos/${this.repo}/issues/comments/${commentId}/reactions`;
|
|
106
|
+
const result = await this.get(url);
|
|
107
|
+
if (!Array.isArray(result))
|
|
108
|
+
return [];
|
|
109
|
+
const rows = [];
|
|
110
|
+
for (const raw of result) {
|
|
111
|
+
if (typeof raw !== 'object' || raw === null)
|
|
112
|
+
continue;
|
|
113
|
+
const rec = raw;
|
|
114
|
+
const content = typeof rec.content === 'string' ? rec.content : '';
|
|
115
|
+
const user = typeof rec.user?.login === 'string' ? rec.user.login : 'unknown';
|
|
116
|
+
if (content)
|
|
117
|
+
rows.push({ user, content });
|
|
118
|
+
}
|
|
119
|
+
return rows;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Tally verdict reactions: one vote per user, bots excluded (PURE).
|
|
124
|
+
*
|
|
125
|
+
* - Only `+1`/`-1` carry a verdict; every other content is ignored.
|
|
126
|
+
* - Logins ending in `[bot]` are excluded so the guardian's own automation (or
|
|
127
|
+
* any other bot) can never inflate its own precision.
|
|
128
|
+
* - A user who reacted both 👍 and 👎 is contradictory: counted as `ambiguous`
|
|
129
|
+
* and excluded from both TP and FP rather than guessed at.
|
|
130
|
+
*/
|
|
131
|
+
export function tallyAdjudications(reactions) {
|
|
132
|
+
const up = new Set();
|
|
133
|
+
const down = new Set();
|
|
134
|
+
for (const reaction of reactions) {
|
|
135
|
+
if (reaction.user.endsWith('[bot]'))
|
|
136
|
+
continue;
|
|
137
|
+
if (reaction.content === THUMBS_UP)
|
|
138
|
+
up.add(reaction.user);
|
|
139
|
+
else if (reaction.content === THUMBS_DOWN)
|
|
140
|
+
down.add(reaction.user);
|
|
141
|
+
}
|
|
142
|
+
let ambiguous = 0;
|
|
143
|
+
for (const user of up) {
|
|
144
|
+
if (down.has(user))
|
|
145
|
+
ambiguous += 1;
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
tp: up.size - ambiguous,
|
|
149
|
+
fp: down.size - ambiguous,
|
|
150
|
+
ambiguous,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
// A findings-table row in the sticky comment: `| <icon> <sev> | `path`... |`.
|
|
154
|
+
// The header row's second cell is ` File ` and the separator's is ` --- `,
|
|
155
|
+
// neither of which starts with a backtick, so anchoring on the second cell's
|
|
156
|
+
// leading backtick selects exactly the finding rows. Paths never contain `|`
|
|
157
|
+
// or backticks (see `fileLabel` in pr-check.ts), so the naive anchor is safe.
|
|
158
|
+
const FINDING_ROW_RE = /^\|[^|]*\|\s*`([^`]+)`/;
|
|
159
|
+
/**
|
|
160
|
+
* Extract the file paths of the ACTIVE findings shown in a sticky-comment body
|
|
161
|
+
* (PURE). Reads the rendered table `render(fmt='comment')` emitted — this is
|
|
162
|
+
* deliberately parsing the exact body reviewers reacted to, not the current
|
|
163
|
+
* finding set, so a reaction is attributed to what the reviewer actually saw.
|
|
164
|
+
* Returns `[]` for a no-gaps body (no table).
|
|
165
|
+
*/
|
|
166
|
+
export function activeFindingPaths(commentBody) {
|
|
167
|
+
const paths = [];
|
|
168
|
+
for (const line of commentBody.split(/\r\n|\r|\n/)) {
|
|
169
|
+
const match = FINDING_ROW_RE.exec(line);
|
|
170
|
+
if (match)
|
|
171
|
+
paths.push(match[1]);
|
|
172
|
+
}
|
|
173
|
+
return paths;
|
|
174
|
+
}
|
|
175
|
+
/** ISO-8601 UTC timestamp with a `+00:00` offset (matches analysis-emit). */
|
|
176
|
+
function isoUtcNow() {
|
|
177
|
+
return new Date().toISOString().replace('Z', '+00:00');
|
|
178
|
+
}
|
|
179
|
+
/** Build the v1.0 adjudication record (PURE given `collectedAt`). */
|
|
180
|
+
export function buildAdjudicationRecord(init) {
|
|
181
|
+
const findingPaths = activeFindingPaths(init.commentBody);
|
|
182
|
+
const single = findingPaths.length === 1;
|
|
183
|
+
return {
|
|
184
|
+
schemaVersion: ADJUDICATION_SCHEMA_VERSION,
|
|
185
|
+
source: ADJUDICATION_SOURCE,
|
|
186
|
+
repo: init.repo,
|
|
187
|
+
prNumber: init.prNumber,
|
|
188
|
+
commentId: init.commentId,
|
|
189
|
+
granularity: single ? 'finding' : 'run',
|
|
190
|
+
attributedPath: single ? findingPaths[0] : null,
|
|
191
|
+
findingPaths,
|
|
192
|
+
tp: init.tally.tp,
|
|
193
|
+
fp: init.tally.fp,
|
|
194
|
+
ambiguous: init.tally.ambiguous,
|
|
195
|
+
collectedAt: init.collectedAt ?? isoUtcNow(),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
/** `canary-pr-guardian-adjudication-pr-<n>.json` under the analyses dir. */
|
|
199
|
+
export function adjudicationFilename(prNumber) {
|
|
200
|
+
return `${ADJUDICATION_SOURCE}-pr-${prNumber}.json`;
|
|
201
|
+
}
|
|
202
|
+
/** True iff the harness home (`dirname(analysesDir)`) exists. */
|
|
203
|
+
function channelAvailable(analysesDir) {
|
|
204
|
+
try {
|
|
205
|
+
return statSync(dirname(analysesDir)).isDirectory();
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Read the sticky comment's reactions and persist the PR's adjudication record.
|
|
213
|
+
*
|
|
214
|
+
* Idempotent per PR: the record is the LATEST reaction state, overwritten in
|
|
215
|
+
* place on each collection (reactions live on the comment, which the guardian
|
|
216
|
+
* upserts rather than re-creates, so they accumulate monotonically). Records
|
|
217
|
+
* for different PRs never collide — the store is append-only across PRs.
|
|
218
|
+
*
|
|
219
|
+
* Never throws for an expected shape: a missing comment, zero reactions, or an
|
|
220
|
+
* unavailable channel each return a distinct non-`collected` result so the
|
|
221
|
+
* caller can report honestly instead of crashing the gate.
|
|
222
|
+
*/
|
|
223
|
+
export async function collectAdjudications(client, args) {
|
|
224
|
+
const sticky = findSticky(await client.listComments(), args.marker ?? STICKY_MARKER);
|
|
225
|
+
if (sticky === null) {
|
|
226
|
+
return { action: 'no-comment', path: null, record: null, notice: null };
|
|
227
|
+
}
|
|
228
|
+
const tally = tallyAdjudications(await client.listReactions(sticky.id));
|
|
229
|
+
if (tally.tp + tally.fp + tally.ambiguous === 0) {
|
|
230
|
+
return { action: 'no-reactions', path: null, record: null, notice: null };
|
|
231
|
+
}
|
|
232
|
+
const record = buildAdjudicationRecord({
|
|
233
|
+
repo: args.repo,
|
|
234
|
+
prNumber: args.prNumber,
|
|
235
|
+
commentId: sticky.id,
|
|
236
|
+
commentBody: sticky.body,
|
|
237
|
+
tally,
|
|
238
|
+
collectedAt: args.collectedAt,
|
|
239
|
+
});
|
|
240
|
+
if (!channelAvailable(args.analysesDir)) {
|
|
241
|
+
return {
|
|
242
|
+
action: 'unavailable',
|
|
243
|
+
path: null,
|
|
244
|
+
record,
|
|
245
|
+
notice: 'guardian: harness analyses channel unavailable (.harness/ absent) ' +
|
|
246
|
+
`${EM_DASH} adjudication not persisted`,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
const target = join(args.analysesDir, adjudicationFilename(args.prNumber));
|
|
250
|
+
try {
|
|
251
|
+
mkdirSync(args.analysesDir, { recursive: true });
|
|
252
|
+
// Atomic write (same-dir temp + rename), matching analysis-emit: a torn
|
|
253
|
+
// record would poison every later precision summary.
|
|
254
|
+
const tmp = join(args.analysesDir, `.tmp-${randomBytes(8).toString('hex')}.json`);
|
|
255
|
+
writeFileSync(tmp, JSON.stringify(record, null, 2), 'utf-8');
|
|
256
|
+
try {
|
|
257
|
+
renameSync(tmp, target);
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
try {
|
|
261
|
+
unlinkSync(tmp);
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
// best-effort cleanup
|
|
265
|
+
}
|
|
266
|
+
throw err;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
catch (exc) {
|
|
270
|
+
const message = exc instanceof Error ? exc.message : String(exc);
|
|
271
|
+
return {
|
|
272
|
+
action: 'unavailable',
|
|
273
|
+
path: null,
|
|
274
|
+
record,
|
|
275
|
+
notice: `guardian: adjudication write failed (${message}) ${EM_DASH} ` +
|
|
276
|
+
'adjudication not persisted',
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
return { action: 'collected', path: target, record, notice: null };
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Load every adjudication record under `analysesDir` (best-effort).
|
|
283
|
+
*
|
|
284
|
+
* Reads only `canary-pr-guardian-adjudication-*.json`; pr-check findings
|
|
285
|
+
* records and harness's own records are never touched. A malformed or
|
|
286
|
+
* wrong-`source` file is skipped, never fatal — one corrupt record must not
|
|
287
|
+
* take down the precision report.
|
|
288
|
+
*/
|
|
289
|
+
export function loadAdjudicationRecords(analysesDir) {
|
|
290
|
+
let names;
|
|
291
|
+
try {
|
|
292
|
+
names = readdirSync(analysesDir);
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return [];
|
|
296
|
+
}
|
|
297
|
+
const records = [];
|
|
298
|
+
for (const name of names.sort()) {
|
|
299
|
+
if (!name.startsWith(`${ADJUDICATION_SOURCE}-`) || !name.endsWith('.json'))
|
|
300
|
+
continue;
|
|
301
|
+
try {
|
|
302
|
+
const raw = JSON.parse(readFileSync(join(analysesDir, name), 'utf-8'));
|
|
303
|
+
if (raw !== null &&
|
|
304
|
+
typeof raw === 'object' &&
|
|
305
|
+
raw.source === ADJUDICATION_SOURCE &&
|
|
306
|
+
typeof raw.tp === 'number' &&
|
|
307
|
+
typeof raw.fp === 'number') {
|
|
308
|
+
records.push(raw);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
// skip malformed record
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
return records;
|
|
316
|
+
}
|
|
317
|
+
/** Aggregate records into the precision summary (PURE). */
|
|
318
|
+
export function summarizePrecision(records) {
|
|
319
|
+
let tp = 0;
|
|
320
|
+
let fp = 0;
|
|
321
|
+
let ambiguous = 0;
|
|
322
|
+
let prCount = 0;
|
|
323
|
+
for (const record of records) {
|
|
324
|
+
tp += record.tp;
|
|
325
|
+
fp += record.fp;
|
|
326
|
+
ambiguous += record.ambiguous ?? 0;
|
|
327
|
+
if (record.tp + record.fp > 0)
|
|
328
|
+
prCount += 1;
|
|
329
|
+
}
|
|
330
|
+
const adjudicated = tp + fp;
|
|
331
|
+
return {
|
|
332
|
+
adjudicated,
|
|
333
|
+
tp,
|
|
334
|
+
fp,
|
|
335
|
+
ambiguous,
|
|
336
|
+
prCount,
|
|
337
|
+
precision: adjudicated === 0 ? null : tp / adjudicated,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Render the precision summary as human text (PURE).
|
|
342
|
+
*
|
|
343
|
+
* Zero-denominator discipline: with no adjudications the FIRST word after the
|
|
344
|
+
* label is `unknown` — the report never implies 100% (or any number) from an
|
|
345
|
+
* empty sample. With data, the sample size and its self-selected nature ride
|
|
346
|
+
* alongside the number on the same line.
|
|
347
|
+
*/
|
|
348
|
+
export function renderPrecision(summary) {
|
|
349
|
+
if (summary.precision === null) {
|
|
350
|
+
return (`guardian precision: unknown ${EM_DASH} no adjudications yet ` +
|
|
351
|
+
`(0 reviewer verdicts collected). React with a thumbs-up (finding was ` +
|
|
352
|
+
`right) or thumbs-down (false positive) on the guardian's PR comment.`);
|
|
353
|
+
}
|
|
354
|
+
const pct = (summary.precision * 100).toFixed(1).replace(/\.0$/, '');
|
|
355
|
+
const ambiguousNote = summary.ambiguous > 0
|
|
356
|
+
? ` ${summary.ambiguous} contradictory verdict(s) excluded.`
|
|
357
|
+
: '';
|
|
358
|
+
return (`guardian precision: ${pct}% (${summary.tp} true / ${summary.fp} false ` +
|
|
359
|
+
`positive${summary.adjudicated === 1 ? '' : 's'}, n=${summary.adjudicated} ` +
|
|
360
|
+
`across ${summary.prCount} PR(s)).${ambiguousNote} Sample is ` +
|
|
361
|
+
`self-selected (reviewers who chose to react) ${EM_DASH} a signal, not ` +
|
|
362
|
+
`ground truth.`);
|
|
363
|
+
}
|
|
364
|
+
//# sourceMappingURL=adjudication.js.map
|