canary-test-cli 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/analysis/reachability.js +305 -0
- package/dist/engine/cli.core.js +5 -5
- package/dist/engine/core/company-knowledge.js +6 -0
- package/dist/engine/core/framework-registry.js +9 -6
- package/dist/engine/guardian/cli.js +162 -7
- package/dist/engine/guardian/coverage.js +90 -0
- package/dist/engine/guardian/pr-check.js +194 -20
- package/package.json +2 -3
- /package/{agent → dist/engine/data}/frameworks/registry.json +0 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reachability sweep — the generic crawl primitive from #452.
|
|
3
|
+
*
|
|
4
|
+
* Enumerate every link on a surface and assert it resolves. Cheap, broad, and
|
|
5
|
+
* high-yield: it catches dangling routes and 404s that targeted tests never
|
|
6
|
+
* look for, because a targeted test asserts what a page *does*, not that the
|
|
7
|
+
* things it points at still exist.
|
|
8
|
+
*
|
|
9
|
+
* This module owns the deterministic half — normalization, scoping, and
|
|
10
|
+
* classification. **Fetching is injected** ({@link LinkProbe}), so the logic is
|
|
11
|
+
* unit-testable with no network and a caller can supply whatever transport it
|
|
12
|
+
* already has (plain `fetch`, an authenticated client, a browser page).
|
|
13
|
+
*
|
|
14
|
+
* ## The load-bearing distinction: dead vs. slow
|
|
15
|
+
*
|
|
16
|
+
* The reason this is not simply "fetch and check the status": **a dead link and
|
|
17
|
+
* a slow link must never be confusable.** A 404 is a defect the sweep is
|
|
18
|
+
* entitled to assert on. A timeout, DNS failure, or refused connection is
|
|
19
|
+
* *inconclusive* — the link may be perfectly fine while the network, a cold
|
|
20
|
+
* container, or a rate limiter is not.
|
|
21
|
+
*
|
|
22
|
+
* Conflating them is how a broad sweep turns into a flaky test that teams learn
|
|
23
|
+
* to ignore, which costs more than the check was ever worth. So the two are
|
|
24
|
+
* structurally different outcomes ({@link ReachabilityStatus.Broken} vs
|
|
25
|
+
* {@link ReachabilityStatus.Unreachable}) with different {@link isDefect}
|
|
26
|
+
* verdicts, and only the first is ever asserted on.
|
|
27
|
+
*
|
|
28
|
+
* The same principle decides the other ambiguous cases:
|
|
29
|
+
* - **5xx** — the target exists and is unwell. A server problem, not a
|
|
30
|
+
* dangling reference; blaming the link sends someone to fix the wrong
|
|
31
|
+
* thing.
|
|
32
|
+
* - **401/403** — auth-walled, not missing. The link is fine; the sweep just
|
|
33
|
+
* is not logged in.
|
|
34
|
+
*
|
|
35
|
+
* Only an unambiguous "this target does not exist" (4xx that is not auth)
|
|
36
|
+
* counts as a defect.
|
|
37
|
+
*/
|
|
38
|
+
/** How a probed link resolved. Only {@link Broken} is asserted on. */
|
|
39
|
+
export var ReachabilityStatus;
|
|
40
|
+
(function (ReachabilityStatus) {
|
|
41
|
+
/** Resolved (2xx/3xx), or exists but is auth-walled (401/403). */
|
|
42
|
+
ReachabilityStatus["Ok"] = "ok";
|
|
43
|
+
/** The target does not exist (4xx other than auth). The one real defect. */
|
|
44
|
+
ReachabilityStatus["Broken"] = "broken";
|
|
45
|
+
/** The target exists but errored (5xx) — a server problem, not a bad link. */
|
|
46
|
+
ReachabilityStatus["ServerError"] = "server-error";
|
|
47
|
+
/** No verdict possible: timeout, DNS failure, refused connection. */
|
|
48
|
+
ReachabilityStatus["Unreachable"] = "unreachable";
|
|
49
|
+
/** Off-site and not allowlisted, so deliberately not probed. */
|
|
50
|
+
ReachabilityStatus["SkippedExternal"] = "skipped-external";
|
|
51
|
+
})(ReachabilityStatus || (ReachabilityStatus = {}));
|
|
52
|
+
// Schemes that are not fetchable resources. A `mailto:`/`tel:` is a valid link
|
|
53
|
+
// and probing it is meaningless; `javascript:` is a control, not a destination.
|
|
54
|
+
const NON_FETCHABLE_SCHEMES = new Set([
|
|
55
|
+
'mailto:',
|
|
56
|
+
'tel:',
|
|
57
|
+
'javascript:',
|
|
58
|
+
'data:',
|
|
59
|
+
'blob:',
|
|
60
|
+
'sms:',
|
|
61
|
+
'file:',
|
|
62
|
+
]);
|
|
63
|
+
/** Auth-walled: the target exists, the sweep just cannot see it. */
|
|
64
|
+
const AUTH_STATUSES = new Set([401, 403]);
|
|
65
|
+
/**
|
|
66
|
+
* Resolve `href` against `base` into an absolute, fragment-free URL.
|
|
67
|
+
*
|
|
68
|
+
* Returns `null` for anything that is not a fetchable destination: an empty or
|
|
69
|
+
* fragment-only href (it points at the current page), a non-fetchable scheme,
|
|
70
|
+
* or an href malformed enough that `URL` rejects it.
|
|
71
|
+
*
|
|
72
|
+
* The fragment is dropped deliberately — `/p#a` and `/p#b` are the same
|
|
73
|
+
* request, and keeping them apart would probe the same page repeatedly and
|
|
74
|
+
* report one dangling route as several.
|
|
75
|
+
*/
|
|
76
|
+
export function normalizeLink(href, base) {
|
|
77
|
+
const trimmed = href.trim();
|
|
78
|
+
// Empty, or a pure fragment: this is the current page, not a destination.
|
|
79
|
+
if (!trimmed || trimmed.startsWith('#'))
|
|
80
|
+
return null;
|
|
81
|
+
let url;
|
|
82
|
+
try {
|
|
83
|
+
url = new URL(trimmed, base);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
if (NON_FETCHABLE_SCHEMES.has(url.protocol))
|
|
89
|
+
return null;
|
|
90
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
91
|
+
return null;
|
|
92
|
+
url.hash = '';
|
|
93
|
+
return { url: url.href, host: url.hostname };
|
|
94
|
+
}
|
|
95
|
+
/** True iff `host` is `allowed` or a subdomain of it (never a lookalike). */
|
|
96
|
+
function hostMatches(host, allowed) {
|
|
97
|
+
const h = host.toLowerCase();
|
|
98
|
+
const a = allowed.toLowerCase();
|
|
99
|
+
// The leading dot is what stops `nottrusted.example` matching
|
|
100
|
+
// `trusted.example` — a bare `endsWith` would wrongly accept it.
|
|
101
|
+
return h === a || h.endsWith(`.${a}`);
|
|
102
|
+
}
|
|
103
|
+
/** True iff `link` is in scope: same host as the base, or allowlisted. */
|
|
104
|
+
function inScope(link, baseHost, allowExternal) {
|
|
105
|
+
if (hostMatches(link.host, baseHost))
|
|
106
|
+
return true;
|
|
107
|
+
return allowExternal.some((allowed) => hostMatches(link.host, allowed));
|
|
108
|
+
}
|
|
109
|
+
/** Map an HTTP status to a verdict. See the module docstring for the rationale. */
|
|
110
|
+
function classifyStatus(status) {
|
|
111
|
+
if (status >= 200 && status < 400)
|
|
112
|
+
return ReachabilityStatus.Ok;
|
|
113
|
+
// Auth-walled is not dangling: the target exists.
|
|
114
|
+
if (AUTH_STATUSES.has(status))
|
|
115
|
+
return ReachabilityStatus.Ok;
|
|
116
|
+
if (status >= 400 && status < 500)
|
|
117
|
+
return ReachabilityStatus.Broken;
|
|
118
|
+
if (status >= 500)
|
|
119
|
+
return ReachabilityStatus.ServerError;
|
|
120
|
+
// 1xx as a final status is nonsensical; refuse to guess.
|
|
121
|
+
return ReachabilityStatus.Unreachable;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* True iff `status` is a finding worth failing a sweep on.
|
|
125
|
+
*
|
|
126
|
+
* Only {@link ReachabilityStatus.Broken}. `Unreachable` is inconclusive,
|
|
127
|
+
* `ServerError` belongs to whoever owns the server, and a skipped link was
|
|
128
|
+
* never claimed to be checked. Asserting on any of those would make the sweep
|
|
129
|
+
* fail for reasons the author of the link cannot fix.
|
|
130
|
+
*/
|
|
131
|
+
export function isDefect(status) {
|
|
132
|
+
return status === ReachabilityStatus.Broken;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Sweep `hrefs`, probing each unique in-scope URL exactly once.
|
|
136
|
+
*
|
|
137
|
+
* Results are sorted by URL so a sweep produces a stable, diffable report
|
|
138
|
+
* regardless of the order links appeared on the page.
|
|
139
|
+
*/
|
|
140
|
+
export function sweepLinks(hrefs, options) {
|
|
141
|
+
const { base, probe, allowExternal = [] } = options;
|
|
142
|
+
const baseHost = new URL(base).hostname;
|
|
143
|
+
// Dedup by normalized URL: a nav link repeated on every page is one check,
|
|
144
|
+
// and one dangling route should be reported once, not once per referrer.
|
|
145
|
+
const unique = new Map();
|
|
146
|
+
for (const href of hrefs) {
|
|
147
|
+
const link = normalizeLink(href, base);
|
|
148
|
+
if (link !== null && !unique.has(link.url))
|
|
149
|
+
unique.set(link.url, link);
|
|
150
|
+
}
|
|
151
|
+
const results = [];
|
|
152
|
+
for (const { link, scoped } of planSweep(hrefs, base, allowExternal)) {
|
|
153
|
+
results.push(scoped ? resolveOutcome(link, probe(link.url)) : skippedResult(link));
|
|
154
|
+
}
|
|
155
|
+
return { results, summary: summarize(results) };
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Normalize, dedup, sort, and scope `hrefs` — everything the sync and async
|
|
159
|
+
* drivers must agree on. Shared so the two can never drift apart on which
|
|
160
|
+
* links get probed or in what order.
|
|
161
|
+
*
|
|
162
|
+
* Dedup by normalized URL: a nav link repeated on every page is one check, and
|
|
163
|
+
* one dangling route should be reported once, not once per referrer. Sorted by
|
|
164
|
+
* URL so a sweep produces a stable, diffable report regardless of the order
|
|
165
|
+
* links appeared on the page.
|
|
166
|
+
*/
|
|
167
|
+
function planSweep(hrefs, base, allowExternal) {
|
|
168
|
+
const baseHost = new URL(base).hostname;
|
|
169
|
+
const unique = new Map();
|
|
170
|
+
for (const href of hrefs) {
|
|
171
|
+
const link = normalizeLink(href, base);
|
|
172
|
+
if (link !== null && !unique.has(link.url))
|
|
173
|
+
unique.set(link.url, link);
|
|
174
|
+
}
|
|
175
|
+
return [...unique.values()]
|
|
176
|
+
.sort((a, b) => a.url.localeCompare(b.url))
|
|
177
|
+
.map((link) => ({
|
|
178
|
+
link,
|
|
179
|
+
scoped: inScope(link, baseHost, [...allowExternal]),
|
|
180
|
+
}));
|
|
181
|
+
}
|
|
182
|
+
function skippedResult(link) {
|
|
183
|
+
return {
|
|
184
|
+
url: link.url,
|
|
185
|
+
status: ReachabilityStatus.SkippedExternal,
|
|
186
|
+
httpStatus: null,
|
|
187
|
+
detail: 'off-site and not allowlisted',
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
/** Turn a probe outcome into a verdict, keeping dead and slow apart. */
|
|
191
|
+
function resolveOutcome(link, outcome) {
|
|
192
|
+
if (outcome.kind === 'failed') {
|
|
193
|
+
return {
|
|
194
|
+
url: link.url,
|
|
195
|
+
status: ReachabilityStatus.Unreachable,
|
|
196
|
+
httpStatus: null,
|
|
197
|
+
// The reason is carried verbatim: an inconclusive result is only
|
|
198
|
+
// actionable if the reader can see WHY no verdict was reached.
|
|
199
|
+
detail: `no response (${outcome.reason})`,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
return {
|
|
203
|
+
url: link.url,
|
|
204
|
+
status: classifyStatus(outcome.status),
|
|
205
|
+
httpStatus: outcome.status,
|
|
206
|
+
detail: `HTTP ${outcome.status}`,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* {@link sweepLinks} over an async transport — the shape real network use takes.
|
|
211
|
+
*
|
|
212
|
+
* Probes sequentially on purpose. A sweep enumerates *every* link on a surface,
|
|
213
|
+
* so firing them concurrently at one host is indistinguishable from a small
|
|
214
|
+
* load test and invites the rate limiting that would then be misreported as
|
|
215
|
+
* `unreachable`. A slow, correct sweep beats a fast, self-poisoning one.
|
|
216
|
+
*/
|
|
217
|
+
export async function sweepLinksAsync(hrefs, options) {
|
|
218
|
+
const { base, probe, allowExternal = [] } = options;
|
|
219
|
+
const results = [];
|
|
220
|
+
for (const { link, scoped } of planSweep(hrefs, base, allowExternal)) {
|
|
221
|
+
results.push(scoped
|
|
222
|
+
? resolveOutcome(link, await probe(link.url))
|
|
223
|
+
: skippedResult(link));
|
|
224
|
+
}
|
|
225
|
+
return { results, summary: summarize(results) };
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* A `fetch`-backed {@link AsyncLinkProbe} that keeps dead and slow apart.
|
|
229
|
+
*
|
|
230
|
+
* Any outcome where the server answered — including a 404 — is a `status`. Any
|
|
231
|
+
* outcome where no response arrived (timeout, DNS failure, refused connection)
|
|
232
|
+
* is a `failed`, carrying its reason. Callers should use this rather than
|
|
233
|
+
* hand-rolling a probe: deciding what counts as "no response" IS the guarantee
|
|
234
|
+
* this module exists to provide, and re-implementing it per caller is how the
|
|
235
|
+
* distinction gets quietly lost.
|
|
236
|
+
*
|
|
237
|
+
* Uses `HEAD`, falling back to `GET` when a server rejects `HEAD` outright
|
|
238
|
+
* (405/501) — some servers do, and treating that as a broken link would be a
|
|
239
|
+
* false positive.
|
|
240
|
+
*/
|
|
241
|
+
export function createHttpProbe(options = {}) {
|
|
242
|
+
const { timeoutMs = 10_000, fetchImpl = fetch } = options;
|
|
243
|
+
return async (url) => {
|
|
244
|
+
const attempt = async (method) => {
|
|
245
|
+
const controller = new AbortController();
|
|
246
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
247
|
+
try {
|
|
248
|
+
const res = await fetchImpl(url, {
|
|
249
|
+
method,
|
|
250
|
+
redirect: 'follow',
|
|
251
|
+
signal: controller.signal,
|
|
252
|
+
});
|
|
253
|
+
return { kind: 'status', status: res.status };
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
const reason = controller.signal.aborted
|
|
257
|
+
? `timeout after ${timeoutMs}ms`
|
|
258
|
+
: err instanceof Error
|
|
259
|
+
? err.message
|
|
260
|
+
: String(err);
|
|
261
|
+
return { kind: 'failed', reason };
|
|
262
|
+
}
|
|
263
|
+
finally {
|
|
264
|
+
clearTimeout(timer);
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
const head = await attempt('HEAD');
|
|
268
|
+
if (head.kind === 'status' &&
|
|
269
|
+
(head.status === 405 || head.status === 501)) {
|
|
270
|
+
return attempt('GET');
|
|
271
|
+
}
|
|
272
|
+
return head;
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
function summarize(results) {
|
|
276
|
+
const summary = {
|
|
277
|
+
total: results.length,
|
|
278
|
+
ok: 0,
|
|
279
|
+
broken: 0,
|
|
280
|
+
unreachable: 0,
|
|
281
|
+
skipped: 0,
|
|
282
|
+
defects: 0,
|
|
283
|
+
};
|
|
284
|
+
for (const r of results) {
|
|
285
|
+
switch (r.status) {
|
|
286
|
+
case ReachabilityStatus.Broken:
|
|
287
|
+
summary.broken += 1;
|
|
288
|
+
break;
|
|
289
|
+
case ReachabilityStatus.Unreachable:
|
|
290
|
+
summary.unreachable += 1;
|
|
291
|
+
break;
|
|
292
|
+
case ReachabilityStatus.SkippedExternal:
|
|
293
|
+
summary.skipped += 1;
|
|
294
|
+
break;
|
|
295
|
+
// A 5xx is counted with `ok` for the accounting identity (it is not a
|
|
296
|
+
// link defect); it stays distinguishable via each result's own status.
|
|
297
|
+
default:
|
|
298
|
+
summary.ok += 1;
|
|
299
|
+
}
|
|
300
|
+
if (isDefect(r.status))
|
|
301
|
+
summary.defects += 1;
|
|
302
|
+
}
|
|
303
|
+
return summary;
|
|
304
|
+
}
|
|
305
|
+
//# sourceMappingURL=reachability.js.map
|
package/dist/engine/cli.core.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The main `canary` command --
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* The main `canary` command -- the commander CLI, culmination of the CLI wave.
|
|
3
|
+
* Mounts the sub-apps (`guardian`, `history`, `analyze`) plus the inline
|
|
4
|
+
* `skills` / `workflow` / `company-knowledge` sub-apps, and the ~15 top-level
|
|
5
|
+
* commands. This is the sole shipping entry point (the Python engine was
|
|
6
|
+
* retired in the v6 cutover).
|
|
7
7
|
*
|
|
8
8
|
* Conventions follow `guardian/cli.ts` (see `cli-common.ts`): a
|
|
9
9
|
* {@link createCanaryCommand} factory wired to an injectable {@link MainDeps},
|
|
@@ -112,6 +112,12 @@ const _KNOWN_KEYS = new Set([
|
|
|
112
112
|
'otel_exporter_endpoint',
|
|
113
113
|
'notes',
|
|
114
114
|
'brand',
|
|
115
|
+
// #459: read by the migrator (`_detectFramework` / overlay `deploy_to`
|
|
116
|
+
// matching) to decide which overlay skills deploy. It is NOT parsed into a
|
|
117
|
+
// `CompanyKnowledge` field, but it is emphatically not "ignored" either --
|
|
118
|
+
// warning that it is told anyone adopting an overlay that the single field
|
|
119
|
+
// driving their adoption does nothing.
|
|
120
|
+
'canary_shape',
|
|
115
121
|
]);
|
|
116
122
|
const _HEX_COLOR_RE = /^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/;
|
|
117
123
|
const _BRAND_TEXT_MAX = 200;
|
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Framework Registry — queries the collection of supported testing frameworks.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Reads the framework catalog from `data/frameworks/registry.json`, resolved
|
|
5
|
+
* relative to this module. The source-of-truth lives at
|
|
6
|
+
* `ts/src/data/frameworks/registry.json`; `scripts/copy-data.mjs` mirrors it to
|
|
7
|
+
* `dist/data/` after `tsc`, and the offset `../data/frameworks/registry.json` is
|
|
8
|
+
* stable across all three runtimes: vitest (from `src/core/`), the dev CLI (from
|
|
9
|
+
* `dist/core/`), and the shipped npm bundle (from `dist/engine/core/`).
|
|
7
10
|
*/
|
|
8
11
|
import { readFileSync } from 'node:fs';
|
|
9
12
|
import { dirname, resolve } from 'node:path';
|
|
10
13
|
import { fileURLToPath } from 'node:url';
|
|
11
14
|
import { def } from '../util/coalesce.js';
|
|
12
15
|
import { scaffoldableFrameworks } from './scaffolder.js';
|
|
13
|
-
/** Default registry path:
|
|
16
|
+
/** Default registry path: `<module dir>/../data/frameworks/registry.json`. */
|
|
14
17
|
export function defaultRegistryPath() {
|
|
15
18
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
16
|
-
// here =
|
|
17
|
-
return resolve(here, '..', '
|
|
19
|
+
// here = <core> → sibling data/ dir (src/data in vitest, dist/data at runtime).
|
|
20
|
+
return resolve(here, '..', 'data', 'frameworks', 'registry.json');
|
|
18
21
|
}
|
|
19
22
|
export class FrameworkRegistry {
|
|
20
23
|
frameworks;
|
|
@@ -55,7 +55,7 @@ import { buildApiDelta, writeApiDelta } from './delta-emitter.js';
|
|
|
55
55
|
import { extractApiDiff } from './diff-extractor.js';
|
|
56
56
|
import { HardGateBlocked, RestBranchProtectionClient, applyHardGate, renderPlaybook, } from './hard-gate.js';
|
|
57
57
|
import { mapImpact } from './impact-mapper.js';
|
|
58
|
-
import { applySuppressions, buildFindings, buildWeakTestFindings, computeExitCode, effectiveGraphDepth, filterSkipped, filterTestUnits, findReexportOnly, loadGuardianConfig, render, scopeDiff, } from './pr-check.js';
|
|
58
|
+
import { applySuppressions, buildFindings, buildWeakTestFindings, computeExitCode, effectiveGraphDepth, filterHeuristicNoise, filterSkipped, filterTestUnits, findReexportOnly, loadGuardianConfig, render, scopeDiff, } from './pr-check.js';
|
|
59
59
|
import { RestGitHubClient, degradationAnnotation, upsertStickyComment, } from './pr-comment.js';
|
|
60
60
|
import { buildSummary } from './summary-emitter.js';
|
|
61
61
|
import { resolveTier } from './tier.js';
|
|
@@ -263,17 +263,146 @@ function authoredSentinelPath(deps, root) {
|
|
|
263
263
|
*
|
|
264
264
|
* `source === '-'` reads stdin; a path reads that file; `null` runs `git diff`
|
|
265
265
|
* and falls back to `git diff --staged` when the worktree is clean.
|
|
266
|
+
*
|
|
267
|
+
* This is the AT-DESK resolution: the working tree is the subject. `pr-check`
|
|
268
|
+
* uses {@link readPrDiff} instead, which prefers the PR diff in CI (#369).
|
|
266
269
|
*/
|
|
267
270
|
function readDiff(source, deps) {
|
|
268
271
|
if (source === '-')
|
|
269
272
|
return deps.readStdin();
|
|
270
273
|
if (source !== null)
|
|
271
274
|
return readFileSync(source, 'utf-8');
|
|
275
|
+
return readWorktreeDiff(deps);
|
|
276
|
+
}
|
|
277
|
+
/** `git diff`, falling back to `git diff --staged` on a clean worktree. */
|
|
278
|
+
function readWorktreeDiff(deps) {
|
|
272
279
|
const unstaged = deps.runGit(['diff'])?.stdout ?? '';
|
|
273
280
|
if (unstaged.trim())
|
|
274
281
|
return unstaged;
|
|
275
282
|
return deps.runGit(['diff', '--staged'])?.stdout ?? '';
|
|
276
283
|
}
|
|
284
|
+
/** True when the process looks like a CI runner rather than a dev worktree. */
|
|
285
|
+
function isCiContext(env) {
|
|
286
|
+
return Boolean(env['GITHUB_ACTIONS'] || env['CI']);
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Read `pull_request.base.sha` from the Actions event payload, if present.
|
|
290
|
+
*
|
|
291
|
+
* Optional chaining over a narrow interface (rather than an `unknown` +
|
|
292
|
+
* `typeof` ladder) keeps this at one branch per real failure mode: unreadable
|
|
293
|
+
* file, unparseable JSON, or a payload without a usable sha.
|
|
294
|
+
*/
|
|
295
|
+
function eventBaseSha(env) {
|
|
296
|
+
const eventPath = env['GITHUB_EVENT_PATH'];
|
|
297
|
+
if (!eventPath)
|
|
298
|
+
return null;
|
|
299
|
+
let sha;
|
|
300
|
+
try {
|
|
301
|
+
const event = JSON.parse(readFileSync(eventPath, 'utf-8'));
|
|
302
|
+
sha = event?.pull_request?.base?.sha;
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
return null;
|
|
306
|
+
}
|
|
307
|
+
return typeof sha === 'string' && sha.trim() ? sha.trim() : null;
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Base-rev candidates for the PR diff, most-preferred first.
|
|
311
|
+
*
|
|
312
|
+
* `origin/<ref>` leads because `actions/checkout` fetches the base branch under
|
|
313
|
+
* the remote namespace and usually does NOT create a local branch for it; the
|
|
314
|
+
* bare `<ref>` covers checkouts that do. The event payload's `base.sha` is the
|
|
315
|
+
* last resort — exact, but only present on `pull_request` events.
|
|
316
|
+
*/
|
|
317
|
+
function baseRefCandidates(env) {
|
|
318
|
+
const candidates = [];
|
|
319
|
+
const baseRef = env['GITHUB_BASE_REF']?.trim();
|
|
320
|
+
if (baseRef)
|
|
321
|
+
candidates.push(`origin/${baseRef}`, baseRef);
|
|
322
|
+
const sha = eventBaseSha(env);
|
|
323
|
+
if (sha)
|
|
324
|
+
candidates.push(sha);
|
|
325
|
+
return candidates;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Return the first base candidate that actually resolves to a commit locally.
|
|
329
|
+
*
|
|
330
|
+
* A shallow clone (`fetch-depth: 1`, the `actions/checkout` default) will NOT
|
|
331
|
+
* have the base commit, so every candidate fails `rev-parse` and we return
|
|
332
|
+
* `null` — the caller then falls back to the worktree diff and warns.
|
|
333
|
+
*/
|
|
334
|
+
function resolveBaseRev(deps) {
|
|
335
|
+
for (const candidate of baseRefCandidates(deps.env)) {
|
|
336
|
+
const res = deps.runGit([
|
|
337
|
+
'rev-parse',
|
|
338
|
+
'--verify',
|
|
339
|
+
'--quiet',
|
|
340
|
+
`${candidate}^{commit}`,
|
|
341
|
+
]);
|
|
342
|
+
if (res !== null && res.code === 0 && res.stdout.trim())
|
|
343
|
+
return candidate;
|
|
344
|
+
}
|
|
345
|
+
return null;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Resolve the diff `pr-check` should scope, preferring the PR diff in CI (#369).
|
|
349
|
+
*
|
|
350
|
+
* An explicit `--diff` (stdin or file) always wins and never shells out. With
|
|
351
|
+
* `--diff` omitted:
|
|
352
|
+
*
|
|
353
|
+
* - **In CI** with a resolvable base rev → `git diff <base>...HEAD`. The
|
|
354
|
+
* TRIPLE-dot form diffs against the merge base, so commits that land on the
|
|
355
|
+
* base branch mid-PR never appear as part of this PR's changed surface.
|
|
356
|
+
* - **Otherwise** → the at-desk working-tree diff ({@link readWorktreeDiff}).
|
|
357
|
+
*
|
|
358
|
+
* The legacy behavior was the working-tree diff unconditionally, which is empty
|
|
359
|
+
* on a clean CI checkout — the gate then scoped zero paths and exited 0, so an
|
|
360
|
+
* adopting repo could not tell a working gate from a broken one.
|
|
361
|
+
*/
|
|
362
|
+
export function readPrDiff(source, deps) {
|
|
363
|
+
if (source === '-') {
|
|
364
|
+
return { text: deps.readStdin(), origin: 'stdin', base: null };
|
|
365
|
+
}
|
|
366
|
+
if (source !== null) {
|
|
367
|
+
return { text: readFileSync(source, 'utf-8'), origin: 'file', base: null };
|
|
368
|
+
}
|
|
369
|
+
if (isCiContext(deps.env)) {
|
|
370
|
+
const base = resolveBaseRev(deps);
|
|
371
|
+
if (base !== null) {
|
|
372
|
+
const res = deps.runGit(['diff', `${base}...HEAD`]);
|
|
373
|
+
if (res !== null && res.code === 0) {
|
|
374
|
+
return { text: res.stdout, origin: 'ci-base', base };
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return { text: readWorktreeDiff(deps), origin: 'worktree', base: null };
|
|
379
|
+
}
|
|
380
|
+
const EMPTY_CI_DIFF_NOTICE = 'guardian: 0 changed paths — fell back to a working-tree `git diff`, which ' +
|
|
381
|
+
'is empty on a clean CI checkout, so NOTHING was verified. Pass ' +
|
|
382
|
+
'`--diff <base>...<head>`, or checkout with `fetch-depth: 0` so the PR base ' +
|
|
383
|
+
'ref resolves automatically.';
|
|
384
|
+
/**
|
|
385
|
+
* Warn LOUDLY when a CI run scoped zero paths off the worktree fallback (#369).
|
|
386
|
+
*
|
|
387
|
+
* Fires only for the exact broken shape — `--diff` omitted, CI detected, base
|
|
388
|
+
* rev unresolvable, and zero changed paths. A diff that DID carry paths which
|
|
389
|
+
* were then all skipped is a legitimate no-op and stays quiet.
|
|
390
|
+
*
|
|
391
|
+
* Deliberately non-blocking: it annotates (`::warning::` + step summary +
|
|
392
|
+
* stderr) rather than exiting non-zero, so adopting an engine upgrade never
|
|
393
|
+
* flips a green build red — but a silent green no-op becomes impossible.
|
|
394
|
+
*/
|
|
395
|
+
function warnIfEmptyCiDiff(resolved, unitCount, deps) {
|
|
396
|
+
if (resolved.origin !== 'worktree')
|
|
397
|
+
return;
|
|
398
|
+
if (unitCount > 0)
|
|
399
|
+
return;
|
|
400
|
+
if (!isCiContext(deps.env))
|
|
401
|
+
return;
|
|
402
|
+
deps.out(degradationAnnotation(EMPTY_CI_DIFF_NOTICE));
|
|
403
|
+
appendStepSummary(deps.env, EMPTY_CI_DIFF_NOTICE);
|
|
404
|
+
deps.err(EMPTY_CI_DIFF_NOTICE);
|
|
405
|
+
}
|
|
277
406
|
// --- analyze ------------------------------------------------------------------
|
|
278
407
|
function loadSpec(path, deps) {
|
|
279
408
|
if (!existsSync(path)) {
|
|
@@ -494,6 +623,10 @@ async function postStickyComment(findings, resolution, deps) {
|
|
|
494
623
|
appendStepSummary(deps.env, res.notice);
|
|
495
624
|
}
|
|
496
625
|
}
|
|
626
|
+
/** The gate's no-op line, shared by the pre- and post-filter exits. */
|
|
627
|
+
function nothingToVerify(skippedCount) {
|
|
628
|
+
return `guardian: nothing to verify (${skippedCount} path(s) skipped).`;
|
|
629
|
+
}
|
|
497
630
|
async function prCheckCmd(opts, deps) {
|
|
498
631
|
const [config, warning] = loadGuardianConfig(opts.config);
|
|
499
632
|
if (warning !== null) {
|
|
@@ -507,8 +640,12 @@ async function prCheckCmd(opts, deps) {
|
|
|
507
640
|
throw new CliExit(0);
|
|
508
641
|
}
|
|
509
642
|
const effectiveGate = opts.gate ?? config.pr_gate;
|
|
510
|
-
|
|
643
|
+
// #369: in CI an omitted `--diff` resolves the PR diff from the base ref;
|
|
644
|
+
// the working-tree fallback is empty on a clean checkout.
|
|
645
|
+
const resolvedDiff = readPrDiff(opts.diff ?? null, deps);
|
|
646
|
+
const diffText = resolvedDiff.text;
|
|
511
647
|
const units = scopeDiff(diffText);
|
|
648
|
+
warnIfEmptyCiDiff(resolvedDiff, units.length, deps);
|
|
512
649
|
// SC-2: drop docs/config-only units matching skipGlobs.
|
|
513
650
|
const [keptSkip, skipped] = filterSkipped(units, config.skip_globs);
|
|
514
651
|
// FIX A: drop test-path units -- a test does not itself need a test.
|
|
@@ -521,9 +658,9 @@ async function prCheckCmd(opts, deps) {
|
|
|
521
658
|
const weakFindings = config.weak_tests
|
|
522
659
|
? buildWeakTestFindings(testUnits, diffText)
|
|
523
660
|
: [];
|
|
661
|
+
const preFilterSkipped = skipped.length + testUnits.length + barrelUnits.length;
|
|
524
662
|
if (kept.length === 0 && weakFindings.length === 0) {
|
|
525
|
-
deps.out(
|
|
526
|
-
`(${skipped.length + testUnits.length + barrelUnits.length} path(s) skipped).`);
|
|
663
|
+
deps.out(nothingToVerify(preFilterSkipped));
|
|
527
664
|
throw new CliExit(0);
|
|
528
665
|
}
|
|
529
666
|
const results = resolveCoverage(kept, {
|
|
@@ -532,10 +669,21 @@ async function prCheckCmd(opts, deps) {
|
|
|
532
669
|
// (depth 1); soft stays unbounded. An explicit config value wins.
|
|
533
670
|
graphMaxDepth: effectiveGraphDepth(config, effectiveGate),
|
|
534
671
|
});
|
|
672
|
+
// #413: drop uncovered HEURISTIC verdicts on paths a naming heuristic can
|
|
673
|
+
// never judge (non-source, or an excluded glob). Coverage/graph-verified
|
|
674
|
+
// verdicts on the same paths are real evidence and survive.
|
|
675
|
+
const [scoredResults, noiseResults] = filterHeuristicNoise(results, opts.heuristicExclude ?? config.heuristic_exclude);
|
|
535
676
|
const findings = [
|
|
536
|
-
...applySuppressions(buildFindings(
|
|
677
|
+
...applySuppressions(buildFindings(scoredResults)),
|
|
537
678
|
...weakFindings,
|
|
538
679
|
];
|
|
680
|
+
// #413: if the heuristic filter consumed every scorable unit, report it as a
|
|
681
|
+
// SKIP rather than rendering an empty "0 unaddressed" report -- an adopter
|
|
682
|
+
// must be able to tell "nothing was judgeable" from "everything passed".
|
|
683
|
+
if (scoredResults.length === 0 && findings.length === 0) {
|
|
684
|
+
deps.out(nothingToVerify(preFilterSkipped + noiseResults.length));
|
|
685
|
+
throw new CliExit(0);
|
|
686
|
+
}
|
|
539
687
|
// SC-5 (PR half): resolve the requested tier against actual capability. No
|
|
540
688
|
// agent runtime exists (default NoAgentProbe), so any `pr.tier > 0` drops to
|
|
541
689
|
// tier 0 with a LOUD degradation notice.
|
|
@@ -602,7 +750,10 @@ function buildGaps(diffText, config, coveragePath, graphMaxDepth) {
|
|
|
602
750
|
if (kept.length === 0)
|
|
603
751
|
return [];
|
|
604
752
|
const results = resolveCoverage(kept, { coveragePath, graphMaxDepth });
|
|
605
|
-
|
|
753
|
+
// #413: never hand the authoring tier a heuristic FP -- a generated "test" for
|
|
754
|
+
// a config dotfile is worse noise than the finding was.
|
|
755
|
+
const [scored] = filterHeuristicNoise(results, config.heuristic_exclude);
|
|
756
|
+
return applySuppressions(buildFindings(scored));
|
|
606
757
|
}
|
|
607
758
|
/** Serialize a {@link GeneratedTest} intent for the SKILL (JSON-safe). */
|
|
608
759
|
function intentDict(intent) {
|
|
@@ -729,11 +880,15 @@ export function createGuardianCommand(depsInit = {}) {
|
|
|
729
880
|
program
|
|
730
881
|
.command('pr-check')
|
|
731
882
|
.description('Tier 0 deterministic PR guardian: scope, resolve, gate.')
|
|
732
|
-
.option('--diff <diff>', "Diff file, '-' for stdin, or omit to
|
|
883
|
+
.option('--diff <diff>', "Diff file, '-' for stdin, or omit to auto-resolve: the PR diff " +
|
|
884
|
+
'(`<base>...HEAD`) in CI, else the local working-tree `git diff`.')
|
|
733
885
|
.option('--coverage <path>', 'Coverage report path (lcov/json).')
|
|
734
886
|
.addOption(new Option('--format <fmt>', 'comment|json|text').default('comment'))
|
|
735
887
|
.addOption(new Option('--config <path>').default('harness.config.json'))
|
|
736
888
|
.option('--gate <gate>', 'Override config gate: soft|hard')
|
|
889
|
+
.option('--heuristic-exclude <glob>', 'Glob whose paths never produce a heuristic-tier finding (repeatable). ' +
|
|
890
|
+
'Replaces canary.guardian.pr.heuristicExclude for this run. ' +
|
|
891
|
+
'Coverage/graph-verified findings are unaffected.', (value, previous) => [...(previous ?? []), value])
|
|
737
892
|
.option('--post-comment', 'Post/update the sticky PR comment via the GitHub API (CI).')
|
|
738
893
|
.option('--emit-analysis', 'Write the finding record to the .harness/analyses/ channel ' +
|
|
739
894
|
'(harness handoff, #899); falls back LOUDLY to the sticky comment ' +
|
|
@@ -682,6 +682,96 @@ const TEST_PATH_RE = /(^|\/)tests?\/|(^|\/)test_[^/]*\.py$|\.test\.[^/]+$|\.spec
|
|
|
682
682
|
export function isTestPath(path) {
|
|
683
683
|
return TEST_PATH_RE.test(path);
|
|
684
684
|
}
|
|
685
|
+
/**
|
|
686
|
+
* Extensions that denote hand-authored, executable program source (#413).
|
|
687
|
+
*
|
|
688
|
+
* The membership rule is deliberately simple and defensible: **a programming
|
|
689
|
+
* language belongs; data, config, markup, and style do not.** `.sh` is in (it is
|
|
690
|
+
* executable logic — bats/shunit2 exist); `.json`, `.yaml`, `.sql`, `.css`, and
|
|
691
|
+
* `.html` are out (nothing a naming heuristic could meaningfully judge).
|
|
692
|
+
*
|
|
693
|
+
* A repo that disagrees at the margins tunes the glob layer
|
|
694
|
+
* (`canary.guardian.pr.heuristicExclude`) rather than this list.
|
|
695
|
+
*/
|
|
696
|
+
const SOURCE_EXTENSIONS = new Set([
|
|
697
|
+
// TS/JS + component dialects.
|
|
698
|
+
'.ts',
|
|
699
|
+
'.tsx',
|
|
700
|
+
'.mts',
|
|
701
|
+
'.cts',
|
|
702
|
+
'.js',
|
|
703
|
+
'.jsx',
|
|
704
|
+
'.mjs',
|
|
705
|
+
'.cjs',
|
|
706
|
+
'.vue',
|
|
707
|
+
'.svelte',
|
|
708
|
+
'.astro',
|
|
709
|
+
// Python / Ruby / PHP / Perl / Lua.
|
|
710
|
+
'.py',
|
|
711
|
+
'.pyi',
|
|
712
|
+
'.rb',
|
|
713
|
+
'.php',
|
|
714
|
+
'.pl',
|
|
715
|
+
'.pm',
|
|
716
|
+
'.lua',
|
|
717
|
+
// JVM + .NET.
|
|
718
|
+
'.java',
|
|
719
|
+
'.kt',
|
|
720
|
+
'.kts',
|
|
721
|
+
'.scala',
|
|
722
|
+
'.groovy',
|
|
723
|
+
'.clj',
|
|
724
|
+
'.cljs',
|
|
725
|
+
'.cs',
|
|
726
|
+
'.fs',
|
|
727
|
+
'.vb',
|
|
728
|
+
// Systems.
|
|
729
|
+
'.go',
|
|
730
|
+
'.rs',
|
|
731
|
+
'.c',
|
|
732
|
+
'.h',
|
|
733
|
+
'.cc',
|
|
734
|
+
'.cpp',
|
|
735
|
+
'.cxx',
|
|
736
|
+
'.hpp',
|
|
737
|
+
'.hh',
|
|
738
|
+
'.m',
|
|
739
|
+
'.mm',
|
|
740
|
+
'.swift',
|
|
741
|
+
// Functional / scientific / other.
|
|
742
|
+
'.ex',
|
|
743
|
+
'.exs',
|
|
744
|
+
'.erl',
|
|
745
|
+
'.dart',
|
|
746
|
+
'.r',
|
|
747
|
+
'.jl',
|
|
748
|
+
// Shell.
|
|
749
|
+
'.sh',
|
|
750
|
+
'.bash',
|
|
751
|
+
'.zsh',
|
|
752
|
+
'.ps1',
|
|
753
|
+
'.psm1',
|
|
754
|
+
]);
|
|
755
|
+
/**
|
|
756
|
+
* True if `path` looks like hand-authored program source (#413).
|
|
757
|
+
*
|
|
758
|
+
* Used to gate the Tier-3 naming heuristic. That heuristic asks "does any test
|
|
759
|
+
* file reference this file's stem or a top-level symbol?" — for a config
|
|
760
|
+
* dotfile, a lockfile, or a data blob there are no symbols and no test will
|
|
761
|
+
* ever name it, so the verdict is structurally always "uncovered": a guaranteed
|
|
762
|
+
* false positive rather than a signal. An extension-less file (`Makefile`,
|
|
763
|
+
* `Dockerfile`) and a bare dotfile (`.eslintrc`) are both non-source.
|
|
764
|
+
*/
|
|
765
|
+
export function isSourcePath(path) {
|
|
766
|
+
const base = basename(path);
|
|
767
|
+
// `.eslintrc` — `extname` calls this '' already, but a dotfile WITH a real
|
|
768
|
+
// extension (`.eslintrc.json`) must be judged on that extension, which the
|
|
769
|
+
// normal path handles.
|
|
770
|
+
const ext = extname(base).toLowerCase();
|
|
771
|
+
if (!ext)
|
|
772
|
+
return false;
|
|
773
|
+
return SOURCE_EXTENSIONS.has(ext);
|
|
774
|
+
}
|
|
685
775
|
/**
|
|
686
776
|
* Tier 2: derive coverage from the harness knowledge graph (`GRAPH_VERIFIED`).
|
|
687
777
|
*
|
|
@@ -28,7 +28,7 @@ import { readFileSync } from 'node:fs';
|
|
|
28
28
|
import { extname, join } from 'node:path';
|
|
29
29
|
import { readJsonWithWarning } from '../core/config-validation.js';
|
|
30
30
|
import { isAssertionFreeTest } from '../core/quality-scorer.js';
|
|
31
|
-
import { Fidelity, isTestPath, } from './coverage.js';
|
|
31
|
+
import { Fidelity, isSourcePath, isTestPath, } from './coverage.js';
|
|
32
32
|
import { Severity, severitySortKey } from './impact-mapper.js';
|
|
33
33
|
const HUNK_RE = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/;
|
|
34
34
|
// Suppression annotation: `// canary:allow-untested <reason>` or the `#`
|
|
@@ -359,6 +359,58 @@ export function filterTestUnits(units) {
|
|
|
359
359
|
}
|
|
360
360
|
return [kept, testUnits];
|
|
361
361
|
}
|
|
362
|
+
/**
|
|
363
|
+
* Default glob layer over the {@link isSourcePath} extension floor (#413).
|
|
364
|
+
*
|
|
365
|
+
* These paths carry a *source* extension but still have nothing a naming
|
|
366
|
+
* heuristic could judge: ambient type declarations have no runtime behavior,
|
|
367
|
+
* and generated clients/stubs are regenerated from a schema rather than
|
|
368
|
+
* hand-authored. An explicit `heuristicExclude` in config (even `[]`) replaces
|
|
369
|
+
* this list; the extension floor is NOT config-defeatable.
|
|
370
|
+
*/
|
|
371
|
+
export const DEFAULT_HEURISTIC_EXCLUDE_GLOBS = [
|
|
372
|
+
'**/*.d.ts',
|
|
373
|
+
'**/__generated__/**',
|
|
374
|
+
'**/generated/**',
|
|
375
|
+
'**/*.generated.*',
|
|
376
|
+
'**/*_pb2.py',
|
|
377
|
+
'**/*.pb.go',
|
|
378
|
+
];
|
|
379
|
+
/**
|
|
380
|
+
* Partition coverage results, dropping heuristic false positives (#413).
|
|
381
|
+
*
|
|
382
|
+
* A result is dropped iff ALL of:
|
|
383
|
+
*
|
|
384
|
+
* - its fidelity is `HEURISTIC` (the last-resort naming tier), AND
|
|
385
|
+
* - it is **uncovered** (a covered result raises no finding anyway), AND
|
|
386
|
+
* - its path is not program source ({@link isSourcePath}) OR it matches an
|
|
387
|
+
* `excludeGlobs` entry.
|
|
388
|
+
*
|
|
389
|
+
* The narrowness is the point. A `COVERAGE_VERIFIED` or `GRAPH_VERIFIED`
|
|
390
|
+
* verdict on the very same path rests on real evidence (an lcov row, a graph
|
|
391
|
+
* edge) and still fires — the suppression is scoped to the tier, never to the
|
|
392
|
+
* path. Returns `[kept, dropped]`, order-preserving in both.
|
|
393
|
+
*
|
|
394
|
+
* Why this matters beyond noise: the soft→hard gate promotion is earned by
|
|
395
|
+
* reviewer adjudication feeding `precision = TP / (TP + FP)`. A repo that
|
|
396
|
+
* routinely touches config files accumulated 👎 on findings that could never
|
|
397
|
+
* have been true, holding it below its promotion bar indefinitely.
|
|
398
|
+
*/
|
|
399
|
+
export function filterHeuristicNoise(results, excludeGlobs) {
|
|
400
|
+
const kept = [];
|
|
401
|
+
const dropped = [];
|
|
402
|
+
for (const result of results) {
|
|
403
|
+
const ineligible = result.fidelity === Fidelity.Heuristic &&
|
|
404
|
+
!result.covered &&
|
|
405
|
+
(!isSourcePath(result.unit.path) ||
|
|
406
|
+
excludeGlobs.some((glob) => globMatches(result.unit.path, glob)));
|
|
407
|
+
if (ineligible)
|
|
408
|
+
dropped.push(result);
|
|
409
|
+
else
|
|
410
|
+
kept.push(result);
|
|
411
|
+
}
|
|
412
|
+
return [kept, dropped];
|
|
413
|
+
}
|
|
362
414
|
/**
|
|
363
415
|
* A single guardian finding about a changed unit.
|
|
364
416
|
*
|
|
@@ -612,6 +664,46 @@ export function computeExitCode(findings, gate) {
|
|
|
612
664
|
return 0;
|
|
613
665
|
}
|
|
614
666
|
const STICKY_MARKER = '<!-- canary-pr-guardian -->';
|
|
667
|
+
// Severity → status icon for the sticky comment (encodes severity in form, not
|
|
668
|
+
// just text, so the most urgent findings read at a glance).
|
|
669
|
+
//
|
|
670
|
+
// Written as `\u{...}` escapes, not literal glyphs: this file is `.ts`, and the
|
|
671
|
+
// house rule keeps emitted non-ASCII out of non-Markdown source (see the
|
|
672
|
+
// "Output data glyphs" block in `cli.ts`). They are emitted verbatim.
|
|
673
|
+
/**
|
|
674
|
+
* Character budget for a rendered sticky comment (#457).
|
|
675
|
+
*
|
|
676
|
+
* GitHub rejects an issue/PR comment body over **65,536** characters. The post
|
|
677
|
+
* path reports that as "could not post", so an over-long body means the gate
|
|
678
|
+
* silently produces nothing on exactly the large PRs that need it most -- the
|
|
679
|
+
* same silent-green failure #369 was filed for.
|
|
680
|
+
*
|
|
681
|
+
* 60,000 leaves ~5.5k of headroom for anything appended outside `render`
|
|
682
|
+
* (degradation annotations, upsert wrappers) without inviting a body that only
|
|
683
|
+
* *just* fits and then breaks when a filename grows.
|
|
684
|
+
*
|
|
685
|
+
* The cap applies ONLY to the comment. The `--emit-analysis` JSON record is the
|
|
686
|
+
* authoritative complete set and is never truncated.
|
|
687
|
+
*/
|
|
688
|
+
export const COMMENT_CHAR_BUDGET = 60_000;
|
|
689
|
+
/** The line that accounts for findings the budget could not fit (#457). */
|
|
690
|
+
function overflowNote(omitted) {
|
|
691
|
+
return (`<sub>${EM_DASH} and ${omitted} more finding(s) omitted to keep this ` +
|
|
692
|
+
`comment under GitHub's size limit. The full set is in the analysis ` +
|
|
693
|
+
`record (\`--emit-analysis\`) and the CI logs.</sub>`);
|
|
694
|
+
}
|
|
695
|
+
const EM_DASH = '\u{2014}';
|
|
696
|
+
const RED_CIRCLE = '\u{1F534}';
|
|
697
|
+
const YELLOW_CIRCLE = '\u{1F7E1}';
|
|
698
|
+
const WHITE_CIRCLE = '\u{26AA}';
|
|
699
|
+
const BABY_CHICK = '\u{1F424}';
|
|
700
|
+
const WHITE_CHECK = '\u{2705}';
|
|
701
|
+
const SEVERITY_ICON = {
|
|
702
|
+
[Severity.CRITICAL]: RED_CIRCLE,
|
|
703
|
+
[Severity.HIGH]: RED_CIRCLE,
|
|
704
|
+
[Severity.MEDIUM]: YELLOW_CIRCLE,
|
|
705
|
+
[Severity.LOW]: WHITE_CIRCLE,
|
|
706
|
+
};
|
|
615
707
|
/**
|
|
616
708
|
* Escape every non-ASCII (>= U+0080) code unit to a `\uXXXX` sequence, matching
|
|
617
709
|
* Python's `json.dumps(..., ensure_ascii=True)` (the library default). `JSON`
|
|
@@ -661,33 +753,79 @@ export function render(findings, fmt, tier = 0, degradedNotice = null) {
|
|
|
661
753
|
}
|
|
662
754
|
const active = ordered.filter((f) => !f.suppressed);
|
|
663
755
|
const suppressed = ordered.filter((f) => f.suppressed);
|
|
756
|
+
// A finding's file label shows the path once, appending the unit only when
|
|
757
|
+
// it is a distinct symbol within the file (never `path (path)`).
|
|
758
|
+
const fileLabel = (f) => f.unit && f.unit !== f.path
|
|
759
|
+
? `\`${f.path}\` → \`${f.unit}\``
|
|
760
|
+
: `\`${f.path}\``;
|
|
761
|
+
const cell = (s) => s.replace(/\|/g, '\\|');
|
|
762
|
+
const CONFIDENCE_NOTE = 'Confidence — **coverage-verified**: measured from a real coverage run · ' +
|
|
763
|
+
'**graph-verified**: inferred from the call graph · **heuristic**: filename ' +
|
|
764
|
+
`guess (lowest). tier ${tier}: deterministic check, no LLM.`;
|
|
765
|
+
const footerLine = `<sub>${CONFIDENCE_NOTE}${degradedNotice ? ` ${EM_DASH} ${degradedNotice}` : ''}</sub>`;
|
|
664
766
|
if (fmt === 'comment') {
|
|
665
|
-
const
|
|
666
|
-
lines
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
`
|
|
672
|
-
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
767
|
+
const fileCount = new Set(active.map((f) => f.path)).size;
|
|
768
|
+
const lines = [STICKY_MARKER];
|
|
769
|
+
if (active.length === 0) {
|
|
770
|
+
lines.push(`## ${BABY_CHICK} Canary PR Guardian ${EM_DASH} ` +
|
|
771
|
+
`${WHITE_CHECK} no test-coverage gaps`);
|
|
772
|
+
if (suppressed.length) {
|
|
773
|
+
lines.push(`_${suppressed.length} finding(s) suppressed as intentional._`);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
else {
|
|
777
|
+
const noun = fileCount === 1 ? 'file needs' : 'files need';
|
|
778
|
+
lines.push(`## ${BABY_CHICK} Canary PR Guardian ${EM_DASH} ` +
|
|
779
|
+
`${fileCount} ${noun} test coverage`);
|
|
780
|
+
lines.push('These lines were changed by this PR but no test exercises them. Add or ' +
|
|
781
|
+
'extend a test that covers them, or reply ' +
|
|
782
|
+
'`/guardian suppress <file> <reason>` if they are intentionally untested.');
|
|
783
|
+
lines.push('', '| Sev | File | What is uncovered | Confidence |', '| --- | --- | --- | --- |');
|
|
784
|
+
// #457: fill rows against a character budget instead of emitting all of
|
|
785
|
+
// them. `active` is already severity-ordered, so the rows that survive
|
|
786
|
+
// are the most severe -- a critical finding is never dropped to make room
|
|
787
|
+
// for a low one.
|
|
788
|
+
const suppressedNote = suppressed.length
|
|
789
|
+
? `\n\n<sub>${suppressed.length} finding(s) suppressed as intentional and not counted above.</sub>`
|
|
790
|
+
: '';
|
|
791
|
+
// Reserved so the tail always fits: footer, suppressed note, and a
|
|
792
|
+
// worst-case overflow line (the real one is shorter).
|
|
793
|
+
const reserve = footerLine.length +
|
|
794
|
+
suppressedNote.length +
|
|
795
|
+
overflowNote(active.length).length +
|
|
796
|
+
4;
|
|
797
|
+
let used = lines.join('\n').length;
|
|
798
|
+
let shown = 0;
|
|
799
|
+
for (const f of active) {
|
|
800
|
+
const row = `| ${SEVERITY_ICON[f.severity] ?? ''} ${f.severity} | ${cell(fileLabel(f))} | ${cell(f.evidence)} | ${f.fidelity} |`;
|
|
801
|
+
if (used + row.length + 1 + reserve > COMMENT_CHAR_BUDGET)
|
|
802
|
+
break;
|
|
803
|
+
lines.push(row);
|
|
804
|
+
used += row.length + 1;
|
|
805
|
+
shown += 1;
|
|
806
|
+
}
|
|
807
|
+
const omitted = active.length - shown;
|
|
808
|
+
if (omitted > 0)
|
|
809
|
+
lines.push('', overflowNote(omitted));
|
|
810
|
+
if (suppressed.length) {
|
|
811
|
+
lines.push('', `<sub>${suppressed.length} finding(s) suppressed as intentional and not counted above.</sub>`);
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
lines.push('', footerLine);
|
|
678
815
|
return lines.join('\n');
|
|
679
816
|
}
|
|
680
817
|
// fmt == "text" (default fallback): plain, no markdown/HTML.
|
|
681
818
|
const lines = [
|
|
682
|
-
|
|
683
|
-
|
|
819
|
+
active.length === 0
|
|
820
|
+
? 'Canary PR Guardian — no test-coverage gaps'
|
|
821
|
+
: `Canary PR Guardian — ${new Set(active.map((f) => f.path)).size} file(s) need test coverage`,
|
|
684
822
|
];
|
|
685
823
|
for (const finding of ordered) {
|
|
824
|
+
const unit = finding.unit && finding.unit !== finding.path ? ` → ${finding.unit}` : '';
|
|
686
825
|
const mark = finding.suppressed ? ' (suppressed)' : '';
|
|
687
|
-
lines.push(`[${finding.severity}] ${finding.path} (${finding.
|
|
688
|
-
`[${finding.fidelity}] ${finding.evidence}${mark}`);
|
|
826
|
+
lines.push(`[${finding.severity}] ${finding.path}${unit} — ${finding.evidence} (${finding.fidelity})${mark}`);
|
|
689
827
|
}
|
|
690
|
-
let footer = `tier ${tier}`;
|
|
828
|
+
let footer = `tier ${tier}: deterministic check, no LLM`;
|
|
691
829
|
if (degradedNotice)
|
|
692
830
|
footer += ` - ${degradedNotice}`;
|
|
693
831
|
lines.push(footer);
|
|
@@ -716,9 +854,30 @@ export const DEFAULT_SKIP_GLOBS = [
|
|
|
716
854
|
'**/*.snap',
|
|
717
855
|
// Generated slash-command artifacts and harness state — regenerated from a
|
|
718
856
|
// tracked source (skill.yaml / graph scans), never hand-authored, so a
|
|
719
|
-
// covering test makes no sense.
|
|
857
|
+
// covering test makes no sense. `**/.harness/**` also catches harness state
|
|
858
|
+
// nested under a subproject (e.g. `services/neo/.harness/…`), which the
|
|
859
|
+
// top-level `.harness/**` misses (#413).
|
|
720
860
|
'agents/commands/**',
|
|
721
861
|
'.harness/**',
|
|
862
|
+
'**/.harness/**',
|
|
863
|
+
// Dotfile config/metadata at any depth (.gitignore, .env, .eslintrc,
|
|
864
|
+
// .neorc*, .dockerignore, .npmrc, …). None carry testable code, so the
|
|
865
|
+
// heuristic tier's "no test file references this" is a false positive on them
|
|
866
|
+
// (#413 — observed on `.gitignore` / `.neorc.dev`). Matches only files whose
|
|
867
|
+
// basename starts with a dot, not source inside a dot-directory.
|
|
868
|
+
'**/.*',
|
|
869
|
+
// Build/tooling config files (not authored product logic).
|
|
870
|
+
'**/*.config.js',
|
|
871
|
+
'**/*.config.ts',
|
|
872
|
+
'**/*.config.mjs',
|
|
873
|
+
'**/*.config.cjs',
|
|
874
|
+
// Test fixtures / mocks and generated code — noise, not logic under test.
|
|
875
|
+
'**/fixtures/**',
|
|
876
|
+
'**/__fixtures__/**',
|
|
877
|
+
'**/__mocks__/**',
|
|
878
|
+
'**/testdata/**',
|
|
879
|
+
'**/generated/**',
|
|
880
|
+
'**/__generated__/**',
|
|
722
881
|
];
|
|
723
882
|
/**
|
|
724
883
|
* Parsed `canary.guardian` config block.
|
|
@@ -744,6 +903,10 @@ export class GuardianConfig {
|
|
|
744
903
|
precommit_gate;
|
|
745
904
|
coverage_paths;
|
|
746
905
|
skip_globs;
|
|
906
|
+
// #413: glob layer suppressing the HEURISTIC tier only (a source path that
|
|
907
|
+
// still has nothing a naming heuristic can judge). Distinct from
|
|
908
|
+
// `skip_globs`, which drops a path from the gate entirely at every tier.
|
|
909
|
+
heuristic_exclude;
|
|
747
910
|
// #320: bound the graph-coverage reverse-BFS. `null` means "gate-derived"
|
|
748
911
|
// (see {@link effectiveGraphDepth} — hard→1 direct edge, soft→unbounded); an
|
|
749
912
|
// explicit int here overrides the gate default on BOTH surfaces.
|
|
@@ -758,6 +921,9 @@ export class GuardianConfig {
|
|
|
758
921
|
this.precommit_gate = init.precommit_gate ?? 'soft';
|
|
759
922
|
this.coverage_paths = init.coverage_paths ?? [];
|
|
760
923
|
this.skip_globs = init.skip_globs ?? [...DEFAULT_SKIP_GLOBS];
|
|
924
|
+
this.heuristic_exclude = init.heuristic_exclude ?? [
|
|
925
|
+
...DEFAULT_HEURISTIC_EXCLUDE_GLOBS,
|
|
926
|
+
];
|
|
761
927
|
this.graph_coverage_max_depth = init.graph_coverage_max_depth ?? null;
|
|
762
928
|
}
|
|
763
929
|
}
|
|
@@ -927,6 +1093,14 @@ export function loadGuardianConfig(configPath = 'harness.config.json') {
|
|
|
927
1093
|
config.pr_gate = coerceGate(pr['gate'], config.pr_gate, 'pr.gate', warnings);
|
|
928
1094
|
}
|
|
929
1095
|
config.weak_tests = pyTruthy(pyGet(pr, 'weakTests', config.weak_tests));
|
|
1096
|
+
// #413: same present-vs-absent contract as `skipGlobs` (FIX B) — absent
|
|
1097
|
+
// keeps the built-in default, an explicit list (including `[]`) is honored
|
|
1098
|
+
// verbatim so `heuristicExclude: []` means "no glob layer". The
|
|
1099
|
+
// {@link isSourcePath} extension floor is unaffected either way.
|
|
1100
|
+
const heuristicExclude = pr['heuristicExclude'];
|
|
1101
|
+
if (Array.isArray(heuristicExclude)) {
|
|
1102
|
+
config.heuristic_exclude = heuristicExclude.map((g) => String(g));
|
|
1103
|
+
}
|
|
930
1104
|
}
|
|
931
1105
|
const precommit = pyGet(block, 'preCommit', {});
|
|
932
1106
|
if (isRecord(precommit)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "canary-test-cli",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.2.0",
|
|
4
4
|
"description": "Canary — AI-powered test automation agent",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -36,8 +36,7 @@
|
|
|
36
36
|
},
|
|
37
37
|
"files": [
|
|
38
38
|
"bin/canary.js",
|
|
39
|
-
"dist/"
|
|
40
|
-
"agent/frameworks/registry.json"
|
|
39
|
+
"dist/"
|
|
41
40
|
],
|
|
42
41
|
"dependencies": {
|
|
43
42
|
"@modelcontextprotocol/sdk": "^1.30.0",
|
|
File without changes
|