canary-test-cli 7.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +23 -4
- package/agents/skills/claude-code/canary-batwoman/SKILL.md +119 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +23 -16
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +3 -1
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +20 -3
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +1 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +15 -0
- package/agents/skills/claude-code/canary-screech/SKILL.md +109 -0
- package/agents/skills/claude-code/canary-screech/scripts/blast.mjs +125 -0
- package/agents/skills/claude-code/canary-screech/scripts/cli.mjs +128 -0
- package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs +97 -0
- package/agents/skills/claude-code/canary-screech/scripts/history.mjs +73 -0
- package/agents/skills/claude-code/canary-screech/scripts/redness.mjs +94 -0
- package/agents/skills/lib/parse-args.mjs +200 -139
- package/dist/engine/analysis/batwoman/audit.js +39 -0
- package/dist/engine/analysis/batwoman/closure.js +159 -0
- package/dist/engine/analysis/batwoman/gh-history.js +119 -0
- package/dist/engine/analysis/batwoman/probes.js +195 -0
- package/dist/engine/analysis/batwoman/registry.js +142 -0
- package/dist/engine/analysis/batwoman/render.js +194 -0
- package/dist/engine/analysis/batwoman/run-window.js +122 -0
- package/dist/engine/analysis/batwoman/text.js +84 -0
- package/dist/engine/analysis/batwoman/triggers.js +122 -0
- package/dist/engine/analysis/batwoman/verdict.js +64 -0
- package/dist/engine/analysis/cli.js +47 -14
- package/dist/engine/analysis/gh-flaky/gh-run-attempts.js +206 -0
- package/dist/engine/batwoman-cli.js +119 -0
- package/dist/engine/ci-ready-cli.js +71 -0
- package/dist/engine/cli-commands.js +46 -7
- package/dist/engine/cli.core.js +16 -0
- package/dist/engine/company-knowledge-cli.js +10 -2
- package/dist/engine/core/ci-ready.js +112 -0
- package/dist/engine/core/company-knowledge.js +8 -0
- package/dist/engine/core/migrator.js +147 -20
- package/dist/engine/core/permission-matrix.js +219 -0
- package/dist/engine/core/quality-scorer.js +13 -18
- package/dist/engine/core/scaling-curve.js +143 -0
- package/dist/engine/core/string-literals.js +3 -1
- package/dist/engine/core/vacuity-scanner.js +151 -6
- package/dist/engine/core/workflow-discovery.js +41 -23
- package/dist/engine/guardian/adjudication-github.js +136 -0
- package/dist/engine/guardian/adjudication.js +119 -340
- package/dist/engine/guardian/cli.js +180 -264
- package/dist/engine/guardian/coverage.js +2 -1
- package/dist/engine/guardian/diff-coverage/coverage-delta.js +162 -0
- package/dist/engine/guardian/diff-coverage/formats/cobertura.js +45 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +25 -21
- package/dist/engine/guardian/diff-coverage/paths.js +5 -9
- package/dist/engine/guardian/diff-coverage/report-tier.js +88 -12
- package/dist/engine/guardian/diff-extractor.js +31 -32
- package/dist/engine/guardian/pr-check.js +262 -430
- package/dist/engine/guardian/pr-comment.js +35 -58
- package/dist/engine/guardian/weak-test.js +236 -0
- package/dist/engine/mcp-server.js +67 -4
- package/dist/engine/permission-matrix-cli.js +51 -0
- package/dist/engine/scaling-curve-cli.js +147 -0
- package/dist/engine/skills-cli.js +48 -32
- package/dist/engine/workflow-cli.js +85 -65
- package/package.json +1 -1
|
@@ -19,9 +19,9 @@
|
|
|
19
19
|
* test never terminates the process.
|
|
20
20
|
*
|
|
21
21
|
* Command surface (kebab-case names; the first seven match the shipping Typer
|
|
22
|
-
* CLI, the last
|
|
22
|
+
* CLI, the last is a TS-native addition, derived per ADR 0025):
|
|
23
23
|
* analyze | validate-coverage | harden-gate | pr-check | author-plan |
|
|
24
|
-
* mark-authored | watch |
|
|
24
|
+
* mark-authored | watch | precision.
|
|
25
25
|
*
|
|
26
26
|
* Python->TS nuances honored:
|
|
27
27
|
* - `json.dumps(obj, indent=2)` -> `ensureAscii(JSON.stringify(obj, null, 2))`
|
|
@@ -50,16 +50,18 @@ import { Command, Option } from 'commander';
|
|
|
50
50
|
import { load as loadYaml } from 'js-yaml';
|
|
51
51
|
import pc from 'picocolors';
|
|
52
52
|
import { AuthoringContext, InSessionAgentProbe, InSessionAgentTier, decideBlock, } from './agent-tier.js';
|
|
53
|
-
import {
|
|
53
|
+
import { deriveReport, renderReport } from './adjudication.js';
|
|
54
|
+
import { GitHubAdjudicationSource, collectEvidence, } from './adjudication-github.js';
|
|
54
55
|
import { emitAnalysis } from './analysis-emit.js';
|
|
55
56
|
import { EXIT_ABSTAINED, gateOutcome, } from '../core/gate-result.js';
|
|
56
|
-
import { coverageDegradedNotice, resolveCoverage, resolveCoverageWithInput, validateCoverageJson, } from './coverage.js';
|
|
57
|
+
import { coverageDegradedNotice, coverageDeltaNotice, isSourcePath, resolveCoverage, resolveCoverageDelta, resolveCoverageWithInput, validateCoverageJson, } from './coverage.js';
|
|
57
58
|
import { buildApiDelta, writeApiDelta } from './delta-emitter.js';
|
|
58
59
|
import { extractApiDiff } from './diff-extractor.js';
|
|
59
60
|
import { HardGateAbstained, HardGateBlocked, RestBranchProtectionClient, applyHardGate, renderPlaybook, } from './hard-gate.js';
|
|
60
61
|
import { mapImpact } from './impact-mapper.js';
|
|
61
62
|
import { ensureAscii } from '../util/ensure-ascii.js';
|
|
62
|
-
import { MERGE_REF_WARNING, provenanceLine, applySuppressions, buildFindings,
|
|
63
|
+
import { MERGE_REF_WARNING, provenanceLine, applySuppressions, buildFindings, buildRegressionFindings, computeExitCode, effectiveGraphDepth, filterHeuristicNoise, filterSkipped, filterTestSupportUnits, filterTestUnits, filterTypeOnlyUnits, findReexportOnly, isCoverageAbstention, loadGuardianConfig, renderFindings, scopeDiff, } from './pr-check.js';
|
|
64
|
+
import { buildWeakTestFindings } from './weak-test.js';
|
|
63
65
|
import { RestGitHubClient, degradationAnnotation, upsertStickyComment, } from './pr-comment.js';
|
|
64
66
|
import { buildSummary } from './summary-emitter.js';
|
|
65
67
|
import { resolveTier } from './tier.js';
|
|
@@ -101,6 +103,33 @@ export class WatchInterruptError extends Error {
|
|
|
101
103
|
this.name = 'WatchInterruptError';
|
|
102
104
|
}
|
|
103
105
|
}
|
|
106
|
+
/** Run `git`; `null` when the binary is missing (Python OSError fail-safe). */
|
|
107
|
+
function spawnGit(args, cwd) {
|
|
108
|
+
const res = spawnSync('git', args, {
|
|
109
|
+
encoding: 'utf-8',
|
|
110
|
+
maxBuffer: Infinity,
|
|
111
|
+
...(cwd ? { cwd } : {}),
|
|
112
|
+
});
|
|
113
|
+
if (res.error)
|
|
114
|
+
return null;
|
|
115
|
+
return { code: res.status ?? 1, stdout: res.stdout ?? '' };
|
|
116
|
+
}
|
|
117
|
+
/** Run `gh` with a 30s timeout; `failed` when it could not be spawned. */
|
|
118
|
+
function spawnGh(args) {
|
|
119
|
+
const res = spawnSync('gh', args, {
|
|
120
|
+
encoding: 'utf-8',
|
|
121
|
+
timeout: 30_000,
|
|
122
|
+
maxBuffer: Infinity,
|
|
123
|
+
});
|
|
124
|
+
if (res.error)
|
|
125
|
+
return { status: null, stdout: '', stderr: '', failed: true };
|
|
126
|
+
return {
|
|
127
|
+
status: res.status,
|
|
128
|
+
stdout: res.stdout ?? '',
|
|
129
|
+
stderr: res.stderr ?? '',
|
|
130
|
+
failed: false,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
104
133
|
/** Process-backed defaults for production (the `guardianCommand` export). */
|
|
105
134
|
export function defaultDeps() {
|
|
106
135
|
return {
|
|
@@ -116,34 +145,10 @@ export function defaultDeps() {
|
|
|
116
145
|
},
|
|
117
146
|
env: process.env,
|
|
118
147
|
cwd: () => process.cwd(),
|
|
119
|
-
runGit:
|
|
120
|
-
|
|
121
|
-
encoding: 'utf-8',
|
|
122
|
-
maxBuffer: Infinity,
|
|
123
|
-
...(cwd ? { cwd } : {}),
|
|
124
|
-
});
|
|
125
|
-
if (res.error)
|
|
126
|
-
return null; // missing binary -> Python OSError fail-safe
|
|
127
|
-
return { code: res.status ?? 1, stdout: res.stdout ?? '' };
|
|
128
|
-
},
|
|
129
|
-
runGh: (args) => {
|
|
130
|
-
const res = spawnSync('gh', args, {
|
|
131
|
-
encoding: 'utf-8',
|
|
132
|
-
timeout: 30_000,
|
|
133
|
-
maxBuffer: Infinity,
|
|
134
|
-
});
|
|
135
|
-
if (res.error) {
|
|
136
|
-
return { status: null, stdout: '', stderr: '', failed: true };
|
|
137
|
-
}
|
|
138
|
-
return {
|
|
139
|
-
status: res.status,
|
|
140
|
-
stdout: res.stdout ?? '',
|
|
141
|
-
stderr: res.stderr ?? '',
|
|
142
|
-
failed: false,
|
|
143
|
-
};
|
|
144
|
-
},
|
|
148
|
+
runGit: spawnGit,
|
|
149
|
+
runGh: spawnGh,
|
|
145
150
|
buildCommentClient: (repo, prNumber) => new RestGitHubClient(repo, prNumber, process.env['GITHUB_TOKEN'] ?? ''),
|
|
146
|
-
|
|
151
|
+
buildAdjudicationSource: (repo, token) => new GitHubAdjudicationSource(repo, token),
|
|
147
152
|
buildBranchProtectionClient: (repo, token) => new RestBranchProtectionClient(repo, token),
|
|
148
153
|
makeAgentTier: () => new InSessionAgentTier(),
|
|
149
154
|
sleep: (secs) => new Promise((resolve) => setTimeout(resolve, secs * 1000)),
|
|
@@ -320,20 +325,9 @@ function headSha(deps, root) {
|
|
|
320
325
|
return res.stdout.trim().toLowerCase() || null;
|
|
321
326
|
}
|
|
322
327
|
/**
|
|
323
|
-
* Is the loop guard live
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
* This is the surviving half of the stage-and-block-once contract (#456). The
|
|
327
|
-
* component that CLEARED the sentinel on the next commit
|
|
328
|
-
* (`hooks/guardian_precommit.py`) was deleted as dead code in #449, which left
|
|
329
|
-
* `author-plan` fail-closed forever: author once in a clone and Tier-2 authoring
|
|
330
|
-
* never ran again. Stamping HEAD makes the guard self-expiring -- once the human
|
|
331
|
-
* reviews and commits the staged tests, `HEAD` moves, the stamp stops matching,
|
|
332
|
-
* and authoring re-enables itself with no manual step and no hook.
|
|
333
|
-
*
|
|
334
|
-
* Every unverifiable state FAILS OPEN (returns `false`, authoring allowed):
|
|
335
|
-
* missing or unreadable sentinel, a malformed/absent `HEAD` header, or a `HEAD`
|
|
336
|
-
* we cannot resolve. Fail-closed here is exactly the bug being fixed.
|
|
328
|
+
* Is the loop guard live, i.e. is the sentinel stamped at the CURRENT `HEAD`
|
|
329
|
+
* (#456)? Stamping HEAD makes the guard self-expiring once the staged tests are
|
|
330
|
+
* committed. Every unverifiable state FAILS OPEN (returns `false`).
|
|
337
331
|
*/
|
|
338
332
|
function authoredSentinelActive(deps, root) {
|
|
339
333
|
let body;
|
|
@@ -404,20 +398,10 @@ function resolveHeadSha(deps) {
|
|
|
404
398
|
return sha || null;
|
|
405
399
|
}
|
|
406
400
|
/**
|
|
407
|
-
* True when
|
|
408
|
-
*
|
|
409
|
-
*
|
|
410
|
-
*
|
|
411
|
-
* merged with the PR head — unless the caller passes an explicit `ref`. Any
|
|
412
|
-
* diff taken to that HEAD includes every commit merged into the base branch
|
|
413
|
-
* since the base sha, because the triple-dot merge base degenerates to the base
|
|
414
|
-
* sha itself (it is an ancestor of the merge commit). A one-file docs PR was
|
|
415
|
-
* analyzed as 43 files that way.
|
|
416
|
-
*
|
|
417
|
-
* Detection is a comparison, not a heuristic: the event payload states the PR
|
|
418
|
-
* head sha outright, so a HEAD that differs from it is diffing something else.
|
|
419
|
-
* Returns false whenever either side is unknown — an undetectable case must not
|
|
420
|
-
* masquerade as a detected-clean one.
|
|
401
|
+
* True when HEAD is a `pull_request` MERGE REF, not the PR head (#761): a diff
|
|
402
|
+
* to `refs/pull/<n>/merge` sweeps in every commit merged into base since the
|
|
403
|
+
* base sha (a one-file PR read as 43 files). A comparison against the event's
|
|
404
|
+
* PR head sha, false whenever either side is unknown.
|
|
421
405
|
*/
|
|
422
406
|
export function detectMergeRef(headSha, deps) {
|
|
423
407
|
if (deps.env['GITHUB_EVENT_NAME'] !== 'pull_request')
|
|
@@ -427,6 +411,28 @@ export function detectMergeRef(headSha, deps) {
|
|
|
427
411
|
return false;
|
|
428
412
|
return declared !== headSha;
|
|
429
413
|
}
|
|
414
|
+
/**
|
|
415
|
+
* The event-declared PR head, when it resolves to a local commit (#883).
|
|
416
|
+
*
|
|
417
|
+
* Diffing to it instead of `HEAD` judges the PR's own changes rather than the
|
|
418
|
+
* merge ref's widened set. Null outside a `pull_request` event or when the sha
|
|
419
|
+
* was not fetched (a shallow checkout) — the caller then diffs to `HEAD` and
|
|
420
|
+
* {@link detectMergeRef} still discloses the widening.
|
|
421
|
+
*/
|
|
422
|
+
function resolvePrHead(deps) {
|
|
423
|
+
if (deps.env['GITHUB_EVENT_NAME'] !== 'pull_request')
|
|
424
|
+
return null;
|
|
425
|
+
const sha = eventHeadSha(deps.env);
|
|
426
|
+
if (!sha)
|
|
427
|
+
return null;
|
|
428
|
+
const res = deps.runGit([
|
|
429
|
+
'rev-parse',
|
|
430
|
+
'--verify',
|
|
431
|
+
'--quiet',
|
|
432
|
+
`${sha}^{commit}`,
|
|
433
|
+
]);
|
|
434
|
+
return res !== null && res.code === 0 ? sha : null;
|
|
435
|
+
}
|
|
430
436
|
/** True when the process looks like a CI runner rather than a dev worktree. */
|
|
431
437
|
function isCiContext(env) {
|
|
432
438
|
return Boolean(env['GITHUB_ACTIONS'] || env['CI']);
|
|
@@ -491,19 +497,9 @@ function resolveBaseRev(deps) {
|
|
|
491
497
|
return null;
|
|
492
498
|
}
|
|
493
499
|
/**
|
|
494
|
-
* Resolve the diff `pr-check` should scope
|
|
495
|
-
*
|
|
496
|
-
*
|
|
497
|
-
* `--diff` omitted:
|
|
498
|
-
*
|
|
499
|
-
* - **In CI** with a resolvable base rev → `git diff <base>...HEAD`. The
|
|
500
|
-
* TRIPLE-dot form diffs against the merge base, so commits that land on the
|
|
501
|
-
* base branch mid-PR never appear as part of this PR's changed surface.
|
|
502
|
-
* - **Otherwise** → the at-desk working-tree diff ({@link readWorktreeDiff}).
|
|
503
|
-
*
|
|
504
|
-
* The legacy behavior was the working-tree diff unconditionally, which is empty
|
|
505
|
-
* on a clean CI checkout — the gate then scoped zero paths and exited 0, so an
|
|
506
|
-
* adopting repo could not tell a working gate from a broken one.
|
|
500
|
+
* Resolve the diff `pr-check` should scope (#369). An explicit `--diff` wins;
|
|
501
|
+
* in CI with a base rev it is `git diff <base>...HEAD` (merge base, so commits
|
|
502
|
+
* landing on base mid-PR stay out); otherwise the working-tree diff.
|
|
507
503
|
*/
|
|
508
504
|
export function readPrDiff(source, deps) {
|
|
509
505
|
if (source === '-') {
|
|
@@ -515,9 +511,10 @@ export function readPrDiff(source, deps) {
|
|
|
515
511
|
if (isCiContext(deps.env)) {
|
|
516
512
|
const base = resolveBaseRev(deps);
|
|
517
513
|
if (base !== null) {
|
|
518
|
-
const
|
|
514
|
+
const head = resolvePrHead(deps);
|
|
515
|
+
const res = deps.runGit(['diff', `${base}...${head ?? 'HEAD'}`]);
|
|
519
516
|
if (res !== null && res.code === 0) {
|
|
520
|
-
return { text: res.stdout, origin: 'ci-base', base };
|
|
517
|
+
return { text: res.stdout, origin: 'ci-base', base, head };
|
|
521
518
|
}
|
|
522
519
|
}
|
|
523
520
|
}
|
|
@@ -740,20 +737,6 @@ function validateCoverageCmd(path, opts, deps) {
|
|
|
740
737
|
throw new CliExitError(1);
|
|
741
738
|
}
|
|
742
739
|
}
|
|
743
|
-
/**
|
|
744
|
-
* Print the precision evidence the promotion contract depends on (#490).
|
|
745
|
-
*
|
|
746
|
-
* The soft→hard promotion is earned by reviewer adjudication feeding
|
|
747
|
-
* `precision = TP / (TP + FP)`; before #490 that contract lived only in a
|
|
748
|
-
* comment with nothing feeding it. This surfaces the measured number — or an
|
|
749
|
-
* honest `unknown` over an empty sample — in the readiness output. Advisory:
|
|
750
|
-
* it informs the operator's decision, it does not block the registration.
|
|
751
|
-
*/
|
|
752
|
-
function reportPrecisionEvidence(analysesDir, deps) {
|
|
753
|
-
const summary = summarizePrecision(loadAdjudicationRecords(analysesDir));
|
|
754
|
-
const line = renderPrecision(summary);
|
|
755
|
-
deps.out(summary.precision === null ? pc.yellow(line) : line);
|
|
756
|
-
}
|
|
757
740
|
async function hardenGateCmd(opts, deps) {
|
|
758
741
|
const repo = opts.repo;
|
|
759
742
|
if (!repo) {
|
|
@@ -761,8 +744,9 @@ async function hardenGateCmd(opts, deps) {
|
|
|
761
744
|
throw new CliExitError(2);
|
|
762
745
|
}
|
|
763
746
|
const playbook = renderPlaybook(repo, opts.branch, opts.check);
|
|
764
|
-
//
|
|
765
|
-
|
|
747
|
+
// ADR 0025: precision is derived from merged PRs, not stored locally.
|
|
748
|
+
deps.out(pc.yellow(`guardian precision: not measured here ${EM_DASH} run ` +
|
|
749
|
+
'`canary guardian precision` (or the weekly Guardian precision workflow).'));
|
|
766
750
|
if (!opts.apply) {
|
|
767
751
|
deps.out(`${pc.bold('Dry run')} ${EM_DASH} would require the '${opts.check}' check on ${repo}@${opts.branch}.`);
|
|
768
752
|
deps.out('On --apply this merges into existing protection (or creates minimal ' +
|
|
@@ -813,65 +797,26 @@ async function hardenGateCmd(opts, deps) {
|
|
|
813
797
|
'(or run pr-check --gate hard).'));
|
|
814
798
|
}
|
|
815
799
|
/**
|
|
816
|
-
*
|
|
817
|
-
*
|
|
818
|
-
*
|
|
819
|
-
* channel) — an operator who asked for a collection must know it did not land.
|
|
800
|
+
* Derive guardian precision from merged PRs on demand (ADR 0025): the sticky's
|
|
801
|
+
* first vs last revision, the merged diff's suppressions, and nothing stored.
|
|
802
|
+
* Exits 2 without a repo or token rather than reporting an empty sample.
|
|
820
803
|
*/
|
|
821
|
-
async function
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
if (ctx !== null) {
|
|
827
|
-
repo = repo ?? ctx[0];
|
|
828
|
-
prNumber = prNumber ?? ctx[1];
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
|
-
if (!repo || prNumber === undefined) {
|
|
832
|
-
deps.out(`${pc.red(pc.bold(`${CROSS} no PR context`))} ${EM_DASH} pass --repo and --pr, or run in Actions.`);
|
|
804
|
+
async function precisionCmd(opts, deps) {
|
|
805
|
+
const token = deps.env['GITHUB_TOKEN'];
|
|
806
|
+
if (!opts.repo || !token) {
|
|
807
|
+
deps.out(`${pc.red(pc.bold(`${CROSS} precision needs --repo and GITHUB_TOKEN`))} ` +
|
|
808
|
+
`${EM_DASH} nothing was measured.`);
|
|
833
809
|
throw new CliExitError(2);
|
|
834
810
|
}
|
|
835
|
-
const
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
else if (res.action === 'collected' && res.record) {
|
|
845
|
-
deps.out(pc.green(`${CHECK} adjudication recorded (${res.record.tp} up / ` +
|
|
846
|
-
`${res.record.fp} down, ${res.record.granularity}-level) ` +
|
|
847
|
-
`${RIGHT_ARROW} ${res.path}`));
|
|
848
|
-
}
|
|
849
|
-
else if (res.action === 'no-comment') {
|
|
850
|
-
deps.out(`guardian: no sticky comment on ${repo}#${prNumber} ${EM_DASH} nothing to adjudicate.`);
|
|
851
|
-
}
|
|
852
|
-
else if (res.action === 'no-reactions') {
|
|
853
|
-
deps.out(`guardian: sticky comment on ${repo}#${prNumber} has no reviewer ` +
|
|
854
|
-
`verdicts yet ${EM_DASH} nothing recorded (no reaction is neutral, ` +
|
|
855
|
-
`not a data point).`);
|
|
856
|
-
}
|
|
857
|
-
if (res.action === 'unavailable') {
|
|
858
|
-
deps.out(pc.red(pc.bold(`${CROSS} ${res.notice ?? 'not persisted'}`)));
|
|
859
|
-
throw new CliExitError(1);
|
|
860
|
-
}
|
|
861
|
-
}
|
|
862
|
-
/** Aggregate the persisted adjudications into the promotion evidence (#490). */
|
|
863
|
-
function precisionCmd(opts, deps) {
|
|
864
|
-
const analysesDir = resolveAnalysesDir(opts.analysesDir, deps);
|
|
865
|
-
const records = loadAdjudicationRecords(analysesDir);
|
|
866
|
-
const summary = summarizePrecision(records);
|
|
867
|
-
if (opts.json) {
|
|
868
|
-
// `precision: null` is the honest zero-denominator value — consumers must
|
|
869
|
-
// treat it as unknown, never as 1.0 (#490).
|
|
870
|
-
deps.out(ensureAscii(JSON.stringify({ ...summary, records: records.length }, null, 2)));
|
|
871
|
-
return;
|
|
872
|
-
}
|
|
873
|
-
const line = renderPrecision(summary);
|
|
874
|
-
deps.out(summary.precision === null ? pc.yellow(line) : line);
|
|
811
|
+
const since = new Date(Date.now() - opts.days * 86_400_000)
|
|
812
|
+
.toISOString()
|
|
813
|
+
.slice(0, 10);
|
|
814
|
+
const source = deps.buildAdjudicationSource(opts.repo, token);
|
|
815
|
+
const { evidence, scanned } = await collectEvidence(source, since);
|
|
816
|
+
const report = deriveReport(evidence, scanned);
|
|
817
|
+
deps.out(opts.json
|
|
818
|
+
? JSON.stringify({ since, ...report }, null, 2)
|
|
819
|
+
: `${renderReport(report)}\nwindow: merged since ${since}`);
|
|
875
820
|
}
|
|
876
821
|
// --- pr-check -----------------------------------------------------------------
|
|
877
822
|
/**
|
|
@@ -894,11 +839,17 @@ async function postStickyComment(findings, resolution, deps, gateMeta = null) {
|
|
|
894
839
|
appendStepSummary(deps.env, res.notice);
|
|
895
840
|
}
|
|
896
841
|
}
|
|
842
|
+
/** An abstained run judged nothing, so no agent tier was ever in play. */
|
|
843
|
+
const NO_TIER = {
|
|
844
|
+
requested: 0,
|
|
845
|
+
effective: 0,
|
|
846
|
+
degraded_notice: null,
|
|
847
|
+
};
|
|
897
848
|
/** The gate's no-op line, shared by the pre- and post-filter exits. */
|
|
898
849
|
// D7: every filtered path stays visible as a SkipEntry, never folded
|
|
899
850
|
// into "passed". One entry per path so the rendered count still equals
|
|
900
851
|
// the path count the old `N path(s) skipped` line reported.
|
|
901
|
-
function prCheckSkipEntries(skipped, testUnits, barrelUnits, supportUnits = [], typeOnlyUnits = []) {
|
|
852
|
+
function prCheckSkipEntries(skipped, testUnits, barrelUnits, supportUnits = [], typeOnlyUnits = [], nonSourceUnits = []) {
|
|
902
853
|
return [
|
|
903
854
|
...skipped.map((u) => ({ name: u.path, reason: 'skipGlobs' })),
|
|
904
855
|
...testUnits.map((u) => ({ name: u.path, reason: 'test path' })),
|
|
@@ -909,6 +860,8 @@ function prCheckSkipEntries(skipped, testUnits, barrelUnits, supportUnits = [],
|
|
|
909
860
|
// #562: likewise distinct -- adjudication has to be able to measure this
|
|
910
861
|
// class separately, since it is the one that held precision at 13/20.
|
|
911
862
|
...typeOnlyUnits.map((u) => ({ name: u.path, reason: 'type-only module' })),
|
|
863
|
+
// #928: config/data files, below the source floor.
|
|
864
|
+
...nonSourceUnits.map((u) => ({ name: u.path, reason: 'non-source' })),
|
|
912
865
|
...barrelUnits.map((u) => ({
|
|
913
866
|
name: u.path,
|
|
914
867
|
reason: 're-export barrel',
|
|
@@ -934,27 +887,39 @@ const PR_CHECK_ABSTAIN_REMEDIATION = [
|
|
|
934
887
|
'non-empty and skipGlobs/heuristicExclude are not filtering ' +
|
|
935
888
|
'every path.',
|
|
936
889
|
];
|
|
937
|
-
/**
|
|
938
|
-
|
|
890
|
+
/**
|
|
891
|
+
* Exit 3 with the structural abstention line + remediation (#508). Nothing was
|
|
892
|
+
* eligible (docs, tests, config), so under `--post-comment` it also upserts the
|
|
893
|
+
* ✅ "nothing to test" sticky and an earlier ⚠️ one cannot linger (#928).
|
|
894
|
+
*/
|
|
895
|
+
async function abstainPrCheck(skipped, opts, deps, provenance = null) {
|
|
939
896
|
const outcome = gateOutcome({ checked: 0, findings: [], skipped }, 'gate', {
|
|
940
897
|
noun: 'unit(s)',
|
|
941
898
|
});
|
|
942
899
|
deps.out(outcome.summaryLine);
|
|
943
|
-
// #761:
|
|
944
|
-
// question is "over WHAT?", and the run that motivated this feature is
|
|
945
|
-
// precisely one that should have abstained. Stating the range here is what
|
|
946
|
-
// separates "correctly abstained on a docs-only PR" from "abstained because
|
|
947
|
-
// the diff was wrong", which read identically without it.
|
|
900
|
+
// #761: state the range, so "correctly abstained" and "wrong diff" differ.
|
|
948
901
|
if (provenance)
|
|
949
902
|
deps.out(provenanceLine(provenance));
|
|
950
903
|
for (const line of PR_CHECK_ABSTAIN_REMEDIATION)
|
|
951
904
|
deps.out(line);
|
|
952
|
-
if (
|
|
905
|
+
if (opts.postComment) {
|
|
906
|
+
const coverage = { requested: null, found: false, parsed: false };
|
|
907
|
+
await postStickyComment([], NO_TIER, deps, {
|
|
908
|
+
checked: 0,
|
|
909
|
+
abstained: false,
|
|
910
|
+
coverage: {
|
|
911
|
+
...coverage,
|
|
912
|
+
filesInReport: 0,
|
|
913
|
+
unitsMatched: 0,
|
|
914
|
+
unitsTotal: 0,
|
|
915
|
+
},
|
|
916
|
+
skipped,
|
|
917
|
+
provenance,
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
if (opts.format === 'json') {
|
|
953
921
|
deps.out(ensureAscii(JSON.stringify(
|
|
954
|
-
// #579: `skipped`
|
|
955
|
-
// to. Without it a consumer sees `abstained: true` and cannot tell
|
|
956
|
-
// WHAT was dropped or why -- the #508 class one layer down, on the
|
|
957
|
-
// only surface a machine can read.
|
|
922
|
+
// #579: `skipped` is the denominator the abstention collapsed to.
|
|
958
923
|
{
|
|
959
924
|
findings: [],
|
|
960
925
|
tier: 0,
|
|
@@ -972,53 +937,6 @@ function abstainPrCheck(skipped, format, deps, provenance = null) {
|
|
|
972
937
|
function resolveAnalysesDir(override, deps) {
|
|
973
938
|
return override ?? join(gitToplevel(deps), '.harness', 'analyses');
|
|
974
939
|
}
|
|
975
|
-
/**
|
|
976
|
-
* Collect 👍/👎 adjudications off the sticky comment, best-effort (#490).
|
|
977
|
-
*
|
|
978
|
-
* Runs on the NEXT `pr-check` for a PR (the collection loop the #490 sketch
|
|
979
|
-
* chose over a scheduled sweep — the guardian already authenticates against
|
|
980
|
-
* this API and already finds its own comment by marker). Read-only against
|
|
981
|
-
* GitHub, so it works where the fork-degraded poster could not write.
|
|
982
|
-
*
|
|
983
|
-
* NEVER affects the gate: any failure prints a `::warning::` and returns —
|
|
984
|
-
* a broken feedback loop must not turn a coverage gate red.
|
|
985
|
-
*/
|
|
986
|
-
async function collectAdjudicationsBestEffort(analysesDirOverride, deps) {
|
|
987
|
-
const ctx = prContextFromEnv(deps.env);
|
|
988
|
-
if (ctx === null)
|
|
989
|
-
return; // no PR context — nothing to collect against
|
|
990
|
-
if (!deps.env['GITHUB_TOKEN']) {
|
|
991
|
-
// Every read here needs a token; skipping LOUDLY beats a guaranteed 401.
|
|
992
|
-
deps.out(degradationAnnotation(`guardian: no GITHUB_TOKEN ${EM_DASH} reviewer adjudications not ` +
|
|
993
|
-
'collected this run'));
|
|
994
|
-
return;
|
|
995
|
-
}
|
|
996
|
-
// Resolved lazily (shells out to git) only once a collection will happen —
|
|
997
|
-
// an explicit `--diff` run without PR context must stay subprocess-free.
|
|
998
|
-
const analysesDir = resolveAnalysesDir(analysesDirOverride, deps);
|
|
999
|
-
try {
|
|
1000
|
-
const client = deps.buildReactionsClient(ctx[0], ctx[1]);
|
|
1001
|
-
const res = await collectAdjudications(client, {
|
|
1002
|
-
repo: ctx[0],
|
|
1003
|
-
prNumber: ctx[1],
|
|
1004
|
-
analysesDir,
|
|
1005
|
-
});
|
|
1006
|
-
if (res.action === 'collected' && res.record) {
|
|
1007
|
-
deps.out(`guardian: adjudication recorded (${res.record.tp} up / ` +
|
|
1008
|
-
`${res.record.fp} down) ${RIGHT_ARROW} ${res.path}`);
|
|
1009
|
-
}
|
|
1010
|
-
else if (res.action === 'unavailable' && res.notice) {
|
|
1011
|
-
deps.out(degradationAnnotation(res.notice));
|
|
1012
|
-
}
|
|
1013
|
-
// no-comment / no-reactions: nothing to say — absence of a reaction is
|
|
1014
|
-
// neutral, not a data point (#490).
|
|
1015
|
-
}
|
|
1016
|
-
catch (exc) {
|
|
1017
|
-
const message = exc instanceof Error ? exc.message : String(exc);
|
|
1018
|
-
deps.out(degradationAnnotation(`guardian: adjudication collection failed (${message}) ${EM_DASH} ` +
|
|
1019
|
-
'findings and gate unaffected'));
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
940
|
async function prCheckCmd(opts, deps) {
|
|
1023
941
|
const [config, warning] = loadGuardianConfig(opts.config);
|
|
1024
942
|
if (warning !== null) {
|
|
@@ -1032,12 +950,6 @@ async function prCheckCmd(opts, deps) {
|
|
|
1032
950
|
throw new CliExitError(0);
|
|
1033
951
|
}
|
|
1034
952
|
const effectiveGate = opts.gate ?? config.pr_gate;
|
|
1035
|
-
// #490: read reviewer 👍/👎 off the PREVIOUS run's sticky comment before this
|
|
1036
|
-
// run touches it. Runs on the posting/emitting (CI) surfaces only, before the
|
|
1037
|
-
// early exits so a docs-only follow-up push still harvests the verdicts.
|
|
1038
|
-
if (opts.postComment || opts.emitAnalysis) {
|
|
1039
|
-
await collectAdjudicationsBestEffort(opts.analysesDir, deps);
|
|
1040
|
-
}
|
|
1041
953
|
// #369: in CI an omitted `--diff` resolves the PR diff from the base ref;
|
|
1042
954
|
// the working-tree fallback is empty on a clean checkout.
|
|
1043
955
|
const resolvedDiff = readPrDiff(opts.diff ?? null, deps);
|
|
@@ -1050,7 +962,8 @@ async function prCheckCmd(opts, deps) {
|
|
|
1050
962
|
// Populated even for an explicit `--diff` (where `base` is unknowable): the
|
|
1051
963
|
// merge-ref warning and the file count are exactly what was missing on
|
|
1052
964
|
// the consumer run that surfaced #761, which passed `--diff` from a file.
|
|
1053
|
-
|
|
965
|
+
// #883: a diff taken to the PR head is the PR's own, so it is not widened.
|
|
966
|
+
const headSha = resolvedDiff.head ?? resolveHeadSha(deps);
|
|
1054
967
|
const mergeRef = detectMergeRef(headSha, deps);
|
|
1055
968
|
const provenance = {
|
|
1056
969
|
base: resolvedDiff.base,
|
|
@@ -1080,22 +993,20 @@ async function prCheckCmd(opts, deps) {
|
|
|
1080
993
|
// FIX 2: drop pure re-export/barrel files.
|
|
1081
994
|
const reexportPaths = findReexportOnly(diffText);
|
|
1082
995
|
const barrelUnits = keptTyped.filter((u) => reexportPaths.has(u.path));
|
|
1083
|
-
const
|
|
996
|
+
const keptBarrel = keptTyped.filter((u) => !reexportPaths.has(u.path));
|
|
997
|
+
// #928: the heuristic tier's source floor (#413) applies to coverage units
|
|
998
|
+
// too; a config/data file can never match a coverage report.
|
|
999
|
+
const nonSourceUnits = keptBarrel.filter((u) => !isSourcePath(u.path));
|
|
1000
|
+
const kept = keptBarrel.filter((u) => isSourcePath(u.path));
|
|
1084
1001
|
// Advisory weak-test findings for added tests that assert nothing.
|
|
1085
1002
|
const weakFindings = config.weak_tests
|
|
1086
1003
|
? buildWeakTestFindings(testUnits, diffText)
|
|
1087
1004
|
: [];
|
|
1088
|
-
// #582: build the skip list ONCE, above the abstain exit, so
|
|
1089
|
-
//
|
|
1090
|
-
|
|
1091
|
-
// coverage ladder has run, so it is appended below rather than passed here.
|
|
1092
|
-
//
|
|
1093
|
-
// This supersedes a `preFilterSkipped` count that was computed at this point
|
|
1094
|
-
// and read by nothing — the fossil of an earlier attempt to surface the same
|
|
1095
|
-
// number on this path.
|
|
1096
|
-
const preCoverageSkips = prCheckSkipEntries(skipped, testUnits, barrelUnits, supportUnits, typeOnlyUnits);
|
|
1005
|
+
// #582: build the skip list ONCE, above the abstain exit, so both paths carry
|
|
1006
|
+
// the same denominator; the heuristic-noise class is appended below.
|
|
1007
|
+
const preCoverageSkips = prCheckSkipEntries(skipped, testUnits, barrelUnits, supportUnits, typeOnlyUnits, nonSourceUnits);
|
|
1097
1008
|
if (kept.length === 0 && weakFindings.length === 0) {
|
|
1098
|
-
abstainPrCheck(preCoverageSkips, opts
|
|
1009
|
+
await abstainPrCheck(preCoverageSkips, opts, deps, provenance);
|
|
1099
1010
|
}
|
|
1100
1011
|
const { results, coverage } = resolveCoverageWithInput(kept, {
|
|
1101
1012
|
coveragePath: opts.coverage ?? null,
|
|
@@ -1107,8 +1018,17 @@ async function prCheckCmd(opts, deps) {
|
|
|
1107
1018
|
// never judge (non-source, or an excluded glob). Coverage/graph-verified
|
|
1108
1019
|
// verdicts on the same paths are real evidence and survive.
|
|
1109
1020
|
const [scoredResults, noiseResults] = filterHeuristicNoise(results, opts.heuristicExclude ?? config.heuristic_exclude);
|
|
1021
|
+
// #606: the second coverage question — did coverage go DOWN on a unit this
|
|
1022
|
+
// PR touches? Run over the same `kept` units the ladder scored, so the two
|
|
1023
|
+
// denominators are the same surface. With no base artifact this returns no
|
|
1024
|
+
// deltas and a state that says so, which the notice below reports loudly.
|
|
1025
|
+
const { deltas, state: coverageDelta } = resolveCoverageDelta(kept, {
|
|
1026
|
+
baseCoveragePath: opts.baseCoverage ?? null,
|
|
1027
|
+
headCoveragePath: opts.coverage ?? null,
|
|
1028
|
+
});
|
|
1110
1029
|
const findings = [
|
|
1111
1030
|
...applySuppressions(buildFindings(scoredResults)),
|
|
1031
|
+
...buildRegressionFindings(deltas),
|
|
1112
1032
|
...weakFindings,
|
|
1113
1033
|
];
|
|
1114
1034
|
// The complete skip list for this run: the pre-coverage filters plus the
|
|
@@ -1121,7 +1041,7 @@ async function prCheckCmd(opts, deps) {
|
|
|
1121
1041
|
// SKIP rather than rendering an empty "0 unaddressed" report -- an adopter
|
|
1122
1042
|
// must be able to tell "nothing was judgeable" from "everything passed".
|
|
1123
1043
|
if (scoredResults.length === 0 && findings.length === 0) {
|
|
1124
|
-
abstainPrCheck(allSkips, opts
|
|
1044
|
+
await abstainPrCheck(allSkips, opts, deps, provenance);
|
|
1125
1045
|
}
|
|
1126
1046
|
// SC-5 (PR half): resolve the requested tier against actual capability. No
|
|
1127
1047
|
// agent runtime exists (default NoAgentProbe), so any `pr.tier > 0` drops to
|
|
@@ -1149,20 +1069,31 @@ async function prCheckCmd(opts, deps) {
|
|
|
1149
1069
|
checked: scoredResults.length,
|
|
1150
1070
|
abstained: coverageAbstained,
|
|
1151
1071
|
coverage,
|
|
1072
|
+
// #606: the delta's own denominator, so "no regressions" can never be read
|
|
1073
|
+
// as "compared and clean" on a run that compared nothing.
|
|
1074
|
+
coverageDelta,
|
|
1152
1075
|
// #761: the endpoints every count above is scoped by.
|
|
1153
1076
|
provenance,
|
|
1154
1077
|
// #582: `checked` is the numerator of a fraction whose denominator was
|
|
1155
1078
|
// never printed. This is the rest of it.
|
|
1156
1079
|
skipped: allSkips,
|
|
1157
1080
|
};
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1081
|
+
// #606: the delta degradation rides the exact same surfaces as the coverage
|
|
1082
|
+
// one — a head-only run is as blind about regressions as a report-less run is
|
|
1083
|
+
// about coverage, and must be as loud.
|
|
1084
|
+
const notices = [
|
|
1085
|
+
coverageDegradedNotice(coverage),
|
|
1086
|
+
coverageDeltaNotice(coverageDelta),
|
|
1087
|
+
];
|
|
1088
|
+
// `--format json` owns stdout: a `::warning::` line there would make the
|
|
1089
|
+
// document unparseable, so the annotation goes to stderr on that path. Both
|
|
1090
|
+
// streams are scanned for workflow commands, so CI still sees it.
|
|
1091
|
+
const machineStdout = !opts.postComment && !opts.emitAnalysis && opts.format === 'json';
|
|
1092
|
+
for (const notice of notices) {
|
|
1093
|
+
if (!notice)
|
|
1094
|
+
continue;
|
|
1095
|
+
(machineStdout ? deps.err : deps.out)(degradationAnnotation(notice));
|
|
1096
|
+
appendStepSummary(deps.env, notice);
|
|
1166
1097
|
}
|
|
1167
1098
|
let commentPosted = false;
|
|
1168
1099
|
if (opts.emitAnalysis) {
|
|
@@ -1254,19 +1185,10 @@ function intentDict(intent) {
|
|
|
1254
1185
|
};
|
|
1255
1186
|
}
|
|
1256
1187
|
/**
|
|
1257
|
-
* author-plan's denominator decision (#508
|
|
1258
|
-
*
|
|
1259
|
-
*
|
|
1260
|
-
*
|
|
1261
|
-
* took pr-check. On an EMPTY diff this surface emitted
|
|
1262
|
-
* `block: false, authored_count: 0` and exited 0 -- "we examined nothing,
|
|
1263
|
-
* therefore do not block", which is the #456 class verbatim.
|
|
1264
|
-
*
|
|
1265
|
-
* ADVISORY, not a gate: author-plan is an authoring aid whose JSON an agent
|
|
1266
|
-
* reads (see `canary-pr-guardian/SKILL.md`); the exit-code contract belongs to
|
|
1267
|
-
* `pr-check` and the pre-commit gate. So the exit stays 0 and stdout stays a
|
|
1268
|
-
* single parseable object -- `checked`/`abstained` ride the payload additively
|
|
1269
|
-
* and the loud line goes to stderr, keeping `--json` consumers byte-compatible.
|
|
1188
|
+
* author-plan's denominator decision (#508): an empty diff must not read as
|
|
1189
|
+
* "examined nothing, so do not block" (#456). ADVISORY, not a gate: exit stays
|
|
1190
|
+
* 0 and stdout one parseable object; `checked`/`abstained` ride the payload and
|
|
1191
|
+
* the loud line goes to stderr.
|
|
1270
1192
|
*/
|
|
1271
1193
|
function authorPlanOutcome(checked, results, deps) {
|
|
1272
1194
|
const outcome = gateOutcome({ checked, findings: [...results] }, 'advisory', {
|
|
@@ -1403,29 +1325,20 @@ export function createGuardianCommand(depsInit = {}) {
|
|
|
1403
1325
|
.addOption(new Option('--check <check>', 'Status-check context to require (the guardian workflow job).').default('guardian'))
|
|
1404
1326
|
.addOption(new Option('--token <token>', 'Admin token for --apply.').env('GITHUB_TOKEN'))
|
|
1405
1327
|
.option('--force', 'Skip the check-context-exists verification (risky).')
|
|
1406
|
-
.addOption(new Option('--analyses-dir <dir>', 'Override the analyses dir (tests).').hideHelp())
|
|
1407
1328
|
.action(async (opts) => {
|
|
1408
1329
|
await hardenGateCmd(opts, deps);
|
|
1409
1330
|
});
|
|
1410
1331
|
program
|
|
1411
|
-
.command('
|
|
1412
|
-
.description(
|
|
1413
|
-
'
|
|
1332
|
+
.command('precision')
|
|
1333
|
+
.description('Derive guardian finding precision (TP / (TP + FP)) from merged PRs, ' +
|
|
1334
|
+
'with its sample size (ADR 0025).')
|
|
1414
1335
|
.addOption(new Option('--repo <repo>', 'owner/repo.').env('GITHUB_REPOSITORY'))
|
|
1415
|
-
.addOption(new Option('--
|
|
1416
|
-
.
|
|
1417
|
-
.
|
|
1336
|
+
.addOption(new Option('--days <n>', 'Merged-PR window in days.')
|
|
1337
|
+
.default(14)
|
|
1338
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
1339
|
+
.option('--json', 'Emit the report as JSON (precision null = unknown).')
|
|
1418
1340
|
.action(async (opts) => {
|
|
1419
|
-
await
|
|
1420
|
-
});
|
|
1421
|
-
program
|
|
1422
|
-
.command('precision')
|
|
1423
|
-
.description('Report guardian finding precision (TP / (TP + FP)) from collected ' +
|
|
1424
|
-
'adjudications, with its sample size.')
|
|
1425
|
-
.option('--json', 'Emit the summary as JSON (precision null = unknown).')
|
|
1426
|
-
.addOption(new Option('--analyses-dir <dir>', 'Override the analyses dir (tests).').hideHelp())
|
|
1427
|
-
.action((opts) => {
|
|
1428
|
-
precisionCmd(opts, deps);
|
|
1341
|
+
await precisionCmd(opts, deps);
|
|
1429
1342
|
});
|
|
1430
1343
|
program
|
|
1431
1344
|
.command('pr-check')
|
|
@@ -1433,6 +1346,9 @@ export function createGuardianCommand(depsInit = {}) {
|
|
|
1433
1346
|
.option('--diff <diff>', "Diff file, '-' for stdin, or omit to auto-resolve: the PR diff " +
|
|
1434
1347
|
'(`<base>...HEAD`) in CI, else the local working-tree `git diff`.')
|
|
1435
1348
|
.option('--coverage <path>', 'Coverage report path (lcov/json).')
|
|
1349
|
+
.option('--base-coverage <path>', "Coverage report for the PR's BASE ref (#606). Enables coverage-" +
|
|
1350
|
+
'regression findings on touched units. Without it the run degrades ' +
|
|
1351
|
+
'LOUDLY to head-only — it never silently reports no regressions.')
|
|
1436
1352
|
.addOption(new Option('--format <fmt>', 'comment|json|text').default('comment'))
|
|
1437
1353
|
.addOption(new Option('--config <path>').default('harness.config.json'))
|
|
1438
1354
|
.option('--gate <gate>', 'Override config gate: soft|hard')
|