canary-test-cli 7.1.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +327 -0
- package/agents/skills/canary:generate.md +49 -0
- package/agents/skills/canary:init.md +37 -0
- package/agents/skills/canary:migrate.md +66 -0
- package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
- package/agents/skills/claude-code/canary-batwoman/SKILL.md +119 -0
- package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +187 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +270 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +178 -0
- package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
- package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
- package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
- package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
- package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +197 -0
- package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
- package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
- package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
- package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
- package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
- package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
- package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
- package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
- package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
- package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +144 -0
- package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
- package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
- package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
- package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
- package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
- package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
- package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-screech/SKILL.md +109 -0
- package/agents/skills/claude-code/canary-screech/scripts/blast.mjs +125 -0
- package/agents/skills/claude-code/canary-screech/scripts/cli.mjs +128 -0
- package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs +97 -0
- package/agents/skills/claude-code/canary-screech/scripts/history.mjs +73 -0
- package/agents/skills/claude-code/canary-screech/scripts/redness.mjs +94 -0
- package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
- package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
- package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
- package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
- package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
- package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
- package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
- package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
- package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
- package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
- package/agents/skills/lib/parse-args.mjs +275 -0
- package/dist/engine/analysis/batwoman/audit.js +39 -0
- package/dist/engine/analysis/batwoman/closure.js +159 -0
- package/dist/engine/analysis/batwoman/gh-history.js +119 -0
- package/dist/engine/analysis/batwoman/probes.js +195 -0
- package/dist/engine/analysis/batwoman/registry.js +142 -0
- package/dist/engine/analysis/batwoman/render.js +194 -0
- package/dist/engine/analysis/batwoman/run-window.js +122 -0
- package/dist/engine/analysis/batwoman/text.js +84 -0
- package/dist/engine/analysis/batwoman/triggers.js +122 -0
- package/dist/engine/analysis/batwoman/verdict.js +64 -0
- package/dist/engine/analysis/cli.js +47 -14
- package/dist/engine/analysis/gh-flaky/gh-run-attempts.js +206 -0
- package/dist/engine/batwoman-cli.js +119 -0
- package/dist/engine/ci-ready-cli.js +71 -0
- package/dist/engine/cli-commands.js +49 -72
- package/dist/engine/cli.core.js +16 -0
- package/dist/engine/company-knowledge-cli.js +10 -2
- package/dist/engine/core/ci-ready.js +112 -0
- package/dist/engine/core/company-knowledge.js +8 -0
- package/dist/engine/core/migrator.js +147 -20
- package/dist/engine/core/permission-matrix.js +219 -0
- package/dist/engine/core/quality-scorer.js +27 -19
- package/dist/engine/core/scaling-curve.js +143 -0
- package/dist/engine/core/skill-dispatch.js +115 -0
- package/dist/engine/core/skill-examples.js +103 -3
- package/dist/engine/core/skill-registry.js +59 -4
- package/dist/engine/core/string-literals.js +3 -1
- package/dist/engine/core/test-files.js +77 -0
- package/dist/engine/core/vacuity-scanner.js +330 -15
- package/dist/engine/core/workflow-discovery.js +41 -23
- package/dist/engine/guardian/adjudication-github.js +136 -0
- package/dist/engine/guardian/adjudication.js +119 -340
- package/dist/engine/guardian/analysis-emit.js +7 -2
- package/dist/engine/guardian/cli.js +277 -249
- package/dist/engine/guardian/coverage.js +2 -1
- package/dist/engine/guardian/diff-coverage/coverage-delta.js +162 -0
- package/dist/engine/guardian/diff-coverage/formats/cobertura.js +45 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +25 -21
- package/dist/engine/guardian/diff-coverage/paths.js +5 -9
- package/dist/engine/guardian/diff-coverage/report-tier.js +88 -12
- package/dist/engine/guardian/diff-extractor.js +31 -32
- package/dist/engine/guardian/pr-check.js +354 -223
- package/dist/engine/guardian/pr-comment.js +35 -58
- package/dist/engine/guardian/weak-test.js +236 -0
- package/dist/engine/mcp-server.js +67 -4
- package/dist/engine/permission-matrix-cli.js +51 -0
- package/dist/engine/scaling-curve-cli.js +147 -0
- package/dist/engine/skills-cli.js +171 -51
- package/dist/engine/workflow-cli.js +85 -65
- package/dist/reporters/testtracker.d.ts +1 -1
- package/dist/reporters/testtracker.js +1 -1
- package/package.json +3 -2
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `canary batwoman --issue N [--json]` (spec Phase 4).
|
|
3
|
+
*
|
|
4
|
+
* The first surface a human meets, and the point where every earlier phase is
|
|
5
|
+
* wired together: the closure adapter names the change, the probes decide per
|
|
6
|
+
* file, and the renderer says it in the reader's register.
|
|
7
|
+
*
|
|
8
|
+
* **The two output paths are not interchangeable.** The human report is
|
|
9
|
+
* persona-shaped prose; `--json` is the CI wrapper's input and is
|
|
10
|
+
* persona-independent, because a machine consumer's parser must not shift with
|
|
11
|
+
* whoever happened to run the command. Both carry every changed file, and
|
|
12
|
+
* neither carries an aggregate a reader could mistake for a pass -- `--json`
|
|
13
|
+
* especially, since a convenient `passed: true` is exactly the field a CI
|
|
14
|
+
* wrapper would grow later (spec criteria 2 and 3).
|
|
15
|
+
*
|
|
16
|
+
* A failure to resolve the closure exits non-zero and prints nothing to
|
|
17
|
+
* stdout: there is no report to give. A failure to read run history does not,
|
|
18
|
+
* because "could not tell" is a real verdict over a known set of files, and
|
|
19
|
+
* losing the whole report to it would be worse than reporting the abstentions.
|
|
20
|
+
*/
|
|
21
|
+
import { Command, InvalidArgumentError } from 'commander';
|
|
22
|
+
import { auditIssue, } from './analysis/batwoman/audit.js';
|
|
23
|
+
import { renderReport } from './analysis/batwoman/render.js';
|
|
24
|
+
import { tallyVerdicts } from './analysis/batwoman/verdict.js';
|
|
25
|
+
import { resolvePersona } from './core/persona.js';
|
|
26
|
+
import { CliExitError } from './cli-common.js';
|
|
27
|
+
/**
|
|
28
|
+
* `owner/name` for the repository under audit.
|
|
29
|
+
*
|
|
30
|
+
* Read from the environment GitHub Actions already sets, so the CI wrapper
|
|
31
|
+
* needs no argument. Outside Actions it is required explicitly rather than
|
|
32
|
+
* guessed from a git remote: a wrong repo would silently audit someone else's
|
|
33
|
+
* run history and report it as this one's.
|
|
34
|
+
*/
|
|
35
|
+
function repoFromEnv(env) {
|
|
36
|
+
const repo = env['GITHUB_REPOSITORY'];
|
|
37
|
+
if (repo === undefined || repo.trim() === '') {
|
|
38
|
+
throw new Error('no repository to audit: set GITHUB_REPOSITORY to "owner/name". It is ' +
|
|
39
|
+
'not inferred from a git remote, because auditing the wrong ' +
|
|
40
|
+
"repository's run history would report a confident answer about the " +
|
|
41
|
+
'wrong thing.');
|
|
42
|
+
}
|
|
43
|
+
return repo;
|
|
44
|
+
}
|
|
45
|
+
/** The machine shape. Deliberately flat, and deliberately without a verdict. */
|
|
46
|
+
function toJson({ closure, verdicts }, repo) {
|
|
47
|
+
const tally = tallyVerdicts(verdicts);
|
|
48
|
+
return JSON.stringify({
|
|
49
|
+
issue: closure.header.issue,
|
|
50
|
+
pullRequest: closure.pullRequest,
|
|
51
|
+
repo,
|
|
52
|
+
mergeSha: closure.header.mergeSha,
|
|
53
|
+
mergeSubject: closure.header.mergeSubject,
|
|
54
|
+
mergedAt: closure.header.mergedAt.toISOString(),
|
|
55
|
+
// `changed` and `byStatus` only. There is no `assessed`, no `passed`,
|
|
56
|
+
// and no score: a consumer wanting a subtotal has to add the statuses up
|
|
57
|
+
// in the open, where a reviewer can see which ones it folded together.
|
|
58
|
+
summary: { changed: tally.changed, byStatus: tally.byStatus },
|
|
59
|
+
files: verdicts.map((v) => ({
|
|
60
|
+
file: v.file,
|
|
61
|
+
status: v.status,
|
|
62
|
+
explanation: v.explanation,
|
|
63
|
+
...('evidence' in v && v.evidence !== undefined
|
|
64
|
+
? { evidence: v.evidence }
|
|
65
|
+
: {}),
|
|
66
|
+
})),
|
|
67
|
+
}, null, 2);
|
|
68
|
+
}
|
|
69
|
+
/** Build the `batwoman` subcommand. */
|
|
70
|
+
export function buildBatwomanCommand(deps, cli = {}) {
|
|
71
|
+
const wiring = {
|
|
72
|
+
repo: () => repoFromEnv(deps.env),
|
|
73
|
+
root: () => deps.cwd(),
|
|
74
|
+
...cli,
|
|
75
|
+
};
|
|
76
|
+
const command = new Command('batwoman');
|
|
77
|
+
command
|
|
78
|
+
.description('Report whether the files a closed issue changed have executed since ' +
|
|
79
|
+
'the fix merged. Never asserts the fix is correct.')
|
|
80
|
+
.requiredOption('--issue <number>', 'the closed issue to audit', (raw) => {
|
|
81
|
+
const n = Number.parseInt(raw, 10);
|
|
82
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
83
|
+
// commander's own error type, so a bad flag exits as a USAGE error
|
|
84
|
+
// (2) with a clean message, rather than as an unexpected crash with a
|
|
85
|
+
// stack trace in front of the user.
|
|
86
|
+
throw new InvalidArgumentError(`--issue must be a positive integer, got "${raw}"`);
|
|
87
|
+
}
|
|
88
|
+
return n;
|
|
89
|
+
})
|
|
90
|
+
.option('--json', 'emit the machine shape instead of the report')
|
|
91
|
+
.action(async (opts) => {
|
|
92
|
+
let audit;
|
|
93
|
+
let repo;
|
|
94
|
+
try {
|
|
95
|
+
repo = wiring.repo();
|
|
96
|
+
audit = await auditIssue(opts.issue, repo, wiring.root(), wiring);
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
// Nothing is printed to stdout: there is no report to give, and half a
|
|
100
|
+
// report would be read as a whole one. Only a failure to resolve the
|
|
101
|
+
// closure lands here -- an unreadable run history becomes per-file
|
|
102
|
+
// abstentions inside the audit, which are worth printing.
|
|
103
|
+
deps.err(err instanceof Error ? err.message : String(err));
|
|
104
|
+
throw new CliExitError(1);
|
|
105
|
+
}
|
|
106
|
+
if (opts.json === true) {
|
|
107
|
+
deps.out(toJson(audit, repo));
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
deps.out(renderReport({
|
|
111
|
+
header: audit.closure.header,
|
|
112
|
+
repo,
|
|
113
|
+
verdicts: audit.verdicts,
|
|
114
|
+
persona: resolvePersona(wiring.registry === undefined ? {} : { registry: wiring.registry }),
|
|
115
|
+
}));
|
|
116
|
+
});
|
|
117
|
+
return command;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=batwoman-cli.js.map
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `canary ci-ready`: the deterministic scorer behind the canary-ci-ready skill.
|
|
3
|
+
*
|
|
4
|
+
* Reads its inputs under `--root` and hands them to the pure scoring in
|
|
5
|
+
* `core/ci-ready.ts`. Exit codes follow the CLI-wide gate contract:
|
|
6
|
+
* - 3 (EXIT_ABSTAINED): every check skipped, so nothing was scored.
|
|
7
|
+
* - 1: at least one check failed.
|
|
8
|
+
* - 0: nothing failed. That covers both `ready` and `incomplete`; the text
|
|
9
|
+
* output says loudly which one it is.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync } from 'node:fs';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { Command } from 'commander';
|
|
14
|
+
import { CliExitError } from './cli-common.js';
|
|
15
|
+
import { EXIT_ABSTAINED } from './core/gate-result.js';
|
|
16
|
+
import { scoreCiReady } from './core/ci-ready.js';
|
|
17
|
+
import { NdjsonHistoryStore } from './history/ndjson-store.js';
|
|
18
|
+
/** Same default location `canary history` writes to (DEFAULT_HISTORY_FILE in history/cli.ts). */
|
|
19
|
+
const HISTORY_FILE = join('test-results', 'reports', 'history-v2.jsonl');
|
|
20
|
+
function readRuns(root) {
|
|
21
|
+
const path = join(root, HISTORY_FILE);
|
|
22
|
+
return existsSync(path) ? new NdjsonHistoryStore(path).readAll() : null;
|
|
23
|
+
}
|
|
24
|
+
function renderText(report) {
|
|
25
|
+
const lines = [
|
|
26
|
+
`CI readiness: ${report.verdict} — ${report.checked} of ${report.checks.length} checks scored`,
|
|
27
|
+
];
|
|
28
|
+
for (const c of report.checks)
|
|
29
|
+
lines.push(` ${c.verdict.padEnd(4)} ${c.name}: ${c.reason}`);
|
|
30
|
+
const skipped = report.checks.length - report.checked;
|
|
31
|
+
if (report.verdict === 'incomplete') {
|
|
32
|
+
lines.push(`Incomplete: ${skipped} check(s) skipped for missing inputs, so this is not a full readiness result.`);
|
|
33
|
+
}
|
|
34
|
+
if (report.verdict === 'abstained') {
|
|
35
|
+
lines.push('Abstained: no check had an input to score.');
|
|
36
|
+
}
|
|
37
|
+
return lines;
|
|
38
|
+
}
|
|
39
|
+
function exitCodeFor(report) {
|
|
40
|
+
if (report.verdict === 'abstained')
|
|
41
|
+
return EXIT_ABSTAINED;
|
|
42
|
+
return report.verdict === 'not-ready' ? 1 : 0;
|
|
43
|
+
}
|
|
44
|
+
export function buildCiReadyCommand(deps) {
|
|
45
|
+
const command = new Command('ci-ready');
|
|
46
|
+
command
|
|
47
|
+
.description('Score CI readiness across five checks; checks with no input report skip, never pass.')
|
|
48
|
+
.option('--root <dir>', 'Repository root to read inputs from (default: current directory).')
|
|
49
|
+
.option('--json', 'Output the verdict and every check as JSON.')
|
|
50
|
+
.action((opts) => {
|
|
51
|
+
const root = opts.root ?? deps.cwd();
|
|
52
|
+
const report = scoreCiReady({
|
|
53
|
+
runs: readRuns(root),
|
|
54
|
+
historyPath: HISTORY_FILE,
|
|
55
|
+
hasInventory: existsSync(join(root, '.canary', 'test-inventory.json')),
|
|
56
|
+
hasCriticalAreas: existsSync(join(root, '.canary', 'critical-areas.json')),
|
|
57
|
+
});
|
|
58
|
+
if (opts.json === true) {
|
|
59
|
+
deps.out(JSON.stringify(report, null, 2));
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
for (const line of renderText(report))
|
|
63
|
+
deps.out(line);
|
|
64
|
+
}
|
|
65
|
+
const code = exitCodeFor(report);
|
|
66
|
+
if (code !== 0)
|
|
67
|
+
throw new CliExitError(code);
|
|
68
|
+
});
|
|
69
|
+
return command;
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=ci-ready-cli.js.map
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* strips on a non-TTY sink so plain text is byte-exact), and output glyphs as
|
|
11
11
|
* `\u{...}` escapes emitted verbatim.
|
|
12
12
|
*/
|
|
13
|
-
import { existsSync,
|
|
13
|
+
import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
14
14
|
import { basename, extname, join, resolve } from 'node:path';
|
|
15
15
|
import pc from 'picocolors';
|
|
16
16
|
import { CliExitError, jsonIndent2 } from './cli-common.js';
|
|
@@ -22,7 +22,8 @@ import { ckInitCmd } from './company-knowledge-cli.js';
|
|
|
22
22
|
import { extractFrameworkHint } from './core/classifier.js';
|
|
23
23
|
import { VALID_CATEGORIES, buildFeedback } from './core/feedback.js';
|
|
24
24
|
import { OverlayNotFound, listOverlays, resolveOverlay, } from './core/overlays.js';
|
|
25
|
-
import {
|
|
25
|
+
import { frameworkForPath } from './core/static-linter.js';
|
|
26
|
+
import { SCANNABLE_DESC, collectTestFiles, isDir } from './core/test-files.js';
|
|
26
27
|
import { RunSummary } from './core/ticket-updater.js';
|
|
27
28
|
import { renderBanner } from './ui/banner.js';
|
|
28
29
|
import { ARROW, CHECK, CHECK_MARK, CROSS, EM_DASH, HAMMER, NEXT, REDX, ROCKET, WARN, WRENCH, } from './main-deps.js';
|
|
@@ -42,67 +43,6 @@ function isFile(p) {
|
|
|
42
43
|
return false;
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
|
-
function isDir(p) {
|
|
46
|
-
try {
|
|
47
|
-
return statSync(p).isDirectory();
|
|
48
|
-
}
|
|
49
|
-
catch {
|
|
50
|
-
return false;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
/**
|
|
54
|
-
* Directories never worth walking. A dependency's own test suite is not the
|
|
55
|
-
* consumer's to fix: before #566, `node_modules` accounted for 254 of 256
|
|
56
|
-
* findings in one downstream run, and the only `critical` sat inside vendored
|
|
57
|
-
* code. `pattern-matcher.ts` has carried this set since the Python port; this
|
|
58
|
-
* walk was the copy that never got it.
|
|
59
|
-
*/
|
|
60
|
-
const IGNORED_DIRS = new Set([
|
|
61
|
-
'node_modules',
|
|
62
|
-
'.git',
|
|
63
|
-
'__pycache__',
|
|
64
|
-
'.venv',
|
|
65
|
-
'venv',
|
|
66
|
-
'dist',
|
|
67
|
-
'build',
|
|
68
|
-
'.next',
|
|
69
|
-
'.nuxt',
|
|
70
|
-
]);
|
|
71
|
-
function walkFiles(dir) {
|
|
72
|
-
const out = [];
|
|
73
|
-
let entries;
|
|
74
|
-
try {
|
|
75
|
-
entries = readdirSync(dir, { withFileTypes: true });
|
|
76
|
-
}
|
|
77
|
-
catch {
|
|
78
|
-
return out;
|
|
79
|
-
}
|
|
80
|
-
for (const e of entries) {
|
|
81
|
-
const full = join(dir, e.name);
|
|
82
|
-
if (e.isDirectory()) {
|
|
83
|
-
if (!IGNORED_DIRS.has(e.name))
|
|
84
|
-
out.push(...walkFiles(full));
|
|
85
|
-
}
|
|
86
|
-
else if (e.isFile())
|
|
87
|
-
out.push(full);
|
|
88
|
-
}
|
|
89
|
-
return out;
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* `test_*.py` plus `*.test.*` / `*.spec.*` over every extension the scanners
|
|
93
|
-
* can actually read -- `.mjs` and `.cjs` included, which is the half of #566
|
|
94
|
-
* that made a directory of ESM tests collect zero files.
|
|
95
|
-
*/
|
|
96
|
-
const JS_TEST_FILE_RE = new RegExp(`\\.(test|spec)\\.(${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join('|')})$`);
|
|
97
|
-
/** Recursive test-file glob matching Python's `rglob` union, sorted by path. */
|
|
98
|
-
function collectTestFiles(dir) {
|
|
99
|
-
return walkFiles(dir)
|
|
100
|
-
.filter((p) => {
|
|
101
|
-
const b = basename(p);
|
|
102
|
-
return ((b.startsWith('test_') && b.endsWith('.py')) || JS_TEST_FILE_RE.test(b));
|
|
103
|
-
})
|
|
104
|
-
.sort();
|
|
105
|
-
}
|
|
106
46
|
export function recommendFrameworkCmd(promptText, opts, deps) {
|
|
107
47
|
const classifier = deps.makeClassifier();
|
|
108
48
|
const recommender = deps.makeRecommender();
|
|
@@ -429,6 +369,14 @@ export function migrateCmd(opts, deps) {
|
|
|
429
369
|
note: r.note,
|
|
430
370
|
})),
|
|
431
371
|
installed_workflows: report.installed_workflows.map((r) => r.to_dict()),
|
|
372
|
+
// #504 part 1 (spec test 28), additive: every key above keeps its name
|
|
373
|
+
// and value. `existing_suites` closes a #585 gap -- it was on the
|
|
374
|
+
// report but never in JSON, leaving a scripted consumer with
|
|
375
|
+
// `would_create: []` and no reason attached. `workspace` is `null`
|
|
376
|
+
// for a single-package repo; `shapes` is the set deployment used.
|
|
377
|
+
existing_suites: report.existing_suites,
|
|
378
|
+
workspace: report.workspace,
|
|
379
|
+
shapes: report.shapes,
|
|
432
380
|
}));
|
|
433
381
|
return;
|
|
434
382
|
}
|
|
@@ -453,8 +401,6 @@ function findingPayload(f) {
|
|
|
453
401
|
suggestion: f.suggestion,
|
|
454
402
|
};
|
|
455
403
|
}
|
|
456
|
-
/** Human-readable list of what the collectors look for, for remedy text. */
|
|
457
|
-
const SCANNABLE_DESC = `test_*.py, *.test|spec.{${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join(',')}}`;
|
|
458
404
|
/** Emit the abstention notice in the caller's output mode, then exit 3. */
|
|
459
405
|
function abstain(remedy, deps, json, skipped = []) {
|
|
460
406
|
const result = { checked: 0, findings: [] };
|
|
@@ -659,6 +605,7 @@ export function flakeCheckCmd(path, opts, deps) {
|
|
|
659
605
|
*/
|
|
660
606
|
export function vacuityCheckCmd(path, opts, deps) {
|
|
661
607
|
const json = opts.json === true;
|
|
608
|
+
const verbose = opts.verbose === true;
|
|
662
609
|
const files = isDir(path) ? collectTestFiles(path) : [path];
|
|
663
610
|
const findings = [];
|
|
664
611
|
const skipped = [];
|
|
@@ -676,10 +623,26 @@ export function vacuityCheckCmd(path, opts, deps) {
|
|
|
676
623
|
reason: `no test file matched (looked for ${SCANNABLE_DESC})`,
|
|
677
624
|
});
|
|
678
625
|
}
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
626
|
+
// #860: the summary line carries one entry per skip REASON with its count,
|
|
627
|
+
// not one per test -- 232 titles on one line is how skips get ignored. The
|
|
628
|
+
// per-test list stays reachable (--verbose, --json), so every skip remains
|
|
629
|
+
// countable and locatable; only the summary line is compacted.
|
|
630
|
+
const byReason = new Map();
|
|
631
|
+
for (const s of skipped)
|
|
632
|
+
byReason.set(s.reason, (byReason.get(s.reason) ?? 0) + 1);
|
|
633
|
+
// The suffix is built here rather than by `skippedSuffix`, whose count is the
|
|
634
|
+
// number of ENTRIES -- fed per-reason groups it would report "1 skipped" for
|
|
635
|
+
// 13 skipped tests.
|
|
636
|
+
const groups = [...byReason]
|
|
637
|
+
.map(([reason, n]) => `${n} test(s) [${reason}]`)
|
|
638
|
+
.join('; ');
|
|
639
|
+
const outcome = gateOutcome({ checked, findings }, 'advisory', {
|
|
640
|
+
noun: 'test(s)',
|
|
641
|
+
});
|
|
642
|
+
const summaryLine = skipped.length === 0
|
|
643
|
+
? outcome.summaryLine
|
|
644
|
+
: `${outcome.summaryLine} (${skipped.length} skipped: ${groups})` +
|
|
645
|
+
(verbose ? '' : ` ${pc.dim('(--verbose to list skipped tests)')}`);
|
|
683
646
|
// `advisory` keeps findings at exit 0; the abstention still has to be loud, so
|
|
684
647
|
// the exit code for a zero denominator is taken from the gate contract.
|
|
685
648
|
const exitCode = outcome.abstained ? EXIT_ABSTAINED : 0;
|
|
@@ -701,9 +664,14 @@ export function vacuityCheckCmd(path, opts, deps) {
|
|
|
701
664
|
deps.out(` ${f.test}: ${f.message}`);
|
|
702
665
|
deps.out(` ${pc.dim(`${ARROW} ${f.suggestion}`)}\n`);
|
|
703
666
|
}
|
|
667
|
+
if (verbose && skipped.length > 0) {
|
|
668
|
+
deps.out(pc.bold('Skipped:'));
|
|
669
|
+
for (const s of skipped)
|
|
670
|
+
deps.out(` ${s.name} ${pc.dim(`[${s.reason}]`)}`);
|
|
671
|
+
}
|
|
704
672
|
deps.out(outcome.abstained
|
|
705
|
-
? pc.bold(pc.yellow(
|
|
706
|
-
: `${pc.bold(
|
|
673
|
+
? pc.bold(pc.yellow(summaryLine))
|
|
674
|
+
: `${pc.bold(summaryLine)}`);
|
|
707
675
|
if (exitCode !== 0)
|
|
708
676
|
throw new CliExitError(exitCode);
|
|
709
677
|
}
|
|
@@ -855,7 +823,16 @@ export async function ticketUpdateCmd(opts, deps) {
|
|
|
855
823
|
let reportData = {};
|
|
856
824
|
if (opts.result) {
|
|
857
825
|
try {
|
|
858
|
-
|
|
826
|
+
const parsed = JSON.parse(readFileSync(opts.result, 'utf-8'));
|
|
827
|
+
// Valid JSON is not necessarily a report. A bare `null` or a top-level
|
|
828
|
+
// array parses fine and then gets indexed: `null` threw a raw TypeError
|
|
829
|
+
// out of the handler, and an array silently defaulted every field. Both
|
|
830
|
+
// belong on the same "could not read result file" path as a parse error.
|
|
831
|
+
if (parsed === null ||
|
|
832
|
+
typeof parsed !== 'object' ||
|
|
833
|
+
Array.isArray(parsed))
|
|
834
|
+
throw new Error('expected a JSON object at the top level');
|
|
835
|
+
reportData = parsed;
|
|
859
836
|
}
|
|
860
837
|
catch (exc) {
|
|
861
838
|
deps.out(pc.red(`Could not read result file '${opts.result}': ${exc instanceof Error ? exc.message : String(exc)}`));
|
package/dist/engine/cli.core.js
CHANGED
|
@@ -20,6 +20,10 @@ import { Command, Option } from 'commander';
|
|
|
20
20
|
import { CliExitError, normalizeUsageExit } from './cli-common.js';
|
|
21
21
|
import { createAnalyzeCommand } from './analysis/cli.js';
|
|
22
22
|
import { doctorCmd, feedbackCmd, flakeCheckCmd, listFrameworksCmd, healTestCmd, initCmd, migrateCmd, overlayCmd, promoteCheckCmd, recommendFrameworkCmd, reviewTestCmd, runCmd, setupCmd, ticketUpdateCmd, uninstallCmd, upgradeCmd, vacuityCheckCmd, versionCmd, } from './cli-commands.js';
|
|
23
|
+
import { buildBatwomanCommand } from './batwoman-cli.js';
|
|
24
|
+
import { buildCiReadyCommand } from './ci-ready-cli.js';
|
|
25
|
+
import { buildScalingCurveCommand } from './scaling-curve-cli.js';
|
|
26
|
+
import { buildPermissionMatrixCommand } from './permission-matrix-cli.js';
|
|
23
27
|
import { buildCompanyKnowledgeCommand } from './company-knowledge-cli.js';
|
|
24
28
|
import { createGuardianCommand } from './guardian/cli.js';
|
|
25
29
|
import { createHistoryCommand } from './history/cli.js';
|
|
@@ -145,6 +149,7 @@ export function createCanaryCommand(depsInit = {}) {
|
|
|
145
149
|
.description('Find tests that pass without proving anything -- advisory, no LLM required.')
|
|
146
150
|
.argument('<path>', 'Test file or directory to scan.')
|
|
147
151
|
.option('--json', 'Output the verdict and its denominator as JSON.')
|
|
152
|
+
.option('--verbose', 'List every skipped test with its file:line.')
|
|
148
153
|
.action((path, opts) => {
|
|
149
154
|
vacuityCheckCmd(path, opts, deps);
|
|
150
155
|
});
|
|
@@ -157,6 +162,13 @@ export function createCanaryCommand(depsInit = {}) {
|
|
|
157
162
|
.option('--dry-run', 'Show what would change without writing.')
|
|
158
163
|
.option('--json', 'Output results as JSON.')
|
|
159
164
|
.action((path, opts) => {
|
|
165
|
+
// heal-test applies only pattern fixes, so --no-pattern leaves nothing to
|
|
166
|
+
// do. Guarded here rather than in healTestCmd to keep that function under
|
|
167
|
+
// the complexity threshold the perf ratchet enforces.
|
|
168
|
+
if (opts.pattern === false) {
|
|
169
|
+
deps.out('Pattern fixes disabled (--no-pattern); file left unchanged.');
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
160
172
|
healTestCmd(path, opts, deps);
|
|
161
173
|
});
|
|
162
174
|
program
|
|
@@ -223,6 +235,10 @@ export function createCanaryCommand(depsInit = {}) {
|
|
|
223
235
|
program.addCommand(buildSkillsCommand(deps));
|
|
224
236
|
program.addCommand(buildWorkflowCommand(deps));
|
|
225
237
|
program.addCommand(buildCompanyKnowledgeCommand(deps));
|
|
238
|
+
program.addCommand(buildBatwomanCommand(deps));
|
|
239
|
+
program.addCommand(buildScalingCurveCommand(deps));
|
|
240
|
+
program.addCommand(buildPermissionMatrixCommand(deps));
|
|
241
|
+
program.addCommand(buildCiReadyCommand(deps));
|
|
226
242
|
// Propagate the usage-exit normalization to every top-level command (the
|
|
227
243
|
// sub-apps also set it on their own subcommands internally).
|
|
228
244
|
for (const sub of program.commands) {
|
|
@@ -78,8 +78,16 @@ function ckShowCmd(opts, deps) {
|
|
|
78
78
|
export function ckInitCmd(opts, deps) {
|
|
79
79
|
const canaryDir = join(deps.cwd(), '.canary');
|
|
80
80
|
const outPath = join(canaryDir, 'company.json');
|
|
81
|
-
// Existing values become the shown defaults (load() returns empty when
|
|
82
|
-
|
|
81
|
+
// Existing values become the shown defaults (load() returns empty when
|
|
82
|
+
// absent). `--force` means "start from scratch", which is what the
|
|
83
|
+
// already-exists banner promises and what the flag's own help text says, so
|
|
84
|
+
// it must drop those defaults entirely -- otherwise every prompt still falls
|
|
85
|
+
// back to the old value, the doc-URL accumulator and brand map are still
|
|
86
|
+
// pre-seeded, and --force only silences the banner while rewriting the old
|
|
87
|
+
// file verbatim, with no way to clear a field.
|
|
88
|
+
const existing = opts.force
|
|
89
|
+
? new CompanyKnowledge()
|
|
90
|
+
: CompanyKnowledge.load(deps.cwd(), null, deps.home());
|
|
83
91
|
if (existsSync(outPath) && !opts.force) {
|
|
84
92
|
deps.out(`${pc.yellow(WARN)} ${pc.bold(outPath)} already exists.\nExisting values will be shown as defaults. Pass ${pc.bold('--force')} to start from scratch.`);
|
|
85
93
|
deps.out('');
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/** Matches `canary analyze flaky`'s defaults: a 30-run window, 10% flake rate. */
|
|
2
|
+
const FLAKY_WINDOW_RUNS = 30;
|
|
3
|
+
const FLAKY_FAIL_RATE = 0.1;
|
|
4
|
+
const INVENTORY = '.canary/test-inventory.json';
|
|
5
|
+
const CRITICAL_AREAS = '.canary/critical-areas.json';
|
|
6
|
+
function inventoryMissingReason() {
|
|
7
|
+
return `no ${INVENTORY}: nothing in canary produces it (the documented \`canary coverage\` command does not exist)`;
|
|
8
|
+
}
|
|
9
|
+
/** Per-test flake rate across the window, for tests that appeared at all. */
|
|
10
|
+
function flakeRates(runs) {
|
|
11
|
+
const seen = new Map();
|
|
12
|
+
for (const run of runs) {
|
|
13
|
+
for (const t of run.tests ?? []) {
|
|
14
|
+
const s = seen.get(t.test_name) ?? { present: 0, flaky: 0 };
|
|
15
|
+
s.present += 1;
|
|
16
|
+
if (t.status === 'flaky')
|
|
17
|
+
s.flaky += 1;
|
|
18
|
+
seen.set(t.test_name, s);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const rates = new Map();
|
|
22
|
+
for (const [name, s] of seen) {
|
|
23
|
+
if (s.flaky > 0)
|
|
24
|
+
rates.set(name, s.flaky / s.present);
|
|
25
|
+
}
|
|
26
|
+
return rates;
|
|
27
|
+
}
|
|
28
|
+
function scoreFlakiness(runs, historyPath) {
|
|
29
|
+
const name = 'flakiness';
|
|
30
|
+
if (runs === null || runs.length === 0) {
|
|
31
|
+
return {
|
|
32
|
+
name,
|
|
33
|
+
verdict: 'skip',
|
|
34
|
+
reason: `no runs recorded in ${historyPath}`,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
const window = runs.slice(-FLAKY_WINDOW_RUNS);
|
|
38
|
+
const rates = flakeRates(window);
|
|
39
|
+
if (rates.size === 0) {
|
|
40
|
+
return {
|
|
41
|
+
name,
|
|
42
|
+
verdict: 'pass',
|
|
43
|
+
reason: `0 flaky tests across ${window.length} run(s)`,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
let worst = ['', 0];
|
|
47
|
+
for (const entry of rates)
|
|
48
|
+
if (entry[1] > worst[1])
|
|
49
|
+
worst = entry;
|
|
50
|
+
const pct = Math.round(worst[1] * 100);
|
|
51
|
+
const verdict = worst[1] >= FLAKY_FAIL_RATE ? 'fail' : 'warn';
|
|
52
|
+
return {
|
|
53
|
+
name,
|
|
54
|
+
verdict,
|
|
55
|
+
reason: `${rates.size} flaky test(s) across ${window.length} run(s); worst is ${worst[0]} at ${pct}%`,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function scoreInventoryCheck(name, hasInventory) {
|
|
59
|
+
const reason = hasInventory
|
|
60
|
+
? `${INVENTORY} is present, but no documented schema exists to score it against`
|
|
61
|
+
: inventoryMissingReason();
|
|
62
|
+
return { name, verdict: 'skip', reason };
|
|
63
|
+
}
|
|
64
|
+
function scoreCriticalPaths(hasCriticalAreas, hasInventory) {
|
|
65
|
+
const name = 'critical-paths';
|
|
66
|
+
if (!hasCriticalAreas) {
|
|
67
|
+
return {
|
|
68
|
+
name,
|
|
69
|
+
verdict: 'skip',
|
|
70
|
+
reason: `no ${CRITICAL_AREAS}, and cross-referencing it also needs ${INVENTORY}`,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
name,
|
|
75
|
+
verdict: 'skip',
|
|
76
|
+
reason: scoreInventoryCheck(name, hasInventory).reason,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function scoreRuntime() {
|
|
80
|
+
return {
|
|
81
|
+
name: 'suite-runtime',
|
|
82
|
+
verdict: 'skip',
|
|
83
|
+
reason: 'the run-history store records no durations, so a p95 runtime cannot be computed',
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
function readinessVerdict(checks) {
|
|
87
|
+
const scored = checks.filter((c) => c.verdict !== 'skip');
|
|
88
|
+
if (scored.length === 0)
|
|
89
|
+
return 'abstained';
|
|
90
|
+
if (scored.some((c) => c.verdict === 'fail'))
|
|
91
|
+
return 'not-ready';
|
|
92
|
+
if (scored.length === checks.length &&
|
|
93
|
+
scored.every((c) => c.verdict === 'pass')) {
|
|
94
|
+
return 'ready';
|
|
95
|
+
}
|
|
96
|
+
return 'incomplete';
|
|
97
|
+
}
|
|
98
|
+
export function scoreCiReady(inputs) {
|
|
99
|
+
const checks = [
|
|
100
|
+
scoreInventoryCheck('coverage-depth', inputs.hasInventory),
|
|
101
|
+
scoreFlakiness(inputs.runs, inputs.historyPath),
|
|
102
|
+
scoreInventoryCheck('assertion-quality', inputs.hasInventory),
|
|
103
|
+
scoreCriticalPaths(inputs.hasCriticalAreas, inputs.hasInventory),
|
|
104
|
+
scoreRuntime(),
|
|
105
|
+
];
|
|
106
|
+
return {
|
|
107
|
+
verdict: readinessVerdict(checks),
|
|
108
|
+
checked: checks.filter((c) => c.verdict !== 'skip').length,
|
|
109
|
+
checks,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=ci-ready.js.map
|
|
@@ -448,6 +448,11 @@ function parseLayer(data, source) {
|
|
|
448
448
|
const warns = [];
|
|
449
449
|
const confluence_spaces = validateStrings(dictGet(data, 'confluence_spaces', []), 'confluence_spaces', (v) => _SPACE_OR_PROJECT_RE.test(v), (v) => v.toUpperCase(), warns);
|
|
450
450
|
const jira_projects = validateStrings(dictGet(data, 'jira_projects', []), 'jira_projects', (v) => _SPACE_OR_PROJECT_RE.test(v), (v) => v.toUpperCase(), warns);
|
|
451
|
+
// `internal_doc_urls` cannot go through `validateStrings` (each entry needs
|
|
452
|
+
// the URL validator's own per-entry warning), but it is still a list field
|
|
453
|
+
// and must abstain out loud like one: a type-confused value used to be
|
|
454
|
+
// dropped in silence, so the operator's reference docs vanished from every
|
|
455
|
+
// generated prompt with nothing in `company-knowledge show` to say why.
|
|
451
456
|
const rawUrls = dictGet(data, 'internal_doc_urls', []);
|
|
452
457
|
const internal_doc_urls = [];
|
|
453
458
|
if (Array.isArray(rawUrls)) {
|
|
@@ -459,6 +464,9 @@ function parseLayer(data, source) {
|
|
|
459
464
|
internal_doc_urls.push(validated);
|
|
460
465
|
}
|
|
461
466
|
}
|
|
467
|
+
else {
|
|
468
|
+
warns.push(`internal_doc_urls: expected list, got ${pyTypeName(rawUrls)} ${EMDASH} skipped`);
|
|
469
|
+
}
|
|
462
470
|
const internal_domains = validateStrings(dictGet(data, 'internal_domains', []), 'internal_domains', (v) => _DOMAIN_RE.test(v), (v) => v.toLowerCase(), warns);
|
|
463
471
|
const mcp_servers = validateStrings(dictGet(data, 'mcp_servers', []), 'mcp_servers', (v) => _MCP_SERVER_RE.test(v), null, warns);
|
|
464
472
|
const claude_code_skills = validateStrings(dictGet(data, 'claude_code_skills', []), 'claude_code_skills', (v) => _SKILL_RE.test(v), (v) => v.toLowerCase(), warns);
|