canary-test-cli 7.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +23 -4
- package/agents/skills/claude-code/canary-batwoman/SKILL.md +119 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +23 -16
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +3 -1
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +20 -3
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +1 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +15 -0
- package/agents/skills/claude-code/canary-screech/SKILL.md +109 -0
- package/agents/skills/claude-code/canary-screech/scripts/blast.mjs +125 -0
- package/agents/skills/claude-code/canary-screech/scripts/cli.mjs +128 -0
- package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs +97 -0
- package/agents/skills/claude-code/canary-screech/scripts/history.mjs +73 -0
- package/agents/skills/claude-code/canary-screech/scripts/redness.mjs +94 -0
- package/agents/skills/lib/parse-args.mjs +200 -139
- package/dist/engine/analysis/batwoman/audit.js +39 -0
- package/dist/engine/analysis/batwoman/closure.js +159 -0
- package/dist/engine/analysis/batwoman/gh-history.js +119 -0
- package/dist/engine/analysis/batwoman/probes.js +195 -0
- package/dist/engine/analysis/batwoman/registry.js +142 -0
- package/dist/engine/analysis/batwoman/render.js +194 -0
- package/dist/engine/analysis/batwoman/run-window.js +122 -0
- package/dist/engine/analysis/batwoman/text.js +84 -0
- package/dist/engine/analysis/batwoman/triggers.js +122 -0
- package/dist/engine/analysis/batwoman/verdict.js +64 -0
- package/dist/engine/analysis/cli.js +47 -14
- package/dist/engine/analysis/gh-flaky/gh-run-attempts.js +206 -0
- package/dist/engine/batwoman-cli.js +119 -0
- package/dist/engine/ci-ready-cli.js +71 -0
- package/dist/engine/cli-commands.js +46 -7
- package/dist/engine/cli.core.js +16 -0
- package/dist/engine/company-knowledge-cli.js +10 -2
- package/dist/engine/core/ci-ready.js +112 -0
- package/dist/engine/core/company-knowledge.js +8 -0
- package/dist/engine/core/migrator.js +147 -20
- package/dist/engine/core/permission-matrix.js +219 -0
- package/dist/engine/core/quality-scorer.js +13 -18
- package/dist/engine/core/scaling-curve.js +143 -0
- package/dist/engine/core/string-literals.js +3 -1
- package/dist/engine/core/vacuity-scanner.js +151 -6
- package/dist/engine/core/workflow-discovery.js +41 -23
- package/dist/engine/guardian/adjudication-github.js +136 -0
- package/dist/engine/guardian/adjudication.js +119 -340
- package/dist/engine/guardian/cli.js +180 -264
- package/dist/engine/guardian/coverage.js +2 -1
- package/dist/engine/guardian/diff-coverage/coverage-delta.js +162 -0
- package/dist/engine/guardian/diff-coverage/formats/cobertura.js +45 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +25 -21
- package/dist/engine/guardian/diff-coverage/paths.js +5 -9
- package/dist/engine/guardian/diff-coverage/report-tier.js +88 -12
- package/dist/engine/guardian/diff-extractor.js +31 -32
- package/dist/engine/guardian/pr-check.js +262 -430
- package/dist/engine/guardian/pr-comment.js +35 -58
- package/dist/engine/guardian/weak-test.js +236 -0
- package/dist/engine/mcp-server.js +67 -4
- package/dist/engine/permission-matrix-cli.js +51 -0
- package/dist/engine/scaling-curve-cli.js +147 -0
- package/dist/engine/skills-cli.js +48 -32
- package/dist/engine/workflow-cli.js +85 -65
- package/package.json +1 -1
|
@@ -1,42 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Deterministic
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* This module posts/updates the single sticky guardian findings comment on a
|
|
7
|
-
* pull request. It is **deterministic HTTP behind an interface seam** — it
|
|
8
|
-
* imports no agent/LLM module (SC-11).
|
|
9
|
-
*
|
|
10
|
-
* Design:
|
|
11
|
-
*
|
|
12
|
-
* - {@link GitHubClient} is the seam every consumer talks to
|
|
13
|
-
* (`listComments` / `createComment` / `updateComment`).
|
|
14
|
-
* - {@link FakeGitHubClient} is the in-memory implementation used by every unit
|
|
15
|
-
* test — **no network**. It can simulate a fork read-only token via
|
|
16
|
-
* `deny_writes=true` (writes reject with {@link GitHubPermissionError}).
|
|
17
|
-
* - {@link RestGitHubClient} (Python's private `_RestGitHubClient`) is the thin
|
|
18
|
-
* real client. Network lives **only** here; `guardian-rest-clients.test.ts`
|
|
19
|
-
* drives it through a stubbed global `fetch`, so the URL, headers, error
|
|
20
|
-
* mapping, and #528 pagination are covered without a socket.
|
|
21
|
-
*
|
|
22
|
-
* Python→TS nuances:
|
|
23
|
-
* - **async**: Python's `urllib` client is synchronous; Node's global `fetch`
|
|
24
|
-
* is async. The seam methods are therefore `Promise`-returning, so the
|
|
25
|
-
* real client can `await fetch`. The fakes satisfy the async interface by
|
|
26
|
-
* being `async` (returning already-resolved values), and
|
|
27
|
-
* {@link upsertStickyComment} becomes `async`. The pure helpers
|
|
28
|
-
* ({@link findSticky}, {@link degradationAnnotation}) stay synchronous.
|
|
29
|
-
* - **error mapping**: `fetch` resolves (does not throw) on a 4xx/5xx status,
|
|
30
|
-
* so the 403→{@link GitHubPermissionError} mapping is done off `resp.status`
|
|
31
|
-
* rather than off a raised `HTTPError`. As in the oracle, ONLY 403 maps to a
|
|
32
|
-
* permission error here; any other non-2xx propagates as a generic error.
|
|
2
|
+
* Deterministic poster of the single sticky guardian PR comment (Tier 0; no
|
|
3
|
+
* agent/LLM import, SC-11). {@link GitHubClient} is the seam;
|
|
4
|
+
* {@link FakeGitHubClient} is the in-memory test double (`deny_writes` models a
|
|
5
|
+
* 403); {@link RestGitHubClient} is the only code that touches the network.
|
|
33
6
|
*/
|
|
34
7
|
import { readAllPages, restPageReader } from './github-paging.js';
|
|
35
|
-
//
|
|
36
|
-
// `pr_check.renderFindings` emits the identical literal at the head of a
|
|
37
|
-
// `comment`-format body so `findSticky` can locate the guardian comment for
|
|
38
|
-
// in-place upsert.
|
|
8
|
+
// `renderFindings` writes this literal on a comment body's first line.
|
|
39
9
|
export const STICKY_MARKER = '<!-- canary-pr-guardian -->';
|
|
10
|
+
const DEFAULT_IDENTITY = { login: 'github-actions[bot]' };
|
|
40
11
|
/**
|
|
41
12
|
* A client cannot write (fork read-only token → HTTP 403).
|
|
42
13
|
*
|
|
@@ -49,12 +20,7 @@ export class GitHubPermissionError extends Error {
|
|
|
49
20
|
this.name = 'GitHubPermissionError';
|
|
50
21
|
}
|
|
51
22
|
}
|
|
52
|
-
/**
|
|
53
|
-
* Map a non-2xx status to an error. As in the Python reference, ONLY 403 is a
|
|
54
|
-
* permission error; every other non-2xx propagates (the analog of urllib's
|
|
55
|
-
* HTTPError re-raise). Shared by the write path and the paged read path so the
|
|
56
|
-
* two cannot drift.
|
|
57
|
-
*/
|
|
23
|
+
/** Map a non-2xx status to an error; ONLY 403 is a permission error. */
|
|
58
24
|
function toGitHubError(status, url) {
|
|
59
25
|
return status === 403
|
|
60
26
|
? new GitHubPermissionError(`GitHub API 403 (read-only token / fork PR?): ${url}`)
|
|
@@ -84,7 +50,11 @@ export class FakeGitHubClient {
|
|
|
84
50
|
throw new GitHubPermissionError('read-only token: cannot create comment');
|
|
85
51
|
}
|
|
86
52
|
this.nextId += 1;
|
|
87
|
-
const row = {
|
|
53
|
+
const row = {
|
|
54
|
+
id: this.nextId,
|
|
55
|
+
body,
|
|
56
|
+
user: { login: DEFAULT_IDENTITY.login, type: 'Bot' },
|
|
57
|
+
};
|
|
88
58
|
this.comments.push(row);
|
|
89
59
|
return row;
|
|
90
60
|
}
|
|
@@ -101,21 +71,31 @@ export class FakeGitHubClient {
|
|
|
101
71
|
throw new Error(`no comment with id ${commentId}`);
|
|
102
72
|
}
|
|
103
73
|
}
|
|
104
|
-
/**
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Return guardian's sticky comment, else `null` (#931). A comment qualifies
|
|
76
|
+
* only when its body STARTS WITH `marker` and `identity` wrote it, so a human
|
|
77
|
+
* who pasted guardian's output is never overwritten. Newest qualifier wins.
|
|
78
|
+
*/
|
|
79
|
+
export function findSticky(comments, marker = STICKY_MARKER, identity = DEFAULT_IDENTITY) {
|
|
80
|
+
let best = null;
|
|
81
|
+
for (const c of comments.filter((x) => isSticky(x, marker, identity))) {
|
|
82
|
+
if (!best || (c.created_at ?? '') >= (best.created_at ?? ''))
|
|
83
|
+
best = c;
|
|
110
84
|
}
|
|
111
|
-
return
|
|
85
|
+
return best;
|
|
86
|
+
}
|
|
87
|
+
function isSticky(c, marker, id) {
|
|
88
|
+
return isAuthoredBy(c, id) && (c.body ?? '').trimStart().startsWith(marker);
|
|
89
|
+
}
|
|
90
|
+
function isAuthoredBy(c, identity) {
|
|
91
|
+
if (c.user?.login === identity.login)
|
|
92
|
+
return true;
|
|
93
|
+
const slug = c.performed_via_github_app?.slug;
|
|
94
|
+
return c.user?.type === 'Bot' && !!slug && slug === identity.appSlug;
|
|
112
95
|
}
|
|
113
96
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
* Locates the existing comment by `marker`; updates it in place when present,
|
|
117
|
-
* otherwise creates a new one. Never stacks duplicates. A read-only token (fork
|
|
118
|
-
* PR?) degrades loudly to a `degraded` result rather than crashing (OT-4).
|
|
97
|
+
* Update guardian's own sticky in place, else create one (SC-9: never stacks).
|
|
98
|
+
* A 403 degrades to a `degraded` result instead of crashing the job (OT-4).
|
|
119
99
|
*/
|
|
120
100
|
export async function upsertStickyComment(client, body, marker = STICKY_MARKER) {
|
|
121
101
|
const existing = findSticky(await client.listComments(), marker);
|
|
@@ -129,13 +109,10 @@ export async function upsertStickyComment(client, body, marker = STICKY_MARKER)
|
|
|
129
109
|
}
|
|
130
110
|
catch (err) {
|
|
131
111
|
if (err instanceof GitHubPermissionError) {
|
|
132
|
-
// OT-4 / SC-1+D6: a read-only token (fork PR?) must degrade loudly, not
|
|
133
|
-
// crash the job. The caller emits `notice` as a `::warning::` annotation.
|
|
134
112
|
return {
|
|
135
113
|
action: 'degraded',
|
|
136
114
|
comment_id: null,
|
|
137
|
-
notice:
|
|
138
|
-
'a comment',
|
|
115
|
+
notice: `guardian: token lacks write permission on PR comments (HTTP 403) — findings not posted as a comment`,
|
|
139
116
|
};
|
|
140
117
|
}
|
|
141
118
|
throw err;
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The advisory `weak-test` finding: an ADDED test that asserts nothing (#747).
|
|
3
|
+
*
|
|
4
|
+
* Split out of `pr-check.ts` as a self-contained judgement — test declarations,
|
|
5
|
+
* block spans, assertion presence. The dependency runs one way (this module
|
|
6
|
+
* reads `pr-check`, never the reverse); `guardian/cli.ts` calls into here.
|
|
7
|
+
*/
|
|
8
|
+
import { readFileSync } from 'node:fs';
|
|
9
|
+
import { Fidelity } from './coverage.js';
|
|
10
|
+
import { Severity } from './impact-mapper.js';
|
|
11
|
+
import { GuardianFinding, addedContentByPath, linesInRanges, walkDiff, } from './pr-check.js';
|
|
12
|
+
import { hasAssertion, isAssertionFreeTest } from '../core/quality-scorer.js';
|
|
13
|
+
// Test-file extension -> the quality scorer's framework. Unknown -> pytest.
|
|
14
|
+
const TEST_FRAMEWORK_BY_EXT = {
|
|
15
|
+
'.py': 'pytest',
|
|
16
|
+
'.ts': 'vitest',
|
|
17
|
+
'.tsx': 'vitest',
|
|
18
|
+
'.js': 'vitest',
|
|
19
|
+
'.jsx': 'vitest',
|
|
20
|
+
'.mjs': 'vitest',
|
|
21
|
+
'.cjs': 'vitest',
|
|
22
|
+
};
|
|
23
|
+
function frameworkForTestPath(path) {
|
|
24
|
+
const ext = /\.[^./\\]+$/.exec(path)?.[0] ?? '';
|
|
25
|
+
return TEST_FRAMEWORK_BY_EXT[ext.toLowerCase()] ?? 'pytest';
|
|
26
|
+
}
|
|
27
|
+
// A signature / decorator / block-close / comment — not a test *body*. A rename
|
|
28
|
+
// that adds only `def test_new():` over an unchanged body has nothing to judge.
|
|
29
|
+
const TEST_SIGNATURE_RE = /^\s*(?:async\s+)?def\s+test\w*\s*\(|^\s*(?:it|test|describe)\s*\(/;
|
|
30
|
+
const BLOCK_DELIMITERS = new Set(['})', '});', '}', ')', '{']);
|
|
31
|
+
const NON_BODY_PREFIXES = ['#', '//', '@', '*', '/*'];
|
|
32
|
+
/** True iff `added` holds a real body line, not just signature/comment/brace. */
|
|
33
|
+
function hasAddedTestBody(added) {
|
|
34
|
+
return added.some((line) => {
|
|
35
|
+
const stripped = line.trim();
|
|
36
|
+
if (!stripped || BLOCK_DELIMITERS.has(stripped))
|
|
37
|
+
return false;
|
|
38
|
+
if (NON_BODY_PREFIXES.some((p) => stripped.startsWith(p)))
|
|
39
|
+
return false;
|
|
40
|
+
return !TEST_SIGNATURE_RE.test(line);
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
// The declaration of a single test. Narrower than TEST_SIGNATURE_RE on purpose:
|
|
44
|
+
// `describe(` opens a group, and judging a whole group would let an asserting
|
|
45
|
+
// sibling excuse an empty test. `it.only` / `test.each` still open one test.
|
|
46
|
+
const TEST_DECL_PY = /^\s*(?:async\s+)?def\s+test\w*\s*\(/;
|
|
47
|
+
const TEST_DECL_JS = /^\s*(?:async\s+)?(?:it|test)(?:\.\w+)*\s*\(/;
|
|
48
|
+
// A same-file helper definition (#929): `function f(`, `const f = (` / `=
|
|
49
|
+
// function` / `= x =>`, or Python `def f(`.
|
|
50
|
+
const HELPER_DEF_PY = /^\s*(?:async\s+)?def\s+(\w+)\s*\(/;
|
|
51
|
+
const HELPER_DEF_JS = /^\s*(?:export\s+)?(?:async\s+)?(?:function\s*\*?\s*(\w+)\s*\(|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:function\b|\(|\w+\s*=>))/;
|
|
52
|
+
const TEST_TITLE_RE = /\(\s*(['"`])((?:\\.|(?!\1).)*)\1|def\s+(\w+)/;
|
|
53
|
+
function indentWidth(line) {
|
|
54
|
+
return line.length - line.trimStart().length;
|
|
55
|
+
}
|
|
56
|
+
// Strings and comments are blanked before delimiter counting.
|
|
57
|
+
const JS_STRING_OR_COMMENT = /(['"`])(?:\\.|(?!\1).)*?\1|\/\/.*$|\/\*[\s\S]*?\*\//g;
|
|
58
|
+
const NEW_FILE_RE = /^--- \/dev\/null\r?\n\+\+\+ b\/(.+?)\r?$/gm;
|
|
59
|
+
function visibleLinesByPath(diffText) {
|
|
60
|
+
const files = new Map();
|
|
61
|
+
walkDiff(diffText, (line) => {
|
|
62
|
+
let file = files.get(line.path);
|
|
63
|
+
if (!file) {
|
|
64
|
+
file = { text: new Map(), added: new Set(), eof: null };
|
|
65
|
+
files.set(line.path, file);
|
|
66
|
+
}
|
|
67
|
+
file.text.set(line.lineno, line.text);
|
|
68
|
+
if (line.added)
|
|
69
|
+
file.added.add(line.lineno);
|
|
70
|
+
});
|
|
71
|
+
for (const m of diffText.matchAll(NEW_FILE_RE)) {
|
|
72
|
+
const file = files.get(m[1]);
|
|
73
|
+
if (file)
|
|
74
|
+
file.eof = Math.max(...file.text.keys());
|
|
75
|
+
}
|
|
76
|
+
return files;
|
|
77
|
+
}
|
|
78
|
+
/** The file's own text under `root`, or `null` when it cannot be read. */
|
|
79
|
+
function fileAtRoot(root, path) {
|
|
80
|
+
try {
|
|
81
|
+
const lines = readFileSync(`${root}/${path}`, 'utf-8').split(/\r?\n/);
|
|
82
|
+
const text = new Map(lines.map((t, i) => [i + 1, t]));
|
|
83
|
+
return { text, added: new Set(), eof: lines.length };
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** The enclosing test's declaration line in the contiguous visible run, or null. */
|
|
90
|
+
function enclosingTestDecl(file, lineNo, declRe) {
|
|
91
|
+
for (let n = lineNo; file.text.has(n); n--) {
|
|
92
|
+
if (declRe.test(file.text.get(n)))
|
|
93
|
+
return n;
|
|
94
|
+
}
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
/** Python: a block closes before the first non-blank line dedented to `def`. */
|
|
98
|
+
function pythonBlockEnd(file, start) {
|
|
99
|
+
const declIndent = indentWidth(file.text.get(start));
|
|
100
|
+
let n = start + 1;
|
|
101
|
+
for (; file.text.has(n); n++) {
|
|
102
|
+
const text = file.text.get(n);
|
|
103
|
+
if (text.trim() && indentWidth(text) <= declIndent)
|
|
104
|
+
return n - 1;
|
|
105
|
+
}
|
|
106
|
+
return n - 1 === file.eof ? n - 1 : null;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* The last line of the block opened at `start`, or `null` when its end is not
|
|
110
|
+
* inside the visible run (#929). JS/TS closes when delimiter depth returns to
|
|
111
|
+
* zero. An unknown end means the assertion may be exactly what is out of view
|
|
112
|
+
* — the dominant case, since a test's setup is edited far more than its
|
|
113
|
+
* `expect` — so the caller abstains rather than scoring half a block.
|
|
114
|
+
*/
|
|
115
|
+
function blockEnd(file, start, isPython) {
|
|
116
|
+
if (isPython)
|
|
117
|
+
return pythonBlockEnd(file, start);
|
|
118
|
+
let depth = 0;
|
|
119
|
+
let opened = false;
|
|
120
|
+
let n = start;
|
|
121
|
+
for (; file.text.has(n); n++) {
|
|
122
|
+
const code = file.text.get(n).replace(JS_STRING_OR_COMMENT, '');
|
|
123
|
+
const opens = code.replace(/[^{(]/g, '').length;
|
|
124
|
+
depth += opens - code.replace(/[^})]/g, '').length;
|
|
125
|
+
opened ||= opens > 0;
|
|
126
|
+
if (opened && depth <= 0)
|
|
127
|
+
return n;
|
|
128
|
+
}
|
|
129
|
+
return n - 1 === file.eof ? n - 1 : null;
|
|
130
|
+
}
|
|
131
|
+
function linesOf(file, start, end) {
|
|
132
|
+
const out = [];
|
|
133
|
+
for (let n = start; n <= end; n++) {
|
|
134
|
+
const text = file.text.get(n);
|
|
135
|
+
if (text !== undefined)
|
|
136
|
+
out.push(text);
|
|
137
|
+
}
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
/** Same-file helpers whose CLOSED body asserts, from diff and disk text (#929). */
|
|
141
|
+
function assertingHelpers(sources, framework) {
|
|
142
|
+
const names = new Set();
|
|
143
|
+
for (const src of sources) {
|
|
144
|
+
for (const n of src.text.keys()) {
|
|
145
|
+
const name = assertingHelperAt(src, n, framework);
|
|
146
|
+
if (name)
|
|
147
|
+
names.add(name);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return [...names];
|
|
151
|
+
}
|
|
152
|
+
/** The helper defined at line `n` when its closed body asserts, else null. */
|
|
153
|
+
function assertingHelperAt(src, n, framework) {
|
|
154
|
+
const isPython = framework === 'pytest';
|
|
155
|
+
const m = (isPython ? HELPER_DEF_PY : HELPER_DEF_JS).exec(src.text.get(n));
|
|
156
|
+
const name = m?.[1] ?? m?.[2];
|
|
157
|
+
const end = name ? blockEnd(src, n, isPython) : null;
|
|
158
|
+
if (end === null)
|
|
159
|
+
return null;
|
|
160
|
+
return hasAssertion(linesOf(src, n, end).join('\n'), framework)
|
|
161
|
+
? name
|
|
162
|
+
: null;
|
|
163
|
+
}
|
|
164
|
+
/** True iff the block asserts nothing and has an added body to judge. */
|
|
165
|
+
function isWeakBlock(file, [start, end], framework, helpers) {
|
|
166
|
+
const span = linesOf(file, start, end);
|
|
167
|
+
const added = linesOf(file, start, end).filter((_, i) => file.added.has(start + i));
|
|
168
|
+
// FP-3: only a signature was added over an unchanged body. A wholly added
|
|
169
|
+
// block (e.g. `it('x', () => {})` on one line) is still judged.
|
|
170
|
+
if (!hasAddedTestBody(added) && added.length < span.length)
|
|
171
|
+
return false;
|
|
172
|
+
const code = span.join('\n');
|
|
173
|
+
if (!isAssertionFreeTest(code, framework))
|
|
174
|
+
return false;
|
|
175
|
+
return !helpers.some((h) => new RegExp(`\\b${h}\\s*\\(`).test(code));
|
|
176
|
+
}
|
|
177
|
+
/** The weak test blocks touched by `unit`'s added lines, each visited once. */
|
|
178
|
+
function weakBlocksIn(file, unit, framework, helpers) {
|
|
179
|
+
const declRe = framework === 'pytest' ? TEST_DECL_PY : TEST_DECL_JS;
|
|
180
|
+
const seen = new Set();
|
|
181
|
+
const weak = [];
|
|
182
|
+
for (const lineNo of linesInRanges(unit.added_ranges)) {
|
|
183
|
+
const start = enclosingTestDecl(file, lineNo, declRe);
|
|
184
|
+
if (start === null || seen.has(start))
|
|
185
|
+
continue;
|
|
186
|
+
seen.add(start);
|
|
187
|
+
const end = blockEnd(file, start, framework === 'pytest');
|
|
188
|
+
if (end === null)
|
|
189
|
+
continue;
|
|
190
|
+
if (isWeakBlock(file, [start, end], framework, helpers)) {
|
|
191
|
+
weak.push([start, end]);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return weak;
|
|
195
|
+
}
|
|
196
|
+
function testTitle(decl) {
|
|
197
|
+
const m = TEST_TITLE_RE.exec(decl);
|
|
198
|
+
return m?.[2] ?? m?.[3] ?? decl.trim();
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Advisory `weak-test` findings for ADDED tests that assert nothing — one per
|
|
202
|
+
* weak test block, naming its title and line span (#929).
|
|
203
|
+
*
|
|
204
|
+
* The span scored is the ENCLOSING TEST BLOCK (#747); a block that cannot be
|
|
205
|
+
* resolved, or whose end is out of view, is abstained on. `repoRoot` is where
|
|
206
|
+
* the file's own text is read for helper resolution (`.`, the CLI's repoRoot
|
|
207
|
+
* convention). Findings are `LOW` and never gated (see computeExitCode).
|
|
208
|
+
*/
|
|
209
|
+
export function buildWeakTestFindings(testUnits, diffText, repoRoot = '.') {
|
|
210
|
+
const addedByPath = addedContentByPath(diffText);
|
|
211
|
+
const visibleByPath = visibleLinesByPath(diffText);
|
|
212
|
+
const findings = [];
|
|
213
|
+
for (const unit of testUnits) {
|
|
214
|
+
const file = visibleByPath.get(unit.path);
|
|
215
|
+
if (!file || !addedByPath.get(unit.path)?.length)
|
|
216
|
+
continue;
|
|
217
|
+
const framework = frameworkForTestPath(unit.path);
|
|
218
|
+
const disk = fileAtRoot(repoRoot, unit.path);
|
|
219
|
+
const helpers = assertingHelpers(disk ? [file, disk] : [file], framework);
|
|
220
|
+
for (const [start, end] of weakBlocksIn(file, unit, framework, helpers)) {
|
|
221
|
+
const title = testTitle(file.text.get(start));
|
|
222
|
+
findings.push(new GuardianFinding({
|
|
223
|
+
path: unit.path,
|
|
224
|
+
unit: unit.path,
|
|
225
|
+
kind: 'weak-test',
|
|
226
|
+
fidelity: Fidelity.Heuristic,
|
|
227
|
+
severity: Severity.LOW,
|
|
228
|
+
evidence: `added test "${title}" (L${start}–L${end}) asserts nothing (advisory — never blocks the gate)`,
|
|
229
|
+
suggestion: 'add at least one assertion, or delete the test if it is a placeholder.',
|
|
230
|
+
added_ranges: [[start, end]],
|
|
231
|
+
}));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
return findings;
|
|
235
|
+
}
|
|
236
|
+
//# sourceMappingURL=weak-test.js.map
|
|
@@ -40,8 +40,8 @@
|
|
|
40
40
|
* a single trailing-newline empty tail exactly as `str.splitlines()` does.
|
|
41
41
|
* - File writes are LF + UTF-8 on every platform (matches the sibling ports).
|
|
42
42
|
*/
|
|
43
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
44
|
-
import { basename, dirname, extname, join, relative, resolve } from 'node:path';
|
|
43
|
+
import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync, } from 'node:fs';
|
|
44
|
+
import { basename, dirname, extname, isAbsolute, join, relative, resolve, } from 'node:path';
|
|
45
45
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
46
46
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
47
47
|
import * as z from 'zod';
|
|
@@ -476,8 +476,60 @@ export function analyzeFileImpl(filePath) {
|
|
|
476
476
|
persona,
|
|
477
477
|
};
|
|
478
478
|
}
|
|
479
|
+
/**
|
|
480
|
+
* Resolve `candidate` against `root` and return it only if it stays inside.
|
|
481
|
+
*
|
|
482
|
+
* A tool argument arrives from the MCP host, so a path is untrusted input: an
|
|
483
|
+
* absolute path, a `..` climb, or a symlinked parent directory would otherwise
|
|
484
|
+
* let a caller reach outside the served project. Containment is therefore
|
|
485
|
+
* checked *after* symlink resolution -- a textual `..`-and-absolute test looks
|
|
486
|
+
* like a guard but cannot see a symlink, which is the gap this exists to close.
|
|
487
|
+
*
|
|
488
|
+
* `realpathSync` throws on a missing path, so the nearest existing ancestor is
|
|
489
|
+
* resolved and the not-yet-created tail re-joined onto it. That is what lets a
|
|
490
|
+
* write to a new file still be checked against the real location of the
|
|
491
|
+
* directory it would be created in. Mirrors `resolveCliPath`
|
|
492
|
+
* (`core/skill-registry.ts`), the guard already proven in this codebase.
|
|
493
|
+
*
|
|
494
|
+
* @returns the resolved absolute path, or `null` when it escapes `root`.
|
|
495
|
+
*/
|
|
496
|
+
export function resolveInsideRoot(candidate, root) {
|
|
497
|
+
let realRoot;
|
|
498
|
+
try {
|
|
499
|
+
realRoot = realpathSync(resolve(root));
|
|
500
|
+
}
|
|
501
|
+
catch {
|
|
502
|
+
return null;
|
|
503
|
+
}
|
|
504
|
+
const target = resolve(realRoot, candidate);
|
|
505
|
+
// Walk up to the nearest ancestor that exists, remembering the tail.
|
|
506
|
+
const tail = [];
|
|
507
|
+
let probe = target;
|
|
508
|
+
while (!existsSync(probe)) {
|
|
509
|
+
const parent = dirname(probe);
|
|
510
|
+
if (parent === probe)
|
|
511
|
+
return null;
|
|
512
|
+
tail.unshift(basename(probe));
|
|
513
|
+
probe = parent;
|
|
514
|
+
}
|
|
515
|
+
let resolved;
|
|
516
|
+
try {
|
|
517
|
+
resolved = join(realpathSync(probe), ...tail);
|
|
518
|
+
}
|
|
519
|
+
catch {
|
|
520
|
+
return null;
|
|
521
|
+
}
|
|
522
|
+
const rel = relative(realRoot, resolved);
|
|
523
|
+
if (rel !== '' && (rel.startsWith('..') || isAbsolute(rel)))
|
|
524
|
+
return null;
|
|
525
|
+
return resolved;
|
|
526
|
+
}
|
|
527
|
+
/** The refusal an MCP tool returns when a path argument escapes its root. */
|
|
528
|
+
function escapeError(path) {
|
|
529
|
+
return { error: `path escapes the project root: ${path}` };
|
|
530
|
+
}
|
|
479
531
|
/** Python: `_write_test_file_impl`. */
|
|
480
|
-
export function writeTestFileImpl(filePath, content, framework) {
|
|
532
|
+
export function writeTestFileImpl(filePath, content, framework, root = WORKING_DIR) {
|
|
481
533
|
let outPath = filePath;
|
|
482
534
|
if (pySuffix(basename(filePath)) === '') {
|
|
483
535
|
const extMap = {
|
|
@@ -490,12 +542,23 @@ export function writeTestFileImpl(filePath, content, framework) {
|
|
|
490
542
|
// with no existing suffix to strip, that is a plain concatenation.
|
|
491
543
|
outPath = filePath + (extMap[framework] ?? '.ts');
|
|
492
544
|
}
|
|
545
|
+
// Checked on the FINAL path, after the extension is inferred, and before the
|
|
546
|
+
// first filesystem mutation -- `mkdirSync` below would otherwise create
|
|
547
|
+
// directories outside the root even when the write itself were refused.
|
|
548
|
+
if (resolveInsideRoot(outPath, root) === null)
|
|
549
|
+
return escapeError(outPath);
|
|
493
550
|
mkdirSync(dirname(outPath), { recursive: true });
|
|
494
551
|
writeFileSync(outPath, content, 'utf-8');
|
|
495
552
|
return { written_path: outPath };
|
|
496
553
|
}
|
|
497
554
|
/** Python: `_run_tests_impl`. */
|
|
498
|
-
export function runTestsImpl(testFile) {
|
|
555
|
+
export function runTestsImpl(testFile, root = WORKING_DIR) {
|
|
556
|
+
// The same containment the write path gets: this spawns a test runner over
|
|
557
|
+
// the named file, so an unconfined path is arbitrary code execution, not
|
|
558
|
+
// merely an arbitrary read. Refused before the executor is constructed.
|
|
559
|
+
if (resolveInsideRoot(testFile, root) === null) {
|
|
560
|
+
return { passed: 0, failed: 1, exit_code: 1, ...escapeError(testFile) };
|
|
561
|
+
}
|
|
499
562
|
const suffix = extname(testFile).toLowerCase();
|
|
500
563
|
const framework = suffix === '.py' ? 'pytest' : 'playwright';
|
|
501
564
|
const executor = new CanaryTestExecutor();
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `canary permission-matrix <model.yaml>` (#857).
|
|
3
|
+
*
|
|
4
|
+
* Writes the generated Playwright suite even when cells are undeclared (as
|
|
5
|
+
* `test.fixme`, so they are counted), then exits 3: a matrix with holes cannot
|
|
6
|
+
* verify the boundary it describes, and must not read as a clean generation.
|
|
7
|
+
*/
|
|
8
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
9
|
+
import { Command } from 'commander';
|
|
10
|
+
import pc from 'picocolors';
|
|
11
|
+
import { CliExitError, jsonIndent2 } from './cli-common.js';
|
|
12
|
+
import { EXIT_ABSTAINED } from './core/gate-result.js';
|
|
13
|
+
import { expandMatrix, parseMatrix, probedCells, renderPlaywright, } from './core/permission-matrix.js';
|
|
14
|
+
import { WARN } from './main-deps.js';
|
|
15
|
+
function writeSuite(parsed, cells, undeclared, opts, deps) {
|
|
16
|
+
writeFileSync(opts.out, renderPlaywright(cells, parsed.existenceProbes), 'utf-8');
|
|
17
|
+
const cross = cells.filter((c) => c.actingTenant !== c.targetTenant).length;
|
|
18
|
+
const probes = probedCells(cells, parsed.existenceProbes).length;
|
|
19
|
+
deps.out(`Wrote ${cells.length} cell test(s)` +
|
|
20
|
+
(probes > 0 ? ` + ${probes} existence probe(s)` : '') +
|
|
21
|
+
` to ${opts.out} ` +
|
|
22
|
+
pc.dim(`(${cross} cross-tenant)`));
|
|
23
|
+
if (undeclared.length === 0)
|
|
24
|
+
return;
|
|
25
|
+
deps.out(pc.bold(pc.yellow(`${WARN} ${undeclared.length} UNDECLARED cell(s) -- emitted as ` +
|
|
26
|
+
'test.fixme; this matrix cannot verify them:')));
|
|
27
|
+
for (const u of undeclared)
|
|
28
|
+
deps.out(` - ${u}`);
|
|
29
|
+
}
|
|
30
|
+
export function buildPermissionMatrixCommand(deps) {
|
|
31
|
+
return new Command('permission-matrix')
|
|
32
|
+
.description('Generate server-direct authz tests from a declared role x endpoint ' +
|
|
33
|
+
'grid, across every acting x target tenant. Undeclared cells exit 3.')
|
|
34
|
+
.argument('<model>', 'YAML model: roles, tenants, endpoints grid')
|
|
35
|
+
.option('--out <file>', 'where to write the Playwright suite', 'permission-matrix.spec.ts')
|
|
36
|
+
.option('--json', 'Print the expanded cells as JSON instead of writing.')
|
|
37
|
+
.action((model, opts) => {
|
|
38
|
+
const parsed = parseMatrix(readFileSync(model, 'utf-8'));
|
|
39
|
+
const { cells, undeclared } = expandMatrix(parsed);
|
|
40
|
+
if (opts.json) {
|
|
41
|
+
const { existenceProbes } = parsed;
|
|
42
|
+
deps.out(jsonIndent2({ cells, undeclared, existenceProbes }));
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
writeSuite(parsed, cells, undeclared, opts, deps);
|
|
46
|
+
}
|
|
47
|
+
if (undeclared.length > 0)
|
|
48
|
+
throw new CliExitError(EXIT_ABSTAINED);
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=permission-matrix-cli.js.map
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `canary scaling-curve <points-file>` (#856).
|
|
3
|
+
*
|
|
4
|
+
* Advisory: a verdict exits 0 whatever the exponent, because this is a planning
|
|
5
|
+
* signal, not a gate. An abstention exits 3 (ADR 0009) -- "not enough data to
|
|
6
|
+
* say" must never look like "linear".
|
|
7
|
+
*/
|
|
8
|
+
import { spawnSync } from 'node:child_process';
|
|
9
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
10
|
+
import { tmpdir } from 'node:os';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { Command, InvalidArgumentError } from 'commander';
|
|
13
|
+
import pc from 'picocolors';
|
|
14
|
+
import { CliExitError, jsonIndent2 } from './cli-common.js';
|
|
15
|
+
import { EXIT_ABSTAINED } from './core/gate-result.js';
|
|
16
|
+
import { fitScaling, parsePoints } from './core/scaling-curve.js';
|
|
17
|
+
import { WARN } from './main-deps.js';
|
|
18
|
+
const MEANING = {
|
|
19
|
+
LINEAR_OR_BETTER: 'cost grows no faster than input',
|
|
20
|
+
SUPERLINEAR: 'cost grows faster than input (n log n territory)',
|
|
21
|
+
STRONGLY_SUPERLINEAR: 'cost grows much faster than input (quadratic territory) -- will buckle at peak',
|
|
22
|
+
};
|
|
23
|
+
function positive(raw) {
|
|
24
|
+
const n = Number(raw);
|
|
25
|
+
if (!(Number.isFinite(n) && n > 0)) {
|
|
26
|
+
throw new InvalidArgumentError(`must be a positive number, got "${raw}"`);
|
|
27
|
+
}
|
|
28
|
+
return n;
|
|
29
|
+
}
|
|
30
|
+
/** `trend:stat`, e.g. `http_req_duration:p(95)`, read from k6's summary export. */
|
|
31
|
+
const defaultK6Runner = (script, size, metric) => {
|
|
32
|
+
const [trend, stat = 'p(95)'] = metric.split(':');
|
|
33
|
+
const dir = mkdtempSync(join(tmpdir(), 'canary-scaling-'));
|
|
34
|
+
try {
|
|
35
|
+
const summary = join(dir, 'summary.json');
|
|
36
|
+
const res = spawnSync('k6', [
|
|
37
|
+
'run',
|
|
38
|
+
'--quiet',
|
|
39
|
+
'--summary-export',
|
|
40
|
+
summary,
|
|
41
|
+
'-e',
|
|
42
|
+
`SIZE=${size}`,
|
|
43
|
+
script,
|
|
44
|
+
],
|
|
45
|
+
// k6's own end summary would bury the verdict; its errors still show.
|
|
46
|
+
{ stdio: ['ignore', 'ignore', 'inherit'] });
|
|
47
|
+
if (res.error)
|
|
48
|
+
throw res.error;
|
|
49
|
+
const j = JSON.parse(readFileSync(summary, 'utf-8'));
|
|
50
|
+
const v = j.metrics?.[trend]?.[stat];
|
|
51
|
+
return typeof v === 'number' ? v : null;
|
|
52
|
+
}
|
|
53
|
+
catch (e) {
|
|
54
|
+
if (e.code === 'ENOENT')
|
|
55
|
+
return null;
|
|
56
|
+
throw e;
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
rmSync(dir, { recursive: true, force: true });
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
function sizeList(raw) {
|
|
63
|
+
const sizes = raw.split(',').map((s) => positive(s.trim()));
|
|
64
|
+
return sizes;
|
|
65
|
+
}
|
|
66
|
+
export function buildScalingCurveCommand(deps, runner = defaultK6Runner) {
|
|
67
|
+
return new Command('scaling-curve')
|
|
68
|
+
.description('Fit how a cost metric grows with input size; flag superlinear growth. ' +
|
|
69
|
+
'Advisory; abstains (exit 3) when the data cannot support a verdict.')
|
|
70
|
+
.argument('[points-file]', 'JSON {"metric","points":[{size,value}]} or CSV with a size,value header')
|
|
71
|
+
.option('--target <size>', 'extrapolate the fitted cost to this size', positive)
|
|
72
|
+
.option('--run <k6-script>', 'run the ladder: the script once per size with SIZE set')
|
|
73
|
+
.option('--sizes <list>', 'comma-separated sizes for --run', sizeList)
|
|
74
|
+
.option('--repeats <n>', 'runs per size for --run (noise rule)', positive, 3)
|
|
75
|
+
.option('--metric <trend:stat>', 'k6 metric for --run', 'http_req_duration:p(95)')
|
|
76
|
+
.option('--json', 'Output the verdict and its points as JSON.')
|
|
77
|
+
.action((file, opts) => {
|
|
78
|
+
const { metric, points, missing } = collect(file, opts, runner);
|
|
79
|
+
let r = fitScaling(points, opts.target === undefined ? {} : { target: opts.target });
|
|
80
|
+
// A run that never emitted the metric is an abstention even if the other
|
|
81
|
+
// runs would fit: the missing runs are a hole in the curve, not a zero.
|
|
82
|
+
if (missing > 0 && r.verdict === 'INSUFFICIENT_DATA') {
|
|
83
|
+
r.reasons.unshift(`metric ${metric} was not emitted by ${missing} run(s)`);
|
|
84
|
+
}
|
|
85
|
+
if (missing > 0 && r.verdict !== 'INSUFFICIENT_DATA') {
|
|
86
|
+
r = {
|
|
87
|
+
verdict: 'INSUFFICIENT_DATA',
|
|
88
|
+
reasons: [`metric ${metric} was not emitted by ${missing} run(s)`],
|
|
89
|
+
points: r.points,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
const abstained = r.verdict === 'INSUFFICIENT_DATA';
|
|
93
|
+
if (opts.json) {
|
|
94
|
+
deps.out(jsonIndent2({ metric: metric ?? null, ...r }));
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
const label = metric ?? 'value';
|
|
98
|
+
deps.out(pc.bold(`Scaling curve: ${label} vs size`));
|
|
99
|
+
for (const p of r.points) {
|
|
100
|
+
deps.out(` ${String(p.size).padStart(10)} ${Number(p.value.toPrecision(4))} ${pc.dim(`(${p.samples} sample(s))`)}`);
|
|
101
|
+
}
|
|
102
|
+
if (r.verdict === 'INSUFFICIENT_DATA') {
|
|
103
|
+
deps.out(pc.bold(pc.yellow(`${WARN} INSUFFICIENT_DATA -- this is not a pass.`)));
|
|
104
|
+
for (const reason of r.reasons)
|
|
105
|
+
deps.out(` - ${reason}`);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
const color = r.verdict === 'LINEAR_OR_BETTER' ? pc.green : pc.yellow;
|
|
109
|
+
deps.out(`${color(pc.bold(r.verdict))}: exponent ${r.exponent.toFixed(2)} ` +
|
|
110
|
+
`(R² ${r.r2.toFixed(3)}) -- ${MEANING[r.verdict]}`);
|
|
111
|
+
deps.out(r.knee === null
|
|
112
|
+
? ' no knee: the curve does not bend within the measured range'
|
|
113
|
+
: ` knee at size ${r.knee}: growth steepens from here`);
|
|
114
|
+
if (r.extrapolation) {
|
|
115
|
+
const e = r.extrapolation;
|
|
116
|
+
deps.out(` extrapolated ${label} at ${e.size}: ${e.value.toFixed(1)} ` +
|
|
117
|
+
pc.dim(`(${e.multipleOfLargest}x the largest measured size -- an extrapolation, not a measurement)`));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
if (abstained)
|
|
122
|
+
throw new CliExitError(EXIT_ABSTAINED);
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
function collect(file, opts, runner) {
|
|
126
|
+
if (opts.run === undefined) {
|
|
127
|
+
if (file === undefined) {
|
|
128
|
+
throw new InvalidArgumentError('give a points file, or --run with --sizes');
|
|
129
|
+
}
|
|
130
|
+
return { ...parsePoints(readFileSync(file, 'utf-8')), missing: 0 };
|
|
131
|
+
}
|
|
132
|
+
if (!opts.sizes)
|
|
133
|
+
throw new InvalidArgumentError('--run needs --sizes');
|
|
134
|
+
const points = [];
|
|
135
|
+
let missing = 0;
|
|
136
|
+
for (const size of opts.sizes) {
|
|
137
|
+
for (let i = 0; i < opts.repeats; i += 1) {
|
|
138
|
+
const value = runner(opts.run, size, opts.metric);
|
|
139
|
+
if (value === null)
|
|
140
|
+
missing += 1;
|
|
141
|
+
else
|
|
142
|
+
points.push({ size, value });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return { metric: opts.metric, points, missing };
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=scaling-curve-cli.js.map
|