canary-test-cli 7.2.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/agents/skills/README.md +23 -4
  2. package/agents/skills/claude-code/canary-batwoman/SKILL.md +119 -0
  3. package/agents/skills/claude-code/canary-cassandra/SKILL.md +23 -16
  4. package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +3 -1
  5. package/agents/skills/claude-code/canary-ci-ready/SKILL.md +20 -3
  6. package/agents/skills/claude-code/canary-fleet-health/SKILL.md +1 -0
  7. package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +15 -0
  8. package/agents/skills/claude-code/canary-screech/SKILL.md +109 -0
  9. package/agents/skills/claude-code/canary-screech/scripts/blast.mjs +125 -0
  10. package/agents/skills/claude-code/canary-screech/scripts/cli.mjs +128 -0
  11. package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs +97 -0
  12. package/agents/skills/claude-code/canary-screech/scripts/history.mjs +73 -0
  13. package/agents/skills/claude-code/canary-screech/scripts/redness.mjs +94 -0
  14. package/agents/skills/lib/parse-args.mjs +200 -139
  15. package/dist/engine/analysis/batwoman/audit.js +39 -0
  16. package/dist/engine/analysis/batwoman/closure.js +159 -0
  17. package/dist/engine/analysis/batwoman/gh-history.js +119 -0
  18. package/dist/engine/analysis/batwoman/probes.js +195 -0
  19. package/dist/engine/analysis/batwoman/registry.js +142 -0
  20. package/dist/engine/analysis/batwoman/render.js +194 -0
  21. package/dist/engine/analysis/batwoman/run-window.js +122 -0
  22. package/dist/engine/analysis/batwoman/text.js +84 -0
  23. package/dist/engine/analysis/batwoman/triggers.js +122 -0
  24. package/dist/engine/analysis/batwoman/verdict.js +64 -0
  25. package/dist/engine/analysis/cli.js +47 -14
  26. package/dist/engine/analysis/gh-flaky/gh-run-attempts.js +206 -0
  27. package/dist/engine/batwoman-cli.js +119 -0
  28. package/dist/engine/ci-ready-cli.js +71 -0
  29. package/dist/engine/cli-commands.js +46 -7
  30. package/dist/engine/cli.core.js +16 -0
  31. package/dist/engine/company-knowledge-cli.js +10 -2
  32. package/dist/engine/core/ci-ready.js +112 -0
  33. package/dist/engine/core/company-knowledge.js +8 -0
  34. package/dist/engine/core/migrator.js +147 -20
  35. package/dist/engine/core/permission-matrix.js +219 -0
  36. package/dist/engine/core/quality-scorer.js +13 -18
  37. package/dist/engine/core/scaling-curve.js +143 -0
  38. package/dist/engine/core/string-literals.js +3 -1
  39. package/dist/engine/core/vacuity-scanner.js +151 -6
  40. package/dist/engine/core/workflow-discovery.js +41 -23
  41. package/dist/engine/guardian/adjudication-github.js +136 -0
  42. package/dist/engine/guardian/adjudication.js +119 -340
  43. package/dist/engine/guardian/cli.js +180 -264
  44. package/dist/engine/guardian/coverage.js +2 -1
  45. package/dist/engine/guardian/diff-coverage/coverage-delta.js +162 -0
  46. package/dist/engine/guardian/diff-coverage/formats/cobertura.js +45 -1
  47. package/dist/engine/guardian/diff-coverage/orchestrator.js +25 -21
  48. package/dist/engine/guardian/diff-coverage/paths.js +5 -9
  49. package/dist/engine/guardian/diff-coverage/report-tier.js +88 -12
  50. package/dist/engine/guardian/diff-extractor.js +31 -32
  51. package/dist/engine/guardian/pr-check.js +262 -430
  52. package/dist/engine/guardian/pr-comment.js +35 -58
  53. package/dist/engine/guardian/weak-test.js +236 -0
  54. package/dist/engine/mcp-server.js +67 -4
  55. package/dist/engine/permission-matrix-cli.js +51 -0
  56. package/dist/engine/scaling-curve-cli.js +147 -0
  57. package/dist/engine/skills-cli.js +48 -32
  58. package/dist/engine/workflow-cli.js +85 -65
  59. package/package.json +1 -1
@@ -1,42 +1,13 @@
1
1
  /**
2
- * Deterministic GitHub PR comment poster (Tier 0, agent-free).
3
- *
4
- * Faithful TypeScript port of `agent/guardian/pr_comment.py`.
5
- *
6
- * This module posts/updates the single sticky guardian findings comment on a
7
- * pull request. It is **deterministic HTTP behind an interface seam** — it
8
- * imports no agent/LLM module (SC-11).
9
- *
10
- * Design:
11
- *
12
- * - {@link GitHubClient} is the seam every consumer talks to
13
- * (`listComments` / `createComment` / `updateComment`).
14
- * - {@link FakeGitHubClient} is the in-memory implementation used by every unit
15
- * test — **no network**. It can simulate a fork read-only token via
16
- * `deny_writes=true` (writes reject with {@link GitHubPermissionError}).
17
- * - {@link RestGitHubClient} (Python's private `_RestGitHubClient`) is the thin
18
- * real client. Network lives **only** here; `guardian-rest-clients.test.ts`
19
- * drives it through a stubbed global `fetch`, so the URL, headers, error
20
- * mapping, and #528 pagination are covered without a socket.
21
- *
22
- * Python→TS nuances:
23
- * - **async**: Python's `urllib` client is synchronous; Node's global `fetch`
24
- * is async. The seam methods are therefore `Promise`-returning, so the
25
- * real client can `await fetch`. The fakes satisfy the async interface by
26
- * being `async` (returning already-resolved values), and
27
- * {@link upsertStickyComment} becomes `async`. The pure helpers
28
- * ({@link findSticky}, {@link degradationAnnotation}) stay synchronous.
29
- * - **error mapping**: `fetch` resolves (does not throw) on a 4xx/5xx status,
30
- * so the 403→{@link GitHubPermissionError} mapping is done off `resp.status`
31
- * rather than off a raised `HTTPError`. As in the oracle, ONLY 403 maps to a
32
- * permission error here; any other non-2xx propagates as a generic error.
2
+ * Deterministic poster of the single sticky guardian PR comment (Tier 0; no
3
+ * agent/LLM import, SC-11). {@link GitHubClient} is the seam;
4
+ * {@link FakeGitHubClient} is the in-memory test double (`deny_writes` models a
5
+ * 403); {@link RestGitHubClient} is the only code that touches the network.
33
6
  */
34
7
  import { readAllPages, restPageReader } from './github-paging.js';
35
- // Single source of truth for the sticky-comment marker.
36
- // `pr_check.renderFindings` emits the identical literal at the head of a
37
- // `comment`-format body so `findSticky` can locate the guardian comment for
38
- // in-place upsert.
8
+ // `renderFindings` writes this literal on a comment body's first line.
39
9
  export const STICKY_MARKER = '<!-- canary-pr-guardian -->';
10
+ const DEFAULT_IDENTITY = { login: 'github-actions[bot]' };
40
11
  /**
41
12
  * A client cannot write (fork read-only token → HTTP 403).
42
13
  *
@@ -49,12 +20,7 @@ export class GitHubPermissionError extends Error {
49
20
  this.name = 'GitHubPermissionError';
50
21
  }
51
22
  }
52
- /**
53
- * Map a non-2xx status to an error. As in the Python reference, ONLY 403 is a
54
- * permission error; every other non-2xx propagates (the analog of urllib's
55
- * HTTPError re-raise). Shared by the write path and the paged read path so the
56
- * two cannot drift.
57
- */
23
+ /** Map a non-2xx status to an error; ONLY 403 is a permission error. */
58
24
  function toGitHubError(status, url) {
59
25
  return status === 403
60
26
  ? new GitHubPermissionError(`GitHub API 403 (read-only token / fork PR?): ${url}`)
@@ -84,7 +50,11 @@ export class FakeGitHubClient {
84
50
  throw new GitHubPermissionError('read-only token: cannot create comment');
85
51
  }
86
52
  this.nextId += 1;
87
- const row = { id: this.nextId, body };
53
+ const row = {
54
+ id: this.nextId,
55
+ body,
56
+ user: { login: DEFAULT_IDENTITY.login, type: 'Bot' },
57
+ };
88
58
  this.comments.push(row);
89
59
  return row;
90
60
  }
@@ -101,21 +71,31 @@ export class FakeGitHubClient {
101
71
  throw new Error(`no comment with id ${commentId}`);
102
72
  }
103
73
  }
104
- /** Return the first comment whose body contains `marker`, else `null`. */
105
- export function findSticky(comments, marker = STICKY_MARKER) {
106
- for (const comment of comments) {
107
- if ((comment.body ?? '').includes(marker)) {
108
- return comment;
109
- }
74
+ /**
75
+ * Return guardian's sticky comment, else `null` (#931). A comment qualifies
76
+ * only when its body STARTS WITH `marker` and `identity` wrote it, so a human
77
+ * who pasted guardian's output is never overwritten. Newest qualifier wins.
78
+ */
79
+ export function findSticky(comments, marker = STICKY_MARKER, identity = DEFAULT_IDENTITY) {
80
+ let best = null;
81
+ for (const c of comments.filter((x) => isSticky(x, marker, identity))) {
82
+ if (!best || (c.created_at ?? '') >= (best.created_at ?? ''))
83
+ best = c;
110
84
  }
111
- return null;
85
+ return best;
86
+ }
87
+ function isSticky(c, marker, id) {
88
+ return isAuthoredBy(c, id) && (c.body ?? '').trimStart().startsWith(marker);
89
+ }
90
+ function isAuthoredBy(c, identity) {
91
+ if (c.user?.login === identity.login)
92
+ return true;
93
+ const slug = c.performed_via_github_app?.slug;
94
+ return c.user?.type === 'Bot' && !!slug && slug === identity.appSlug;
112
95
  }
113
96
  /**
114
- * Post or update the single sticky guardian comment (SC-9).
115
- *
116
- * Locates the existing comment by `marker`; updates it in place when present,
117
- * otherwise creates a new one. Never stacks duplicates. A read-only token (fork
118
- * PR?) degrades loudly to a `degraded` result rather than crashing (OT-4).
97
+ * Update guardian's own sticky in place, else create one (SC-9: never stacks).
98
+ * A 403 degrades to a `degraded` result instead of crashing the job (OT-4).
119
99
  */
120
100
  export async function upsertStickyComment(client, body, marker = STICKY_MARKER) {
121
101
  const existing = findSticky(await client.listComments(), marker);
@@ -129,13 +109,10 @@ export async function upsertStickyComment(client, body, marker = STICKY_MARKER)
129
109
  }
130
110
  catch (err) {
131
111
  if (err instanceof GitHubPermissionError) {
132
- // OT-4 / SC-1+D6: a read-only token (fork PR?) must degrade loudly, not
133
- // crash the job. The caller emits `notice` as a `::warning::` annotation.
134
112
  return {
135
113
  action: 'degraded',
136
114
  comment_id: null,
137
- notice: 'guardian: read-only token (fork PR?) — findings not posted as ' +
138
- 'a comment',
115
+ notice: `guardian: token lacks write permission on PR comments (HTTP 403) — findings not posted as a comment`,
139
116
  };
140
117
  }
141
118
  throw err;
@@ -0,0 +1,236 @@
1
+ /**
2
+ * The advisory `weak-test` finding: an ADDED test that asserts nothing (#747).
3
+ *
4
+ * Split out of `pr-check.ts` as a self-contained judgement — test declarations,
5
+ * block spans, assertion presence. The dependency runs one way (this module
6
+ * reads `pr-check`, never the reverse); `guardian/cli.ts` calls into here.
7
+ */
8
+ import { readFileSync } from 'node:fs';
9
+ import { Fidelity } from './coverage.js';
10
+ import { Severity } from './impact-mapper.js';
11
+ import { GuardianFinding, addedContentByPath, linesInRanges, walkDiff, } from './pr-check.js';
12
+ import { hasAssertion, isAssertionFreeTest } from '../core/quality-scorer.js';
13
+ // Test-file extension -> the quality scorer's framework. Unknown -> pytest.
14
+ const TEST_FRAMEWORK_BY_EXT = {
15
+ '.py': 'pytest',
16
+ '.ts': 'vitest',
17
+ '.tsx': 'vitest',
18
+ '.js': 'vitest',
19
+ '.jsx': 'vitest',
20
+ '.mjs': 'vitest',
21
+ '.cjs': 'vitest',
22
+ };
23
+ function frameworkForTestPath(path) {
24
+ const ext = /\.[^./\\]+$/.exec(path)?.[0] ?? '';
25
+ return TEST_FRAMEWORK_BY_EXT[ext.toLowerCase()] ?? 'pytest';
26
+ }
27
+ // A signature / decorator / block-close / comment — not a test *body*. A rename
28
+ // that adds only `def test_new():` over an unchanged body has nothing to judge.
29
+ const TEST_SIGNATURE_RE = /^\s*(?:async\s+)?def\s+test\w*\s*\(|^\s*(?:it|test|describe)\s*\(/;
30
+ const BLOCK_DELIMITERS = new Set(['})', '});', '}', ')', '{']);
31
+ const NON_BODY_PREFIXES = ['#', '//', '@', '*', '/*'];
32
+ /** True iff `added` holds a real body line, not just signature/comment/brace. */
33
+ function hasAddedTestBody(added) {
34
+ return added.some((line) => {
35
+ const stripped = line.trim();
36
+ if (!stripped || BLOCK_DELIMITERS.has(stripped))
37
+ return false;
38
+ if (NON_BODY_PREFIXES.some((p) => stripped.startsWith(p)))
39
+ return false;
40
+ return !TEST_SIGNATURE_RE.test(line);
41
+ });
42
+ }
43
+ // The declaration of a single test. Narrower than TEST_SIGNATURE_RE on purpose:
44
+ // `describe(` opens a group, and judging a whole group would let an asserting
45
+ // sibling excuse an empty test. `it.only` / `test.each` still open one test.
46
+ const TEST_DECL_PY = /^\s*(?:async\s+)?def\s+test\w*\s*\(/;
47
+ const TEST_DECL_JS = /^\s*(?:async\s+)?(?:it|test)(?:\.\w+)*\s*\(/;
48
+ // A same-file helper definition (#929): `function f(`, `const f = (` / `=
49
+ // function` / `= x =>`, or Python `def f(`.
50
+ const HELPER_DEF_PY = /^\s*(?:async\s+)?def\s+(\w+)\s*\(/;
51
+ const HELPER_DEF_JS = /^\s*(?:export\s+)?(?:async\s+)?(?:function\s*\*?\s*(\w+)\s*\(|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:function\b|\(|\w+\s*=>))/;
52
+ const TEST_TITLE_RE = /\(\s*(['"`])((?:\\.|(?!\1).)*)\1|def\s+(\w+)/;
53
+ function indentWidth(line) {
54
+ return line.length - line.trimStart().length;
55
+ }
56
+ // Strings and comments are blanked before delimiter counting.
57
+ const JS_STRING_OR_COMMENT = /(['"`])(?:\\.|(?!\1).)*?\1|\/\/.*$|\/\*[\s\S]*?\*\//g;
58
+ const NEW_FILE_RE = /^--- \/dev\/null\r?\n\+\+\+ b\/(.+?)\r?$/gm;
59
+ function visibleLinesByPath(diffText) {
60
+ const files = new Map();
61
+ walkDiff(diffText, (line) => {
62
+ let file = files.get(line.path);
63
+ if (!file) {
64
+ file = { text: new Map(), added: new Set(), eof: null };
65
+ files.set(line.path, file);
66
+ }
67
+ file.text.set(line.lineno, line.text);
68
+ if (line.added)
69
+ file.added.add(line.lineno);
70
+ });
71
+ for (const m of diffText.matchAll(NEW_FILE_RE)) {
72
+ const file = files.get(m[1]);
73
+ if (file)
74
+ file.eof = Math.max(...file.text.keys());
75
+ }
76
+ return files;
77
+ }
78
+ /** The file's own text under `root`, or `null` when it cannot be read. */
79
+ function fileAtRoot(root, path) {
80
+ try {
81
+ const lines = readFileSync(`${root}/${path}`, 'utf-8').split(/\r?\n/);
82
+ const text = new Map(lines.map((t, i) => [i + 1, t]));
83
+ return { text, added: new Set(), eof: lines.length };
84
+ }
85
+ catch {
86
+ return null;
87
+ }
88
+ }
89
+ /** The enclosing test's declaration line in the contiguous visible run, or null. */
90
+ function enclosingTestDecl(file, lineNo, declRe) {
91
+ for (let n = lineNo; file.text.has(n); n--) {
92
+ if (declRe.test(file.text.get(n)))
93
+ return n;
94
+ }
95
+ return null;
96
+ }
97
+ /** Python: a block closes before the first non-blank line dedented to `def`. */
98
+ function pythonBlockEnd(file, start) {
99
+ const declIndent = indentWidth(file.text.get(start));
100
+ let n = start + 1;
101
+ for (; file.text.has(n); n++) {
102
+ const text = file.text.get(n);
103
+ if (text.trim() && indentWidth(text) <= declIndent)
104
+ return n - 1;
105
+ }
106
+ return n - 1 === file.eof ? n - 1 : null;
107
+ }
108
+ /**
109
+ * The last line of the block opened at `start`, or `null` when its end is not
110
+ * inside the visible run (#929). JS/TS closes when delimiter depth returns to
111
+ * zero. An unknown end means the assertion may be exactly what is out of view
112
+ * — the dominant case, since a test's setup is edited far more than its
113
+ * `expect` — so the caller abstains rather than scoring half a block.
114
+ */
115
+ function blockEnd(file, start, isPython) {
116
+ if (isPython)
117
+ return pythonBlockEnd(file, start);
118
+ let depth = 0;
119
+ let opened = false;
120
+ let n = start;
121
+ for (; file.text.has(n); n++) {
122
+ const code = file.text.get(n).replace(JS_STRING_OR_COMMENT, '');
123
+ const opens = code.replace(/[^{(]/g, '').length;
124
+ depth += opens - code.replace(/[^})]/g, '').length;
125
+ opened ||= opens > 0;
126
+ if (opened && depth <= 0)
127
+ return n;
128
+ }
129
+ return n - 1 === file.eof ? n - 1 : null;
130
+ }
131
+ function linesOf(file, start, end) {
132
+ const out = [];
133
+ for (let n = start; n <= end; n++) {
134
+ const text = file.text.get(n);
135
+ if (text !== undefined)
136
+ out.push(text);
137
+ }
138
+ return out;
139
+ }
140
+ /** Same-file helpers whose CLOSED body asserts, from diff and disk text (#929). */
141
+ function assertingHelpers(sources, framework) {
142
+ const names = new Set();
143
+ for (const src of sources) {
144
+ for (const n of src.text.keys()) {
145
+ const name = assertingHelperAt(src, n, framework);
146
+ if (name)
147
+ names.add(name);
148
+ }
149
+ }
150
+ return [...names];
151
+ }
152
+ /** The helper defined at line `n` when its closed body asserts, else null. */
153
+ function assertingHelperAt(src, n, framework) {
154
+ const isPython = framework === 'pytest';
155
+ const m = (isPython ? HELPER_DEF_PY : HELPER_DEF_JS).exec(src.text.get(n));
156
+ const name = m?.[1] ?? m?.[2];
157
+ const end = name ? blockEnd(src, n, isPython) : null;
158
+ if (end === null)
159
+ return null;
160
+ return hasAssertion(linesOf(src, n, end).join('\n'), framework)
161
+ ? name
162
+ : null;
163
+ }
164
+ /** True iff the block asserts nothing and has an added body to judge. */
165
+ function isWeakBlock(file, [start, end], framework, helpers) {
166
+ const span = linesOf(file, start, end);
167
+ const added = linesOf(file, start, end).filter((_, i) => file.added.has(start + i));
168
+ // FP-3: only a signature was added over an unchanged body. A wholly added
169
+ // block (e.g. `it('x', () => {})` on one line) is still judged.
170
+ if (!hasAddedTestBody(added) && added.length < span.length)
171
+ return false;
172
+ const code = span.join('\n');
173
+ if (!isAssertionFreeTest(code, framework))
174
+ return false;
175
+ return !helpers.some((h) => new RegExp(`\\b${h}\\s*\\(`).test(code));
176
+ }
177
+ /** The weak test blocks touched by `unit`'s added lines, each visited once. */
178
+ function weakBlocksIn(file, unit, framework, helpers) {
179
+ const declRe = framework === 'pytest' ? TEST_DECL_PY : TEST_DECL_JS;
180
+ const seen = new Set();
181
+ const weak = [];
182
+ for (const lineNo of linesInRanges(unit.added_ranges)) {
183
+ const start = enclosingTestDecl(file, lineNo, declRe);
184
+ if (start === null || seen.has(start))
185
+ continue;
186
+ seen.add(start);
187
+ const end = blockEnd(file, start, framework === 'pytest');
188
+ if (end === null)
189
+ continue;
190
+ if (isWeakBlock(file, [start, end], framework, helpers)) {
191
+ weak.push([start, end]);
192
+ }
193
+ }
194
+ return weak;
195
+ }
196
+ function testTitle(decl) {
197
+ const m = TEST_TITLE_RE.exec(decl);
198
+ return m?.[2] ?? m?.[3] ?? decl.trim();
199
+ }
200
+ /**
201
+ * Advisory `weak-test` findings for ADDED tests that assert nothing — one per
202
+ * weak test block, naming its title and line span (#929).
203
+ *
204
+ * The span scored is the ENCLOSING TEST BLOCK (#747); a block that cannot be
205
+ * resolved, or whose end is out of view, is abstained on. `repoRoot` is where
206
+ * the file's own text is read for helper resolution (`.`, the CLI's repoRoot
207
+ * convention). Findings are `LOW` and never gated (see computeExitCode).
208
+ */
209
+ export function buildWeakTestFindings(testUnits, diffText, repoRoot = '.') {
210
+ const addedByPath = addedContentByPath(diffText);
211
+ const visibleByPath = visibleLinesByPath(diffText);
212
+ const findings = [];
213
+ for (const unit of testUnits) {
214
+ const file = visibleByPath.get(unit.path);
215
+ if (!file || !addedByPath.get(unit.path)?.length)
216
+ continue;
217
+ const framework = frameworkForTestPath(unit.path);
218
+ const disk = fileAtRoot(repoRoot, unit.path);
219
+ const helpers = assertingHelpers(disk ? [file, disk] : [file], framework);
220
+ for (const [start, end] of weakBlocksIn(file, unit, framework, helpers)) {
221
+ const title = testTitle(file.text.get(start));
222
+ findings.push(new GuardianFinding({
223
+ path: unit.path,
224
+ unit: unit.path,
225
+ kind: 'weak-test',
226
+ fidelity: Fidelity.Heuristic,
227
+ severity: Severity.LOW,
228
+ evidence: `added test "${title}" (L${start}–L${end}) asserts nothing (advisory — never blocks the gate)`,
229
+ suggestion: 'add at least one assertion, or delete the test if it is a placeholder.',
230
+ added_ranges: [[start, end]],
231
+ }));
232
+ }
233
+ }
234
+ return findings;
235
+ }
236
+ //# sourceMappingURL=weak-test.js.map
@@ -40,8 +40,8 @@
40
40
  * a single trailing-newline empty tail exactly as `str.splitlines()` does.
41
41
  * - File writes are LF + UTF-8 on every platform (matches the sibling ports).
42
42
  */
43
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
44
- import { basename, dirname, extname, join, relative, resolve } from 'node:path';
43
+ import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync, } from 'node:fs';
44
+ import { basename, dirname, extname, isAbsolute, join, relative, resolve, } from 'node:path';
45
45
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
46
46
  import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
47
47
  import * as z from 'zod';
@@ -476,8 +476,60 @@ export function analyzeFileImpl(filePath) {
476
476
  persona,
477
477
  };
478
478
  }
479
+ /**
480
+ * Resolve `candidate` against `root` and return it only if it stays inside.
481
+ *
482
+ * A tool argument arrives from the MCP host, so a path is untrusted input: an
483
+ * absolute path, a `..` climb, or a symlinked parent directory would otherwise
484
+ * let a caller reach outside the served project. Containment is therefore
485
+ * checked *after* symlink resolution -- a textual `..`-and-absolute test looks
486
+ * like a guard but cannot see a symlink, which is the gap this exists to close.
487
+ *
488
+ * `realpathSync` throws on a missing path, so the nearest existing ancestor is
489
+ * resolved and the not-yet-created tail re-joined onto it. That is what lets a
490
+ * write to a new file still be checked against the real location of the
491
+ * directory it would be created in. Mirrors `resolveCliPath`
492
+ * (`core/skill-registry.ts`), the guard already proven in this codebase.
493
+ *
494
+ * @returns the resolved absolute path, or `null` when it escapes `root`.
495
+ */
496
+ export function resolveInsideRoot(candidate, root) {
497
+ let realRoot;
498
+ try {
499
+ realRoot = realpathSync(resolve(root));
500
+ }
501
+ catch {
502
+ return null;
503
+ }
504
+ const target = resolve(realRoot, candidate);
505
+ // Walk up to the nearest ancestor that exists, remembering the tail.
506
+ const tail = [];
507
+ let probe = target;
508
+ while (!existsSync(probe)) {
509
+ const parent = dirname(probe);
510
+ if (parent === probe)
511
+ return null;
512
+ tail.unshift(basename(probe));
513
+ probe = parent;
514
+ }
515
+ let resolved;
516
+ try {
517
+ resolved = join(realpathSync(probe), ...tail);
518
+ }
519
+ catch {
520
+ return null;
521
+ }
522
+ const rel = relative(realRoot, resolved);
523
+ if (rel !== '' && (rel.startsWith('..') || isAbsolute(rel)))
524
+ return null;
525
+ return resolved;
526
+ }
527
+ /** The refusal an MCP tool returns when a path argument escapes its root. */
528
+ function escapeError(path) {
529
+ return { error: `path escapes the project root: ${path}` };
530
+ }
479
531
  /** Python: `_write_test_file_impl`. */
480
- export function writeTestFileImpl(filePath, content, framework) {
532
+ export function writeTestFileImpl(filePath, content, framework, root = WORKING_DIR) {
481
533
  let outPath = filePath;
482
534
  if (pySuffix(basename(filePath)) === '') {
483
535
  const extMap = {
@@ -490,12 +542,23 @@ export function writeTestFileImpl(filePath, content, framework) {
490
542
  // with no existing suffix to strip, that is a plain concatenation.
491
543
  outPath = filePath + (extMap[framework] ?? '.ts');
492
544
  }
545
+ // Checked on the FINAL path, after the extension is inferred, and before the
546
+ // first filesystem mutation -- `mkdirSync` below would otherwise create
547
+ // directories outside the root even when the write itself were refused.
548
+ if (resolveInsideRoot(outPath, root) === null)
549
+ return escapeError(outPath);
493
550
  mkdirSync(dirname(outPath), { recursive: true });
494
551
  writeFileSync(outPath, content, 'utf-8');
495
552
  return { written_path: outPath };
496
553
  }
497
554
  /** Python: `_run_tests_impl`. */
498
- export function runTestsImpl(testFile) {
555
+ export function runTestsImpl(testFile, root = WORKING_DIR) {
556
+ // The same containment the write path gets: this spawns a test runner over
557
+ // the named file, so an unconfined path is arbitrary code execution, not
558
+ // merely an arbitrary read. Refused before the executor is constructed.
559
+ if (resolveInsideRoot(testFile, root) === null) {
560
+ return { passed: 0, failed: 1, exit_code: 1, ...escapeError(testFile) };
561
+ }
499
562
  const suffix = extname(testFile).toLowerCase();
500
563
  const framework = suffix === '.py' ? 'pytest' : 'playwright';
501
564
  const executor = new CanaryTestExecutor();
@@ -0,0 +1,51 @@
1
+ /**
2
+ * `canary permission-matrix <model.yaml>` (#857).
3
+ *
4
+ * Writes the generated Playwright suite even when cells are undeclared (as
5
+ * `test.fixme`, so they are counted), then exits 3: a matrix with holes cannot
6
+ * verify the boundary it describes, and must not read as a clean generation.
7
+ */
8
+ import { readFileSync, writeFileSync } from 'node:fs';
9
+ import { Command } from 'commander';
10
+ import pc from 'picocolors';
11
+ import { CliExitError, jsonIndent2 } from './cli-common.js';
12
+ import { EXIT_ABSTAINED } from './core/gate-result.js';
13
+ import { expandMatrix, parseMatrix, probedCells, renderPlaywright, } from './core/permission-matrix.js';
14
+ import { WARN } from './main-deps.js';
15
+ function writeSuite(parsed, cells, undeclared, opts, deps) {
16
+ writeFileSync(opts.out, renderPlaywright(cells, parsed.existenceProbes), 'utf-8');
17
+ const cross = cells.filter((c) => c.actingTenant !== c.targetTenant).length;
18
+ const probes = probedCells(cells, parsed.existenceProbes).length;
19
+ deps.out(`Wrote ${cells.length} cell test(s)` +
20
+ (probes > 0 ? ` + ${probes} existence probe(s)` : '') +
21
+ ` to ${opts.out} ` +
22
+ pc.dim(`(${cross} cross-tenant)`));
23
+ if (undeclared.length === 0)
24
+ return;
25
+ deps.out(pc.bold(pc.yellow(`${WARN} ${undeclared.length} UNDECLARED cell(s) -- emitted as ` +
26
+ 'test.fixme; this matrix cannot verify them:')));
27
+ for (const u of undeclared)
28
+ deps.out(` - ${u}`);
29
+ }
30
+ export function buildPermissionMatrixCommand(deps) {
31
+ return new Command('permission-matrix')
32
+ .description('Generate server-direct authz tests from a declared role x endpoint ' +
33
+ 'grid, across every acting x target tenant. Undeclared cells exit 3.')
34
+ .argument('<model>', 'YAML model: roles, tenants, endpoints grid')
35
+ .option('--out <file>', 'where to write the Playwright suite', 'permission-matrix.spec.ts')
36
+ .option('--json', 'Print the expanded cells as JSON instead of writing.')
37
+ .action((model, opts) => {
38
+ const parsed = parseMatrix(readFileSync(model, 'utf-8'));
39
+ const { cells, undeclared } = expandMatrix(parsed);
40
+ if (opts.json) {
41
+ const { existenceProbes } = parsed;
42
+ deps.out(jsonIndent2({ cells, undeclared, existenceProbes }));
43
+ }
44
+ else {
45
+ writeSuite(parsed, cells, undeclared, opts, deps);
46
+ }
47
+ if (undeclared.length > 0)
48
+ throw new CliExitError(EXIT_ABSTAINED);
49
+ });
50
+ }
51
+ //# sourceMappingURL=permission-matrix-cli.js.map
@@ -0,0 +1,147 @@
1
+ /**
2
+ * `canary scaling-curve <points-file>` (#856).
3
+ *
4
+ * Advisory: a verdict exits 0 whatever the exponent, because this is a planning
5
+ * signal, not a gate. An abstention exits 3 (ADR 0009) -- "not enough data to
6
+ * say" must never look like "linear".
7
+ */
8
+ import { spawnSync } from 'node:child_process';
9
+ import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
10
+ import { tmpdir } from 'node:os';
11
+ import { join } from 'node:path';
12
+ import { Command, InvalidArgumentError } from 'commander';
13
+ import pc from 'picocolors';
14
+ import { CliExitError, jsonIndent2 } from './cli-common.js';
15
+ import { EXIT_ABSTAINED } from './core/gate-result.js';
16
+ import { fitScaling, parsePoints } from './core/scaling-curve.js';
17
+ import { WARN } from './main-deps.js';
18
+ const MEANING = {
19
+ LINEAR_OR_BETTER: 'cost grows no faster than input',
20
+ SUPERLINEAR: 'cost grows faster than input (n log n territory)',
21
+ STRONGLY_SUPERLINEAR: 'cost grows much faster than input (quadratic territory) -- will buckle at peak',
22
+ };
23
+ function positive(raw) {
24
+ const n = Number(raw);
25
+ if (!(Number.isFinite(n) && n > 0)) {
26
+ throw new InvalidArgumentError(`must be a positive number, got "${raw}"`);
27
+ }
28
+ return n;
29
+ }
30
+ /** `trend:stat`, e.g. `http_req_duration:p(95)`, read from k6's summary export. */
31
+ const defaultK6Runner = (script, size, metric) => {
32
+ const [trend, stat = 'p(95)'] = metric.split(':');
33
+ const dir = mkdtempSync(join(tmpdir(), 'canary-scaling-'));
34
+ try {
35
+ const summary = join(dir, 'summary.json');
36
+ const res = spawnSync('k6', [
37
+ 'run',
38
+ '--quiet',
39
+ '--summary-export',
40
+ summary,
41
+ '-e',
42
+ `SIZE=${size}`,
43
+ script,
44
+ ],
45
+ // k6's own end summary would bury the verdict; its errors still show.
46
+ { stdio: ['ignore', 'ignore', 'inherit'] });
47
+ if (res.error)
48
+ throw res.error;
49
+ const j = JSON.parse(readFileSync(summary, 'utf-8'));
50
+ const v = j.metrics?.[trend]?.[stat];
51
+ return typeof v === 'number' ? v : null;
52
+ }
53
+ catch (e) {
54
+ if (e.code === 'ENOENT')
55
+ return null;
56
+ throw e;
57
+ }
58
+ finally {
59
+ rmSync(dir, { recursive: true, force: true });
60
+ }
61
+ };
62
+ function sizeList(raw) {
63
+ const sizes = raw.split(',').map((s) => positive(s.trim()));
64
+ return sizes;
65
+ }
66
+ export function buildScalingCurveCommand(deps, runner = defaultK6Runner) {
67
+ return new Command('scaling-curve')
68
+ .description('Fit how a cost metric grows with input size; flag superlinear growth. ' +
69
+ 'Advisory; abstains (exit 3) when the data cannot support a verdict.')
70
+ .argument('[points-file]', 'JSON {"metric","points":[{size,value}]} or CSV with a size,value header')
71
+ .option('--target <size>', 'extrapolate the fitted cost to this size', positive)
72
+ .option('--run <k6-script>', 'run the ladder: the script once per size with SIZE set')
73
+ .option('--sizes <list>', 'comma-separated sizes for --run', sizeList)
74
+ .option('--repeats <n>', 'runs per size for --run (noise rule)', positive, 3)
75
+ .option('--metric <trend:stat>', 'k6 metric for --run', 'http_req_duration:p(95)')
76
+ .option('--json', 'Output the verdict and its points as JSON.')
77
+ .action((file, opts) => {
78
+ const { metric, points, missing } = collect(file, opts, runner);
79
+ let r = fitScaling(points, opts.target === undefined ? {} : { target: opts.target });
80
+ // A run that never emitted the metric is an abstention even if the other
81
+ // runs would fit: the missing runs are a hole in the curve, not a zero.
82
+ if (missing > 0 && r.verdict === 'INSUFFICIENT_DATA') {
83
+ r.reasons.unshift(`metric ${metric} was not emitted by ${missing} run(s)`);
84
+ }
85
+ if (missing > 0 && r.verdict !== 'INSUFFICIENT_DATA') {
86
+ r = {
87
+ verdict: 'INSUFFICIENT_DATA',
88
+ reasons: [`metric ${metric} was not emitted by ${missing} run(s)`],
89
+ points: r.points,
90
+ };
91
+ }
92
+ const abstained = r.verdict === 'INSUFFICIENT_DATA';
93
+ if (opts.json) {
94
+ deps.out(jsonIndent2({ metric: metric ?? null, ...r }));
95
+ }
96
+ else {
97
+ const label = metric ?? 'value';
98
+ deps.out(pc.bold(`Scaling curve: ${label} vs size`));
99
+ for (const p of r.points) {
100
+ deps.out(` ${String(p.size).padStart(10)} ${Number(p.value.toPrecision(4))} ${pc.dim(`(${p.samples} sample(s))`)}`);
101
+ }
102
+ if (r.verdict === 'INSUFFICIENT_DATA') {
103
+ deps.out(pc.bold(pc.yellow(`${WARN} INSUFFICIENT_DATA -- this is not a pass.`)));
104
+ for (const reason of r.reasons)
105
+ deps.out(` - ${reason}`);
106
+ }
107
+ else {
108
+ const color = r.verdict === 'LINEAR_OR_BETTER' ? pc.green : pc.yellow;
109
+ deps.out(`${color(pc.bold(r.verdict))}: exponent ${r.exponent.toFixed(2)} ` +
110
+ `(R² ${r.r2.toFixed(3)}) -- ${MEANING[r.verdict]}`);
111
+ deps.out(r.knee === null
112
+ ? ' no knee: the curve does not bend within the measured range'
113
+ : ` knee at size ${r.knee}: growth steepens from here`);
114
+ if (r.extrapolation) {
115
+ const e = r.extrapolation;
116
+ deps.out(` extrapolated ${label} at ${e.size}: ${e.value.toFixed(1)} ` +
117
+ pc.dim(`(${e.multipleOfLargest}x the largest measured size -- an extrapolation, not a measurement)`));
118
+ }
119
+ }
120
+ }
121
+ if (abstained)
122
+ throw new CliExitError(EXIT_ABSTAINED);
123
+ });
124
+ }
125
+ function collect(file, opts, runner) {
126
+ if (opts.run === undefined) {
127
+ if (file === undefined) {
128
+ throw new InvalidArgumentError('give a points file, or --run with --sizes');
129
+ }
130
+ return { ...parsePoints(readFileSync(file, 'utf-8')), missing: 0 };
131
+ }
132
+ if (!opts.sizes)
133
+ throw new InvalidArgumentError('--run needs --sizes');
134
+ const points = [];
135
+ let missing = 0;
136
+ for (const size of opts.sizes) {
137
+ for (let i = 0; i < opts.repeats; i += 1) {
138
+ const value = runner(opts.run, size, opts.metric);
139
+ if (value === null)
140
+ missing += 1;
141
+ else
142
+ points.push({ size, value });
143
+ }
144
+ }
145
+ return { metric: opts.metric, points, missing };
146
+ }
147
+ //# sourceMappingURL=scaling-curve-cli.js.map