canary-test-cli 7.1.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/agents/skills/README.md +327 -0
  2. package/agents/skills/canary:generate.md +49 -0
  3. package/agents/skills/canary:init.md +37 -0
  4. package/agents/skills/canary:migrate.md +66 -0
  5. package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
  6. package/agents/skills/claude-code/canary-batwoman/SKILL.md +119 -0
  7. package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
  8. package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
  9. package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
  10. package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
  11. package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
  12. package/agents/skills/claude-code/canary-cassandra/SKILL.md +187 -0
  13. package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +270 -0
  14. package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
  15. package/agents/skills/claude-code/canary-ci-ready/SKILL.md +178 -0
  16. package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
  17. package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
  18. package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
  19. package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
  20. package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
  21. package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
  22. package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
  23. package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
  24. package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
  25. package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
  26. package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
  27. package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
  28. package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
  29. package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
  30. package/agents/skills/claude-code/canary-fleet-health/SKILL.md +197 -0
  31. package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
  32. package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
  33. package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
  34. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
  35. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
  36. package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
  37. package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
  38. package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
  39. package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
  40. package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
  41. package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
  42. package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
  43. package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +144 -0
  44. package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
  45. package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
  46. package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
  47. package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
  48. package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
  49. package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
  50. package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
  51. package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
  52. package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
  53. package/agents/skills/claude-code/canary-screech/SKILL.md +109 -0
  54. package/agents/skills/claude-code/canary-screech/scripts/blast.mjs +125 -0
  55. package/agents/skills/claude-code/canary-screech/scripts/cli.mjs +128 -0
  56. package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs +97 -0
  57. package/agents/skills/claude-code/canary-screech/scripts/history.mjs +73 -0
  58. package/agents/skills/claude-code/canary-screech/scripts/redness.mjs +94 -0
  59. package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
  60. package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
  61. package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
  62. package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
  63. package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
  64. package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
  65. package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
  66. package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
  67. package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
  68. package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
  69. package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
  70. package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
  71. package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
  72. package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
  73. package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
  74. package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
  75. package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
  76. package/agents/skills/lib/parse-args.mjs +275 -0
  77. package/dist/engine/analysis/batwoman/audit.js +39 -0
  78. package/dist/engine/analysis/batwoman/closure.js +159 -0
  79. package/dist/engine/analysis/batwoman/gh-history.js +119 -0
  80. package/dist/engine/analysis/batwoman/probes.js +195 -0
  81. package/dist/engine/analysis/batwoman/registry.js +142 -0
  82. package/dist/engine/analysis/batwoman/render.js +194 -0
  83. package/dist/engine/analysis/batwoman/run-window.js +122 -0
  84. package/dist/engine/analysis/batwoman/text.js +84 -0
  85. package/dist/engine/analysis/batwoman/triggers.js +122 -0
  86. package/dist/engine/analysis/batwoman/verdict.js +64 -0
  87. package/dist/engine/analysis/cli.js +47 -14
  88. package/dist/engine/analysis/gh-flaky/gh-run-attempts.js +206 -0
  89. package/dist/engine/batwoman-cli.js +119 -0
  90. package/dist/engine/ci-ready-cli.js +71 -0
  91. package/dist/engine/cli-commands.js +49 -72
  92. package/dist/engine/cli.core.js +16 -0
  93. package/dist/engine/company-knowledge-cli.js +10 -2
  94. package/dist/engine/core/ci-ready.js +112 -0
  95. package/dist/engine/core/company-knowledge.js +8 -0
  96. package/dist/engine/core/migrator.js +147 -20
  97. package/dist/engine/core/permission-matrix.js +219 -0
  98. package/dist/engine/core/quality-scorer.js +27 -19
  99. package/dist/engine/core/scaling-curve.js +143 -0
  100. package/dist/engine/core/skill-dispatch.js +115 -0
  101. package/dist/engine/core/skill-examples.js +103 -3
  102. package/dist/engine/core/skill-registry.js +59 -4
  103. package/dist/engine/core/string-literals.js +3 -1
  104. package/dist/engine/core/test-files.js +77 -0
  105. package/dist/engine/core/vacuity-scanner.js +330 -15
  106. package/dist/engine/core/workflow-discovery.js +41 -23
  107. package/dist/engine/guardian/adjudication-github.js +136 -0
  108. package/dist/engine/guardian/adjudication.js +119 -340
  109. package/dist/engine/guardian/analysis-emit.js +7 -2
  110. package/dist/engine/guardian/cli.js +277 -249
  111. package/dist/engine/guardian/coverage.js +2 -1
  112. package/dist/engine/guardian/diff-coverage/coverage-delta.js +162 -0
  113. package/dist/engine/guardian/diff-coverage/formats/cobertura.js +45 -1
  114. package/dist/engine/guardian/diff-coverage/orchestrator.js +25 -21
  115. package/dist/engine/guardian/diff-coverage/paths.js +5 -9
  116. package/dist/engine/guardian/diff-coverage/report-tier.js +88 -12
  117. package/dist/engine/guardian/diff-extractor.js +31 -32
  118. package/dist/engine/guardian/pr-check.js +354 -223
  119. package/dist/engine/guardian/pr-comment.js +35 -58
  120. package/dist/engine/guardian/weak-test.js +236 -0
  121. package/dist/engine/mcp-server.js +67 -4
  122. package/dist/engine/permission-matrix-cli.js +51 -0
  123. package/dist/engine/scaling-curve-cli.js +147 -0
  124. package/dist/engine/skills-cli.js +171 -51
  125. package/dist/engine/workflow-cli.js +85 -65
  126. package/dist/reporters/testtracker.d.ts +1 -1
  127. package/dist/reporters/testtracker.js +1 -1
  128. package/package.json +3 -2
@@ -34,6 +34,15 @@
34
34
  * - neither -- the target cannot be resolved. That is "cannot verify", which is
35
35
  * a finding about the SCAN, so it lands in `skipped` with its reason.
36
36
  *
37
+ * The inference reads four binding forms, because #705 measured what happens
38
+ * when it reads one: 65 of 99 findings on canary's own tree were the namespace
39
+ * import, the dynamic import, and the subprocess launch -- tests invoking
40
+ * exactly what they claimed, through a construct the target set could not see.
41
+ * The issue rules out every quiet answer (a threshold, a mute, a widened blanket
42
+ * skip) on the grounds that a suppressed inference and a passing check must not
43
+ * look alike, so the fix is to WIDEN WHAT THE INFERENCE CAN SEE and leave the
44
+ * rule's authority untouched.
45
+ *
37
46
  * A skip is PER RULE, not per test: `VAC-001` needs no target and always runs,
38
47
  * so a test whose target is unresolvable is still genuinely `checked` and stays
39
48
  * in the denominator. Saying otherwise would understate what was verified. What
@@ -59,9 +68,47 @@ import { blankStringContent } from './string-literals.js';
59
68
  * annotation above the declaration.
60
69
  */
61
70
  const COVERS_PRAGMA = /@covers\s+([A-Za-z_$][\w$]*)/g;
62
- /** An import whose specifier is relative: the local code a test can target. */
63
- const JS_RELATIVE_IMPORT = /import\s+(?:type\s+)?(?:\{([^}]*)\}|(\w+))[^'"]*from\s*['"](\.[^'"]*)['"]/g;
71
+ /**
72
+ * An import whose specifier is relative: the local code a test can target.
73
+ *
74
+ * The namespace form (`import * as ns from './x.js'`) needs its own alternative
75
+ * rather than falling out of `(\w+)`: `*` is not a word character, so a file
76
+ * written entirely in namespace imports resolved to an EMPTY target set and
77
+ * every test in it drew a `VAC-002` (#705). On canary's own `agents/skills/test`
78
+ * tree that single omission was the largest share of the 65 findings the issue
79
+ * counted -- `import * as diffscan from '../.../diffscan.mjs'` is the house
80
+ * style there, and `diffscan.findDeletions(...)` is unmistakably an invocation
81
+ * of the target.
82
+ */
83
+ const JS_RELATIVE_IMPORT = /import\s+(?:type\s+)?(?:\*\s+as\s+(\w+)|\{([^}]*)\}|(\w+))[^'"]*from\s*['"](\.[^'"]*)['"]/g;
64
84
  const JS_RELATIVE_REQUIRE = /(?:const|let|var)\s+(?:\{([^}]*)\}|(\w+))\s*=\s*require\s*\(\s*['"](\.[^'"]*)['"]/g;
85
+ /**
86
+ * `const x = await import('./y.js')` / `const { a } = await import('./y.js')`.
87
+ *
88
+ * A dynamic import leaves no static import statement, so a suite that loads its
89
+ * subject this way -- to control module state per test, or to import a module
90
+ * only after an env var is set -- resolved to no target at all (#705).
91
+ */
92
+ const JS_DYNAMIC_IMPORT = /(?:const|let|var)\s+(?:\{([^}]*)\}|(\w+))\s*=\s*(?:await\s+)?import\s*\(\s*['"](\.[^'"]*)['"]\s*\)/g;
93
+ /** A bare `await import('./y.js')` -- no binding, so it names no symbol. */
94
+ const JS_BARE_DYNAMIC_IMPORT = /(?<![\w$.])import\s*\(\s*['"]\.[^'"]*['"]\s*\)/;
95
+ /**
96
+ * A string literal naming a first-party script -- something a subprocess can be
97
+ * pointed at and that lives in this repo.
98
+ *
99
+ * The discriminator is deliberately the EXTENSION, not the path shape: it is
100
+ * what separates `spawnSync(cli, ...)` where `cli` is
101
+ * `path.join(SCRIPTS, 'cli.mjs')` from `spawnSync('git', args)`. A bare command
102
+ * name is not a repo path and must not make a test look covered.
103
+ */
104
+ const SCRIPT_PATH_LITERAL = /['"`][^'"`\n]*[\w$)/.-]\.(?:mjs|cjs|jsx?|tsx?|py|sh)['"`]/;
105
+ /**
106
+ * A declaration binding one name to an expression -- the statement-bounded form
107
+ * used to spot a handle on a first-party script.
108
+ */
109
+ const JS_SIMPLE_DECL = /(?:^|\n)\s*(?:const|let|var)\s+(\w+)\s*=\s*([^;\n]+)/g;
110
+ /** The child-process launchers whose first argument is an executable target. */
111
+ const SUBPROCESS_LAUNCH = /(?<![\w$.])(?:execFileSync|execSync|spawnSync|execFile|spawn|fork)\s*\(/;
65
112
  /**
66
113
  * Python has no `.`-prefix requirement for a first-party import, so `from x
67
114
  * import y` counts. `import os` and the stdlib are excluded by name below --
@@ -210,26 +257,150 @@ function pythonImportedTargets(code) {
210
257
  }
211
258
  return names;
212
259
  }
260
+ /**
261
+ * One binding form: which capture holds the `{...}` clause, and which the single
262
+ * name (default import, namespace alias, or `const x = ...`).
263
+ */
264
+ const JS_BINDING_FORMS = [
265
+ { re: JS_RELATIVE_IMPORT, clause: 2, singles: [1, 3] },
266
+ { re: JS_RELATIVE_REQUIRE, clause: 1, singles: [2] },
267
+ { re: JS_DYNAMIC_IMPORT, clause: 1, singles: [2] },
268
+ ];
213
269
  /** JS/TS first-party imports: any `import`/`require` with a relative specifier. */
214
270
  function jsImportedTargets(code) {
215
271
  const names = new Set();
216
- for (const re of [JS_RELATIVE_IMPORT, JS_RELATIVE_REQUIRE]) {
272
+ for (const { re, clause, singles } of JS_BINDING_FORMS) {
217
273
  // Reset explicitly: these are module-level `/g` patterns, so a leftover
218
274
  // `lastIndex` from an earlier file would silently skip the head of this one.
219
275
  re.lastIndex = 0;
220
276
  for (const m of code.matchAll(re)) {
221
- for (const n of clauseNames(m[1]))
277
+ for (const n of clauseNames(m[clause]))
222
278
  names.add(n);
223
- if (m[2])
224
- names.add(m[2]);
279
+ for (const g of singles)
280
+ if (m[g])
281
+ names.add(m[g]);
225
282
  }
226
283
  }
284
+ for (const n of subprocessScriptHandles(code))
285
+ names.add(n);
286
+ return names;
287
+ }
288
+ /**
289
+ * Names bound to a first-party SCRIPT PATH -- the target of a subprocess test.
290
+ *
291
+ * `agents/skills/test` drives most skill CLIs the way a user does, by spawning
292
+ * them: `const cli = path.join(SCRIPTS, 'cli.mjs'); spawnSync(cli, ['--help'])`.
293
+ * No symbol crosses that boundary, so import-inferred fidelity saw a test that
294
+ * referenced none of its file's imports and reported `VAC-002` on a test that is
295
+ * in fact exercising exactly what it claims (#705).
296
+ *
297
+ * The handle -- `cli` -- is the symbol that stands in for the target, so binding
298
+ * it is what lets the existing machinery work unchanged, `closeOverLocals`
299
+ * included. A launch site must be present in the file: a path literal on its own
300
+ * is data (a fixture, an expected value), not an invocation.
301
+ */
302
+ function subprocessScriptHandles(code) {
303
+ const names = new Set();
304
+ if (!SUBPROCESS_LAUNCH.test(code))
305
+ return names;
306
+ JS_SIMPLE_DECL.lastIndex = 0;
307
+ for (const m of code.matchAll(JS_SIMPLE_DECL)) {
308
+ if (SCRIPT_PATH_LITERAL.test(m[2] ?? ''))
309
+ names.add(m[1]);
310
+ }
227
311
  return names;
228
312
  }
313
+ /**
314
+ * Does this test body itself reach first-party code the target set cannot name?
315
+ *
316
+ * Two shapes, both of which leave no identifier to match: a subprocess launched
317
+ * at a script path written inline (`spawnSync(path.join(D, 'cli.mjs'), ...)`),
318
+ * and a bare `await import('./x.js')` whose result is never bound. Read from the
319
+ * ORIGINAL source rather than the blanked copy, because the evidence in both
320
+ * cases IS the string literal.
321
+ *
322
+ * `SUBPROCESS_LAUNCH` and `SCRIPT_PATH_LITERAL` are required together: a test
323
+ * that spawns `git` and separately mentions a `.py` fixture path is not covered
324
+ * by either half alone.
325
+ */
326
+ function reachesOutOfBandTarget(rawBody) {
327
+ if (SUBPROCESS_LAUNCH.test(rawBody) && SCRIPT_PATH_LITERAL.test(rawBody))
328
+ return true;
329
+ return JS_BARE_DYNAMIC_IMPORT.test(rawBody);
330
+ }
229
331
  /** Names imported from first-party (relative) modules. */
230
332
  function importedTargets(code, python) {
231
333
  return python ? pythonImportedTargets(code) : jsImportedTargets(code);
232
334
  }
335
+ /**
336
+ * Brace depth immediately BEFORE each character, over already-blanked code.
337
+ *
338
+ * Cheap and approximate on purpose: string content is blanked before this runs,
339
+ * so the only braces it can see are real ones (a `{` inside a comment is the
340
+ * residual inaccuracy, and it can only widen a body, never narrow one).
341
+ */
342
+ function braceDepths(code) {
343
+ const depths = new Int32Array(code.length);
344
+ let d = 0;
345
+ for (let i = 0; i < code.length; i += 1) {
346
+ depths[i] = d;
347
+ const c = code[i];
348
+ if (c === '{')
349
+ d += 1;
350
+ else if (c === '}')
351
+ d -= 1;
352
+ }
353
+ return depths;
354
+ }
355
+ /**
356
+ * Where declaration `i`'s body ends.
357
+ *
358
+ * Bounding it at the NEXT declaration is wrong for any helper that declares
359
+ * something inside itself, and that is the common shape for the subprocess
360
+ * helper #705 is about:
361
+ *
362
+ * ```ts
363
+ * const SCRIPT = join(REPO_ROOT, 'scripts', 'entropy-ratchet.mjs');
364
+ * function run() {
365
+ * const r = spawnSync(process.execPath, [SCRIPT, ...]); // <- next decl
366
+ * ...
367
+ * }
368
+ * ```
369
+ *
370
+ * `run`'s body stopped at `const r`, so it never saw `SCRIPT`, so `run()` did
371
+ * not reach the target and every test calling it read as vacuous. Nesting is the
372
+ * discriminator: the body runs to the next declaration at the same or shallower
373
+ * brace depth, which is the first one that is genuinely a SIBLING.
374
+ */
375
+ function declEnd(code, matches, i, depth) {
376
+ if (depth === null)
377
+ return matches[i + 1]?.index ?? code.length;
378
+ const m = matches[i];
379
+ const own = depth[m.index] ?? 0;
380
+ let sibling = code.length;
381
+ for (let j = i + 1; j < matches.length; j += 1) {
382
+ const at = matches[j].index;
383
+ if ((depth[at] ?? 0) <= own) {
384
+ sibling = at;
385
+ break;
386
+ }
387
+ }
388
+ // A `function` owns the text up to its next sibling. A `const`/`let`/`var`
389
+ // owns only its initializer (#871): inside a test body there is usually no
390
+ // later sibling, so bounding a bystander at one let it absorb the target
391
+ // call below it and read as reaching the target, silencing VAC-003. The
392
+ // statement ends at the first `;` or line break at the declaration's own
393
+ // brace depth, so an arrow helper's braced body still belongs to it.
394
+ if (m[1])
395
+ return sibling;
396
+ const from = m.index + m[0].length;
397
+ for (let k = from; k < sibling; k += 1) {
398
+ const ch = code[k];
399
+ if ((ch === ';' || ch === '\n') && (depth[k] ?? 0) <= own)
400
+ return k;
401
+ }
402
+ return sibling;
403
+ }
233
404
  /**
234
405
  * Grow `targets` with local declarations that themselves reach a target, to a
235
406
  * fixpoint.
@@ -245,13 +416,14 @@ function closeOverLocals(code, targets, python) {
245
416
  const re = python ? PY_LOCAL_DECL : JS_LOCAL_DECL;
246
417
  re.lastIndex = 0;
247
418
  const matches = [...code.matchAll(re)];
419
+ const depth = python ? null : braceDepths(code);
248
420
  for (let i = 0; i < matches.length; i += 1) {
249
421
  const m = matches[i];
250
422
  const names = boundNames(m, python);
251
423
  if (names.length === 0)
252
424
  continue;
253
425
  const start = m.index + m[0].length;
254
- const end = matches[i + 1]?.index ?? code.length;
426
+ const end = declEnd(code, matches, i, depth);
255
427
  decls.push({ names, body: code.slice(start, end) });
256
428
  }
257
429
  if (!python) {
@@ -352,13 +524,14 @@ function pyTautology(line) {
352
524
  return false;
353
525
  return normalize(cmp[1]) === normalize(cmp[2]);
354
526
  }
355
- function scanBlock(code, block, file, python, reaching, annotated, skipped) {
527
+ function scanBlock(code, block, file, python, reaching, annotated, skipped, outOfBand) {
356
528
  const lines = bodyLines(code, block).filter((l) => !isComment(l.text));
357
529
  const targets = annotated !== null ? new Set([annotated]) : reaching;
358
530
  return [
359
531
  ...tautologies(lines, block, file, python),
360
- ...targetNeverInvoked(block, file, reaching, annotated),
532
+ ...targetNeverInvoked(block, file, reaching, annotated, outOfBand),
361
533
  ...absenceOnly(lines, block, file, python, targets, skipped),
534
+ ...presenceOnBystander(lines, block, file, python, targets),
362
535
  ];
363
536
  }
364
537
  /**
@@ -394,7 +567,7 @@ function tautologies(lines, block, file, python) {
394
567
  .map((l) => mk(file, l.line, 'VAC-001', 'critical', block.name, 'Assertion compares a value with itself; no implementation can fail it.', 'Assert the value the code under test should have produced, not the input.'));
395
568
  }
396
569
  /** VAC-002 -- the target is never referenced anywhere in the body. */
397
- function targetNeverInvoked(block, file, reaching, annotated) {
570
+ function targetNeverInvoked(block, file, reaching, annotated, outOfBand) {
398
571
  if (annotated !== null) {
399
572
  if (mentionsAny(block.body, new Set([annotated])))
400
573
  return [];
@@ -402,6 +575,10 @@ function targetNeverInvoked(block, file, reaching, annotated) {
402
575
  mk(file, block.line, 'VAC-002', 'warning', block.name, `Declared target \`${annotated}\` is never referenced in this test.`, 'Invoke the target, or correct the @covers annotation to name what the test actually exercises.', 'annotated'),
403
576
  ];
404
577
  }
578
+ // The subprocess / bare-dynamic-import shapes reach first-party code without
579
+ // naming a symbol, so no target set can ever match them (#705).
580
+ if (outOfBand)
581
+ return [];
405
582
  if (reaching === null || mentionsAny(block.body, reaching))
406
583
  return [];
407
584
  return [
@@ -422,6 +599,20 @@ function targetNeverInvoked(block, file, reaching, annotated) {
422
599
  * the write. Reported at the first absence assertion, the line an author adds a
423
600
  * precondition next to.
424
601
  */
602
+ /**
603
+ * A test title safe to put in a skip label: its first line, capped at 80.
604
+ *
605
+ * The title parser can mis-read a declaration and hand back the source after
606
+ * it (#860 -- whole `describe` bodies reached the summary line). Bounding the
607
+ * label here means a parser slip degrades to a truncated name, never a flood.
608
+ */
609
+ export function boundedTitle(name) {
610
+ const first = name.split('\n', 1)[0].trim();
611
+ return first.length > 80 ? `${first.slice(0, 80)}…` : first;
612
+ }
613
+ function skipLabel(rules, file, block) {
614
+ return `${file}:${block.line} ${rules} (${boundedTitle(block.name)})`;
615
+ }
425
616
  function absenceOnly(lines, block, file, python, targets, skipped) {
426
617
  if (targets === null)
427
618
  return [];
@@ -434,7 +625,7 @@ function absenceOnly(lines, block, file, python, targets, skipped) {
434
625
  // assertion style is one the vocabulary does not know. Both are "cannot
435
626
  // verify", so both are recorded rather than passed over in silence.
436
627
  skipped.push({
437
- name: `VAC-003 (${block.name})`,
628
+ name: skipLabel('VAC-003', file, block),
438
629
  reason: 'no recognised assertion, so absence-only could not be judged -- the test may assert nothing (LINT-006) or use an unrecognised assertion style',
439
630
  });
440
631
  return [];
@@ -447,6 +638,118 @@ function absenceOnly(lines, block, file, python, targets, skipped) {
447
638
  mk(file, assertions[0].line, 'VAC-003', 'warning', block.name, 'Every assertion in this test asserts an absence, and none of them observes the target.', 'Add one assertion proving the operation actually ran (exit code, returned value, a positive existence) -- otherwise the test passes identically when the code crashed before doing anything.'),
448
639
  ];
449
640
  }
641
+ /**
642
+ * Presence matchers that a value the test built for itself satisfies before the
643
+ * target ever runs. The negated-null forms belong here too, but on their own they
644
+ * also match `ABSENCE_ASSERTION` (via `.not.to*`), so an all-negated test stays
645
+ * VAC-003's -- see {@link presenceOnBystander}.
646
+ */
647
+ const TRIVIAL_PRESENCE_JS = /\.toBeDefined\s*\(\s*\)|\.toBeTruthy\s*\(\s*\)|\.not\s*\.\s*(?:toBeNull|toBeUndefined|toBeFalsy)\s*\(\s*\)/;
648
+ const TRIVIAL_PRESENCE_PY = /^assert\s+([A-Za-z_][\w.]*?)(?:\s+is\s+not\s+None)?\s*(?:,|$)/;
649
+ const SUBJECT_ROOT = /^([A-Za-z_$][\w$]*)(?:\.[A-Za-z_$][\w$]*)*$/;
650
+ function subjectRoot(expr) {
651
+ return expr ? (SUBJECT_ROOT.exec(normalize(expr))?.[1] ?? null) : null;
652
+ }
653
+ /** The root identifier a trivial presence assertion observes, or null. */
654
+ function presenceSubject(text, python) {
655
+ const t = text.trim();
656
+ if (python)
657
+ return subjectRoot(TRIVIAL_PRESENCE_PY.exec(t)?.[1]);
658
+ return TRIVIAL_PRESENCE_JS.test(t) ? subjectRoot(expectArgument(t)) : null;
659
+ }
660
+ /**
661
+ * Every binding of a name inside the body: `const x = rhs` / `x = rhs` (JS) or
662
+ * `x = rhs` (pytest), with the RHS bounded to its own line.
663
+ *
664
+ * Deliberately NOT `closeOverLocals`: that bounds a declaration at its next
665
+ * sibling, so a `const subs = [...]` inside a test absorbs the rest of the test
666
+ * -- including the target call -- and `subs` reads as reaching the target. That
667
+ * is exactly the bystander this rule exists to see.
668
+ */
669
+ function bodyBindings(lines, python) {
670
+ const re = python
671
+ ? /^\s*([A-Za-z_]\w*)\s*=(?!=)\s*(.*)$/
672
+ : /^\s*(const\s+|let\s+|var\s+)?([A-Za-z_$][\w$]*)\s*=(?!=)\s*(.*)$/;
673
+ const out = [];
674
+ for (const { text } of lines) {
675
+ const m = re.exec(text);
676
+ if (!m)
677
+ continue;
678
+ if (python)
679
+ out.push({ name: m[1], rhs: m[2], declared: true });
680
+ else
681
+ out.push({ name: m[2], rhs: m[3], declared: m[1] !== undefined });
682
+ }
683
+ return out;
684
+ }
685
+ /** Is `rhs` a whole statement on its line? A dangling bracket means it is not. */
686
+ function balanced(rhs) {
687
+ let d = 0;
688
+ for (const c of rhs) {
689
+ if (c === '(' || c === '[' || c === '{')
690
+ d += 1;
691
+ else if (c === ')' || c === ']' || c === '}')
692
+ d -= 1;
693
+ }
694
+ return d === 0;
695
+ }
696
+ /**
697
+ * VAC-005 -- every assertion is a trivially true presence check, and its subject
698
+ * is a BYSTANDER: a name the test itself bound, never from anything reaching the
699
+ * target (#870). The mirror image of VAC-003:
700
+ *
701
+ * ```js
702
+ * const subs = [{ id: 's1' }];
703
+ * matchSubmissions([], subs);
704
+ * expect(subs).toBeDefined(); // true before the call, true after it
705
+ * ```
706
+ *
707
+ * The bystander clause is the rule, for the same reason VAC-003's is:
708
+ * `const r = target(); expect(r).toBeDefined()` DOES observe the target, and a
709
+ * throw from it would fail the test. Anything the rule cannot prove is a
710
+ * bystander -- a name bound in a hook or at module scope, a multi-line RHS --
711
+ * yields no finding, so the error direction is always a miss, never a false
712
+ * accusation.
713
+ */
714
+ function presenceOnBystander(lines, block, file, python, targets) {
715
+ if (targets === null)
716
+ return [];
717
+ const anyAssertion = python ? PY_ASSERTION : JS_ASSERTION;
718
+ const absence = python ? PY_ABSENCE_ASSERTION : ABSENCE_ASSERTION;
719
+ const assertions = lines.filter((l) => anyAssertion.test(l.text));
720
+ if (assertions.length === 0)
721
+ return [];
722
+ // All-absence (the negated-null forms included) is VAC-003's to report.
723
+ if (assertions.every((l) => absence.test(l.text)))
724
+ return [];
725
+ const subjects = assertions.map((l) => presenceSubject(l.text, python));
726
+ if (subjects.some((s) => s === null))
727
+ return [];
728
+ const bindings = bodyBindings(lines, python);
729
+ if (bindings.some((b) => !balanced(b.rhs)))
730
+ return [];
731
+ // The body's own names are removed first -- `closeOverLocals` over-attributes
732
+ // them (see `bodyBindings`) -- then re-derived here from their real RHS, to a
733
+ // fixpoint so `const r = save(); const s = r;` still reaches.
734
+ const bound = new Set(bindings.map((b) => b.name));
735
+ const reaching = new Set([...targets].filter((t) => !bound.has(t)));
736
+ let grew = true;
737
+ while (grew) {
738
+ grew = false;
739
+ for (const b of bindings) {
740
+ if (reaching.has(b.name) || !mentionsAny(b.rhs, reaching))
741
+ continue;
742
+ reaching.add(b.name);
743
+ grew = true;
744
+ }
745
+ }
746
+ const bystander = (name) => !reaching.has(name) && bindings.some((b) => b.name === name && b.declared);
747
+ if (!subjects.every((s) => bystander(s)))
748
+ return [];
749
+ return [
750
+ mk(file, assertions[0].line, 'VAC-005', 'warning', block.name, `Every assertion is a presence check on \`${subjects[0]}\`, a value the test built itself; it holds whether or not the target ran.`, 'Assert on what the target returns or changes -- e.g. `expect(target(input)).toEqual(expected)` -- so the test fails when the target is wrong.'),
751
+ ];
752
+ }
450
753
  /** A zero-denominator result that names why it could not measure. */
451
754
  function unreadable(path, reason) {
452
755
  return { checked: 0, findings: [], skipped: [{ name: path, reason }] };
@@ -521,7 +824,7 @@ export function scanVacuity(path) {
521
824
  const blocks = enumerateTests(code, source, python);
522
825
  const reaching = resolveTargets(source, code, python);
523
826
  const skipped = [];
524
- const findings = scanAllBlocks({ code, path, python, reaching }, blocks, skipped);
827
+ const findings = scanAllBlocks({ code, source, path, python, reaching }, blocks, skipped);
525
828
  const result = {
526
829
  checked: blocks.length,
527
830
  findings,
@@ -539,17 +842,29 @@ function scanAllBlocks(ctx, blocks, skipped) {
539
842
  const prev = blocks[i - 1];
540
843
  const floor = prev ? prev.bodyStart + prev.body.length : 0;
541
844
  const annotated = annotationFor(ctx.code, block, floor);
845
+ const outOfBand = !ctx.python &&
846
+ annotated === null &&
847
+ reachesOutOfBandTarget(ctx.source.slice(block.bodyStart, block.bodyStart + block.body.length));
542
848
  if (annotated === null && ctx.reaching === null) {
543
849
  // Both target-dependent rules go dark together, and both say so. VAC-003
544
850
  // asks "does any assertion observe the target", which is unanswerable
545
851
  // without a target -- so it abstains rather than falling back to the
546
852
  // 254-false-positive version of itself.
853
+ //
854
+ // An out-of-band reach answers VAC-002 (the test DOES invoke first-party
855
+ // code) but not VAC-003, which needs a SYMBOL to ask "did an assertion
856
+ // observe it". So the skip narrows rather than disappearing: reporting
857
+ // both as dark would overstate the gap, dropping it entirely would hide a
858
+ // real one, and #705 is explicit that a suppressed inference and a passing
859
+ // check must not look alike.
547
860
  skipped.push({
548
- name: `VAC-002/VAC-003 (${block.name})`,
549
- reason: 'target unresolvable: no @covers annotation and no first-party relative import to infer from',
861
+ name: skipLabel(outOfBand ? 'VAC-003' : 'VAC-002/VAC-003', ctx.path, block),
862
+ reason: outOfBand
863
+ ? 'target reached out of band (subprocess or bare dynamic import), so VAC-002 is answered but no symbol exists for absence-only to observe'
864
+ : 'target unresolvable: no @covers annotation and no first-party relative import to infer from',
550
865
  });
551
866
  }
552
- findings.push(...scanBlock(ctx.code, block, ctx.path, ctx.python, ctx.reaching, annotated, skipped));
867
+ findings.push(...scanBlock(ctx.code, block, ctx.path, ctx.python, ctx.reaching, annotated, skipped, outOfBand));
553
868
  }
554
869
  return findings;
555
870
  }
@@ -305,6 +305,45 @@ export class WorkflowMapping {
305
305
  export class WorkflowDiscoveryError extends Error {
306
306
  }
307
307
  // ---------------------------------------------------------------------------
308
+ // Jira request helpers
309
+ // ---------------------------------------------------------------------------
310
+ // Split out of `WorkflowDiscovery.fetchJira` to pay down its complexity. Kept
311
+ // at module level so the class keeps the Python port's method set, and each
312
+ // stays at or under the perf-ratchet warning threshold (10).
313
+ /** Read the ATLASSIAN_* env vars, refusing when any is unset or empty. */
314
+ function jiraCredentials() {
315
+ const baseUrl = rstripChar(process.env['ATLASSIAN_URL'] ?? '', '/');
316
+ const user = process.env['ATLASSIAN_USER'] ?? '';
317
+ const token = process.env['ATLASSIAN_TOKEN'] ?? '';
318
+ if (!pyTruthy(baseUrl) || !pyTruthy(user) || !pyTruthy(token)) {
319
+ throw new WorkflowDiscoveryError('Jira credentials not configured. Set ATLASSIAN_URL, ' +
320
+ 'ATLASSIAN_USER, and ATLASSIAN_TOKEN environment variables.\n' +
321
+ 'Tip: add them to .canary/company.local.json or your shell profile.');
322
+ }
323
+ const auth = Buffer.from(`${user}:${token}`).toString('base64');
324
+ return {
325
+ baseUrl,
326
+ headers: { Authorization: `Basic ${auth}`, Accept: 'application/json' },
327
+ };
328
+ }
329
+ /**
330
+ * The issue-type list from the `issuetypes` response. An error object (Jira
331
+ * answers a missing project with `errorMessages` in a 200) is refused; any
332
+ * other non-list shape yields no issue types.
333
+ */
334
+ function jiraIssueTypeList(issueTypesRaw, projectKey) {
335
+ if (Array.isArray(issueTypesRaw)) {
336
+ return issueTypesRaw;
337
+ }
338
+ if (typeof issueTypesRaw === 'object' &&
339
+ issueTypesRaw !== null &&
340
+ 'errorMessages' in issueTypesRaw) {
341
+ throw new WorkflowDiscoveryError(`Jira project ${pyRepr(projectKey)} not found or access denied: ` +
342
+ `${pyRepr(issueTypesRaw['errorMessages'])}`);
343
+ }
344
+ return [];
345
+ }
346
+ // ---------------------------------------------------------------------------
308
347
  // Main class
309
348
  // ---------------------------------------------------------------------------
310
349
  /** Discovers and caches per-project issue-workflow mappings. */
@@ -399,34 +438,13 @@ export class WorkflowDiscovery {
399
438
  // -- private: Jira ---------------------------------------------------------
400
439
  /** Python: `WorkflowDiscovery._fetch_jira`. */
401
440
  async fetchJira(projectKey) {
402
- const baseUrl = rstripChar(process.env['ATLASSIAN_URL'] ?? '', '/');
403
- const user = process.env['ATLASSIAN_USER'] ?? '';
404
- const token = process.env['ATLASSIAN_TOKEN'] ?? '';
405
- if (!pyTruthy(baseUrl) || !pyTruthy(user) || !pyTruthy(token)) {
406
- throw new WorkflowDiscoveryError('Jira credentials not configured. Set ATLASSIAN_URL, ' +
407
- 'ATLASSIAN_USER, and ATLASSIAN_TOKEN environment variables.\n' +
408
- 'Tip: add them to .canary/company.local.json or your shell profile.');
409
- }
410
- const auth = Buffer.from(`${user}:${token}`).toString('base64');
411
- const headers = {
412
- Authorization: `Basic ${auth}`,
413
- Accept: 'application/json',
414
- };
441
+ const { baseUrl, headers } = jiraCredentials();
415
442
  // Capture the URL so ticket_updater can use it without requiring the env var.
416
443
  const discoveredBaseUrl = baseUrl;
417
444
  // 1. Get issue types for this project.
418
445
  const issueTypesRaw = await this.jiraGet(`${baseUrl}/rest/api/3/project/${projectKey}/issuetypes`, headers);
419
- if (!Array.isArray(issueTypesRaw) &&
420
- typeof issueTypesRaw === 'object' &&
421
- issueTypesRaw !== null &&
422
- 'errorMessages' in issueTypesRaw) {
423
- throw new WorkflowDiscoveryError(`Jira project ${pyRepr(projectKey)} not found or access denied: ` +
424
- `${pyRepr(issueTypesRaw['errorMessages'])}`);
425
- }
446
+ const list = jiraIssueTypeList(issueTypesRaw, projectKey);
426
447
  const issueTypes = [];
427
- const list = Array.isArray(issueTypesRaw)
428
- ? issueTypesRaw
429
- : [];
430
448
  for (const itRaw of list) {
431
449
  const itId = String(pyGet(itRaw, 'id', ''));
432
450
  const itName = String(pyGet(itRaw, 'name', ''));
@@ -0,0 +1,136 @@
1
+ /**
2
+ * GitHub evidence for derived adjudication (ADR 0025, #938).
3
+ *
4
+ * The network seam behind `adjudication.ts`: merged PRs in a window (GraphQL
5
+ * search), the guardian sticky's edit history (REST comments + GraphQL
6
+ * `userContentEdits`), and the merged file patches (REST). The sticky is found
7
+ * with the author-aware {@link findSticky} (#931), never by marker substring.
8
+ *
9
+ * Network lives ONLY in {@link GitHubAdjudicationSource}'s default seams;
10
+ * tests use {@link FakeAdjudicationSource} or inject `read` / `graphql`.
11
+ */
12
+ import { readAllPages, restPageReader } from './github-paging.js';
13
+ import { findSticky } from './pr-comment.js';
14
+ const API = 'https://api.github.com';
15
+ /** The most merged PRs one report walks (bounds API cost and rate limits). */
16
+ const MERGED_PR_LIMIT = 300;
17
+ /** In-memory {@link AdjudicationSource} for tests. */
18
+ export class FakeAdjudicationSource {
19
+ init;
20
+ constructor(init) {
21
+ this.init = init;
22
+ }
23
+ async mergedPrs() {
24
+ return this.init.merged;
25
+ }
26
+ async stickyHistory(pr) {
27
+ const revisions = this.init.stickies?.[pr];
28
+ return revisions === undefined ? null : { revisions };
29
+ }
30
+ async prFiles(pr) {
31
+ return this.init.files?.[pr] ?? [];
32
+ }
33
+ }
34
+ /**
35
+ * Sticky bodies oldest to newest. GitHub returns edits newest first and each
36
+ * `diff` holds the whole body at that revision. A truncated or redacted
37
+ * history is `null` (disclosed as "no edit history"), never a shorter list.
38
+ */
39
+ export function revisionsFrom(body, edits) {
40
+ if (!edits)
41
+ return null;
42
+ if (edits.totalCount === 0)
43
+ return [body];
44
+ const diffs = edits.nodes.map((n) => n.diff);
45
+ if (diffs.length < edits.totalCount || diffs.includes(null))
46
+ return null;
47
+ return [...diffs.reverse(), body];
48
+ }
49
+ const SEARCH_QUERY = `query($q: String!, $after: String) {
50
+ search(query: $q, type: ISSUE, first: 100, after: $after) {
51
+ issueCount pageInfo { hasNextPage endCursor }
52
+ nodes { ... on PullRequest { number } } } }`;
53
+ const EDITS_QUERY = `query($id: ID!) { node(id: $id) { ... on IssueComment {
54
+ userContentEdits(first: 100) { totalCount nodes { diff } } } } }`;
55
+ function defaultGraphql(token) {
56
+ return async (query, variables) => {
57
+ const resp = await fetch(`${API}/graphql`, {
58
+ method: 'POST',
59
+ headers: {
60
+ Authorization: `Bearer ${token}`,
61
+ 'User-Agent': 'canary-pr-guardian',
62
+ },
63
+ body: JSON.stringify({ query, variables }),
64
+ });
65
+ const json = (await resp.json());
66
+ if (!resp.ok || json.errors || !json.data) {
67
+ throw new Error(`GitHub GraphQL ${resp.status}: ${JSON.stringify(json)}`);
68
+ }
69
+ return json.data;
70
+ };
71
+ }
72
+ /** The real {@link AdjudicationSource} over GitHub REST + GraphQL. */
73
+ export class GitHubAdjudicationSource {
74
+ repo;
75
+ read;
76
+ graphql;
77
+ constructor(repo, token, seams = {}) {
78
+ this.repo = repo;
79
+ this.read =
80
+ seams.read ??
81
+ restPageReader({
82
+ Authorization: `Bearer ${token}`,
83
+ Accept: 'application/vnd.github+json',
84
+ 'User-Agent': 'canary-pr-guardian',
85
+ }, (status, url) => new Error(`GitHub API ${status}: ${url}`));
86
+ this.graphql = seams.graphql ?? defaultGraphql(token);
87
+ }
88
+ /** Pages the search; refuses a window past {@link MERGED_PR_LIMIT} rather than truncating. */
89
+ async mergedPrs(since) {
90
+ const q = `repo:${this.repo} is:pr is:merged merged:>=${since}`;
91
+ const numbers = [];
92
+ let after = null;
93
+ do {
94
+ const data = await this.graphql(SEARCH_QUERY, { q, after });
95
+ const search = data['search'];
96
+ if (search.issueCount > MERGED_PR_LIMIT) {
97
+ throw new Error(`${search.issueCount} merged PRs since ${since} exceeds the ` +
98
+ `${MERGED_PR_LIMIT}-PR bound; narrow --days`);
99
+ }
100
+ numbers.push(...search.nodes.map((n) => n.number));
101
+ after = search.pageInfo?.hasNextPage ? search.pageInfo.endCursor : null;
102
+ } while (after !== null);
103
+ return numbers;
104
+ }
105
+ async stickyHistory(pr) {
106
+ const url = `${API}/repos/${this.repo}/issues/${pr}/comments`;
107
+ const comments = (await readAllPages(url, this.read));
108
+ const sticky = findSticky(comments);
109
+ if (sticky === null)
110
+ return null;
111
+ const data = await this.graphql(EDITS_QUERY, { id: sticky.node_id });
112
+ const node = data['node'];
113
+ return { revisions: revisionsFrom(sticky.body, node?.userContentEdits) };
114
+ }
115
+ async prFiles(pr) {
116
+ const url = `${API}/repos/${this.repo}/pulls/${pr}/files`;
117
+ return (await readAllPages(url, this.read));
118
+ }
119
+ }
120
+ /** Walk the merged PRs; PRs without a sticky are scanned but carry no evidence. */
121
+ export async function collectEvidence(source, since) {
122
+ const merged = await source.mergedPrs(since);
123
+ const evidence = [];
124
+ for (const number of merged) {
125
+ const history = await source.stickyHistory(number);
126
+ if (history === null)
127
+ continue;
128
+ evidence.push({
129
+ number,
130
+ revisions: history.revisions,
131
+ files: await source.prFiles(number),
132
+ });
133
+ }
134
+ return { evidence, scanned: merged.length };
135
+ }
136
+ //# sourceMappingURL=adjudication-github.js.map