canary-test-cli 7.1.0 → 7.2.0

package/dist/engine/core/skill-registry.js

@@ -67,11 +67,25 @@ function codePointCompare(a, b) {
     }
     return ca.length - cb.length;
 }
-/** Bundled skills live at `<repo>/agents/skills`. Python: `_AGENTS_SKILLS_DIR`. */
+/**
+ * Bundled skills live at `<root>/agents/skills`, where `<root>` is three
+ * directories above this module. Python: `_AGENTS_SKILLS_DIR`.
+ *
+ * The "three levels up" is a PACKAGING CONTRACT, not an implementation detail
+ * (#757). It holds in the source tree (`ts/src/core`), in the compiled tree
+ * (`ts/dist/core`), and in the published npm package (`dist/engine/core`) --
+ * but only while whatever sits at that root actually ships an `agents/skills`.
+ * It did not: `canary-test-cli@7.1.0` published `bin/` and `dist/` only, so an
+ * installed CLI resolved this to a directory that has never existed and
+ * reported every bundled skill as absent, from any cwd. Exported so
+ * `ts/test/skill-packaging.test.ts` can pin both halves of the contract.
+ */
+export function bundledSkillsDirFrom(moduleDir) {
+    // moduleDir = <root>/<a>/<b>/core -> the root is three levels up.
+    return resolve(moduleDir, '..', '..', '..', 'agents', 'skills');
+}
 function defaultAgentsSkillsDir() {
-    const here = dirname(fileURLToPath(import.meta.url));
-    // here = ts/src/core -> repo root is three levels up (Python: parents[2]).
-    return resolve(here, '..', '..', '..', 'agents', 'skills');
+    return bundledSkillsDirFrom(dirname(fileURLToPath(import.meta.url)));
 }
 /**
  * A discovered skill (Python: `SkillInfo` dataclass).
@@ -150,6 +164,47 @@ export class SkillRegistry {
         }
         return [...skills.values()].sort((a, b) => codePointCompare(a.name, b.name));
     }
+    /**
+     * The directory tiers {@link discover} consults, in precedence order.
+     *
+     * Exists so an empty discovery can state its denominator (#757). "No skills
+     * found." is a claim about the world; what discovery can actually attest is
+     * "none of THESE four roots held one", and the two read very differently to
+     * someone standing in a directory full of SKILL.md files. The local tier
+     * walks cwd up to the git root, so it renders as the range it swept rather
+     * than one line per ancestor.
+     */
+    searchRoots(root) {
+        const searchRoot = resolve(root ?? process.cwd());
+        const ancestors = SkillRegistry.ancestorsToGitRoot(searchRoot);
+        const localDirs = ancestors.map((a) => join(a, '.canary', 'skills'));
+        const overlaysRoot = join(this.home, '.canary', 'overlays');
+        const globalDir = join(this.home, '.canary', 'skills');
+        return [
+            {
+                tier: 'bundled',
+                path: this.agentsSkillsDir,
+                exists: existsSync(this.agentsSkillsDir),
+            },
+            {
+                tier: 'overlay',
+                path: overlaysRoot,
+                exists: SkillRegistry.isDir(overlaysRoot),
+            },
+            {
+                tier: 'global',
+                path: globalDir,
+                exists: SkillRegistry.isDir(globalDir),
+            },
+            {
+                tier: 'local',
+                path: localDirs.length > 1
+                    ? `${localDirs[0]} (and ${localDirs.length - 1} ancestor(s) up to the git root)`
+                    : (localDirs[0] ?? join(searchRoot, '.canary', 'skills')),
+                exists: localDirs.some((d) => existsSync(d)),
+            },
+        ];
+    }
     /** Return the SkillInfo for `name` honoring precedence, or null. */
     find(name, root) {
         for (const skill of this.discover(root)) {

package/dist/engine/core/test-files.js

@@ -0,0 +1,77 @@
+/**
+ * The one answer to "which files in this tree are tests?" (#755).
+ *
+ * Extracted from `cli-commands.ts`, where it was private, because
+ * `canary-cassandra`'s skill CLI needs the SAME answer as `canary
+ * vacuity-check`. The four Tier-0 detectors are meant to be mergeable by a
+ * single consumer, and two collectors disagreeing about the denominator is the
+ * quietest way for that to stop being true: the same run would report a
+ * different `checked` depending on which door it came through.
+ *
+ * The walk's ignore set is load-bearing (#566): a dependency's own test suite
+ * is not the consumer's to fix. One downstream run before that fix produced 254
+ * of 256 findings inside `node_modules`, with the only `critical` in vendored
+ * code.
+ */
+import { readdirSync, statSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { JS_TEST_EXTENSIONS } from './static-linter.js';
+/** True when `p` is a readable directory. A missing path is not one. */
+export function isDir(p) {
+    try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/** Directories never worth walking; see the module docstring for why. */
+const IGNORED_DIRS = new Set([
+    'node_modules',
+    '.git',
+    '__pycache__',
+    '.venv',
+    'venv',
+    'dist',
+    'build',
+    '.next',
+    '.nuxt',
+]);
+function walkFiles(dir) {
+    const out = [];
+    let entries;
+    try {
+        entries = readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return out;
+    }
+    for (const e of entries) {
+        const full = join(dir, e.name);
+        if (e.isDirectory()) {
+            if (!IGNORED_DIRS.has(e.name))
+                out.push(...walkFiles(full));
+        }
+        else if (e.isFile())
+            out.push(full);
+    }
+    return out;
+}
+/**
+ * `test_*.py` plus `*.test.*` / `*.spec.*` over every extension the scanners
+ * can actually read -- `.mjs` and `.cjs` included, which is the half of #566
+ * that made a directory of ESM tests collect zero files.
+ */
+const JS_TEST_FILE_RE = new RegExp(`\\.(test|spec)\\.(${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join('|')})$`);
+/** Recursive test-file glob matching Python's `rglob` union, sorted by path. */
+export function collectTestFiles(dir) {
+    return walkFiles(dir)
+        .filter((p) => {
+        const b = basename(p);
+        return ((b.startsWith('test_') && b.endsWith('.py')) || JS_TEST_FILE_RE.test(b));
+    })
+        .sort();
+}
+/** Human-readable list of what {@link collectTestFiles} looks for. */
+export const SCANNABLE_DESC = `test_*.py, *.test|spec.{${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join(',')}}`;
+//# sourceMappingURL=test-files.js.map

package/dist/engine/core/vacuity-scanner.js

@@ -34,6 +34,15 @@
  * - neither -- the target cannot be resolved. That is "cannot verify", which is
  *   a finding about the SCAN, so it lands in `skipped` with its reason.
  *
+ * The inference reads four binding forms, because #705 measured what happens
+ * when it reads one: 65 of 99 findings on canary's own tree were the namespace
+ * import, the dynamic import, and the subprocess launch -- tests invoking
+ * exactly what they claimed, through a construct the target set could not see.
+ * The issue rules out every quiet answer (a threshold, a mute, a widened blanket
+ * skip) on the grounds that a suppressed inference and a passing check must not
+ * look alike, so the fix is to WIDEN WHAT THE INFERENCE CAN SEE and leave the
+ * rule's authority untouched.
+ *
  * A skip is PER RULE, not per test: `VAC-001` needs no target and always runs,
  * so a test whose target is unresolvable is still genuinely `checked` and stays
  * in the denominator. Saying otherwise would understate what was verified. What
@@ -59,9 +68,47 @@ import { blankStringContent } from './string-literals.js';
  * annotation above the declaration.
  */
 const COVERS_PRAGMA = /@covers\s+([A-Za-z_$][\w$]*)/g;
-/** An import whose specifier is relative: the local code a test can target. */
-const JS_RELATIVE_IMPORT = /import\s+(?:type\s+)?(?:\{([^}]*)\}|(\w+))[^'"]*from\s*['"](\.[^'"]*)['"]/g;
+/**
+ * An import whose specifier is relative: the local code a test can target.
+ *
+ * The namespace form (`import * as ns from './x.js'`) needs its own alternative
+ * rather than falling out of `(\w+)`: `*` is not a word character, so a file
+ * written entirely in namespace imports resolved to an EMPTY target set and
+ * every test in it drew a `VAC-002` (#705). On canary's own `agents/skills/test`
+ * tree that single omission was the largest share of the 65 findings the issue
+ * counted -- `import * as diffscan from '../.../diffscan.mjs'` is the house
+ * style there, and `diffscan.findDeletions(...)` is unmistakably an invocation
+ * of the target.
+ */
+const JS_RELATIVE_IMPORT = /import\s+(?:type\s+)?(?:\*\s+as\s+(\w+)|\{([^}]*)\}|(\w+))[^'"]*from\s*['"](\.[^'"]*)['"]/g;
 const JS_RELATIVE_REQUIRE = /(?:const|let|var)\s+(?:\{([^}]*)\}|(\w+))\s*=\s*require\s*\(\s*['"](\.[^'"]*)['"]/g;
+/**
+ * `const x = await import('./y.js')` / `const { a } = await import('./y.js')`.
+ *
+ * A dynamic import leaves no static import statement, so a suite that loads its
+ * subject this way -- to control module state per test, or to import a module
+ * only after an env var is set -- resolved to no target at all (#705).
+ */
+const JS_DYNAMIC_IMPORT = /(?:const|let|var)\s+(?:\{([^}]*)\}|(\w+))\s*=\s*(?:await\s+)?import\s*\(\s*['"](\.[^'"]*)['"]\s*\)/g;
+/** A bare `await import('./y.js')` -- no binding, so it names no symbol. */
+const JS_BARE_DYNAMIC_IMPORT = /(?<![\w$.])import\s*\(\s*['"]\.[^'"]*['"]\s*\)/;
+/**
+ * A string literal naming a first-party script -- something a subprocess can be
+ * pointed at and that lives in this repo.
+ *
+ * The discriminator is deliberately the EXTENSION, not the path shape: it is
+ * what separates `spawnSync(cli, ...)` where `cli` is
+ * `path.join(SCRIPTS, 'cli.mjs')` from `spawnSync('git', args)`. A bare command
+ * name is not a repo path and must not make a test look covered.
+ */
+const SCRIPT_PATH_LITERAL = /['"`][^'"`\n]*[\w$)/.-]\.(?:mjs|cjs|jsx?|tsx?|py|sh)['"`]/;
+/**
+ * A declaration binding one name to an expression -- the statement-bounded form
+ * used to spot a handle on a first-party script.
+ */
+const JS_SIMPLE_DECL = /(?:^|\n)\s*(?:const|let|var)\s+(\w+)\s*=\s*([^;\n]+)/g;
+/** The child-process launchers whose first argument is an executable target. */
+const SUBPROCESS_LAUNCH = /(?<![\w$.])(?:execFileSync|execSync|spawnSync|execFile|spawn|fork)\s*\(/;
 /**
  * Python has no `.`-prefix requirement for a first-party import, so `from x
  * import y` counts. `import os` and the stdlib are excluded by name below --
@@ -210,26 +257,132 @@ function pythonImportedTargets(code) {
     }
     return names;
 }
+/**
+ * One binding form: which capture holds the `{...}` clause, and which the single
+ * name (default import, namespace alias, or `const x = ...`).
+ */
+const JS_BINDING_FORMS = [
+    { re: JS_RELATIVE_IMPORT, clause: 2, singles: [1, 3] },
+    { re: JS_RELATIVE_REQUIRE, clause: 1, singles: [2] },
+    { re: JS_DYNAMIC_IMPORT, clause: 1, singles: [2] },
+];
 /** JS/TS first-party imports: any `import`/`require` with a relative specifier. */
 function jsImportedTargets(code) {
     const names = new Set();
-    for (const re of [JS_RELATIVE_IMPORT, JS_RELATIVE_REQUIRE]) {
+    for (const { re, clause, singles } of JS_BINDING_FORMS) {
         // Reset explicitly: these are module-level `/g` patterns, so a leftover
         // `lastIndex` from an earlier file would silently skip the head of this one.
         re.lastIndex = 0;
         for (const m of code.matchAll(re)) {
-            for (const n of clauseNames(m[1]))
+            for (const n of clauseNames(m[clause]))
                 names.add(n);
-            if (m[2])
-                names.add(m[2]);
+            for (const g of singles)
+                if (m[g])
+                    names.add(m[g]);
         }
     }
+    for (const n of subprocessScriptHandles(code))
+        names.add(n);
     return names;
 }
+/**
+ * Names bound to a first-party SCRIPT PATH -- the target of a subprocess test.
+ *
+ * `agents/skills/test` drives most skill CLIs the way a user does, by spawning
+ * them: `const cli = path.join(SCRIPTS, 'cli.mjs'); spawnSync(cli, ['--help'])`.
+ * No symbol crosses that boundary, so import-inferred fidelity saw a test that
+ * referenced none of its file's imports and reported `VAC-002` on a test that is
+ * in fact exercising exactly what it claims (#705).
+ *
+ * The handle -- `cli` -- is the symbol that stands in for the target, so binding
+ * it is what lets the existing machinery work unchanged, `closeOverLocals`
+ * included. A launch site must be present in the file: a path literal on its own
+ * is data (a fixture, an expected value), not an invocation.
+ */
+function subprocessScriptHandles(code) {
+    const names = new Set();
+    if (!SUBPROCESS_LAUNCH.test(code))
+        return names;
+    JS_SIMPLE_DECL.lastIndex = 0;
+    for (const m of code.matchAll(JS_SIMPLE_DECL)) {
+        if (SCRIPT_PATH_LITERAL.test(m[2] ?? ''))
+            names.add(m[1]);
+    }
+    return names;
+}
+/**
+ * Does this test body itself reach first-party code the target set cannot name?
+ *
+ * Two shapes, both of which leave no identifier to match: a subprocess launched
+ * at a script path written inline (`spawnSync(path.join(D, 'cli.mjs'), ...)`),
+ * and a bare `await import('./x.js')` whose result is never bound. Read from the
+ * ORIGINAL source rather than the blanked copy, because the evidence in both
+ * cases IS the string literal.
+ *
+ * `SUBPROCESS_LAUNCH` and `SCRIPT_PATH_LITERAL` are required together: a test
+ * that spawns `git` and separately mentions a `.py` fixture path is not covered
+ * by either half alone.
+ */
+function reachesOutOfBandTarget(rawBody) {
+    if (SUBPROCESS_LAUNCH.test(rawBody) && SCRIPT_PATH_LITERAL.test(rawBody))
+        return true;
+    return JS_BARE_DYNAMIC_IMPORT.test(rawBody);
+}
 /** Names imported from first-party (relative) modules. */
 function importedTargets(code, python) {
     return python ? pythonImportedTargets(code) : jsImportedTargets(code);
 }
+/**
+ * Brace depth immediately BEFORE each character, over already-blanked code.
+ *
+ * Cheap and approximate on purpose: string content is blanked before this runs,
+ * so the only braces it can see are real ones (a `{` inside a comment is the
+ * residual inaccuracy, and it can only widen a body, never narrow one).
+ */
+function braceDepths(code) {
+    const depths = new Int32Array(code.length);
+    let d = 0;
+    for (let i = 0; i < code.length; i += 1) {
+        depths[i] = d;
+        const c = code[i];
+        if (c === '{')
+            d += 1;
+        else if (c === '}')
+            d -= 1;
+    }
+    return depths;
+}
+/**
+ * Where declaration `i`'s body ends.
+ *
+ * Bounding it at the NEXT declaration is wrong for any helper that declares
+ * something inside itself, and that is the common shape for the subprocess
+ * helper #705 is about:
+ *
+ * ```ts
+ * const SCRIPT = join(REPO_ROOT, 'scripts', 'entropy-ratchet.mjs');
+ * function run() {
+ *   const r = spawnSync(process.execPath, [SCRIPT, ...]);   // <- next decl
+ *   ...
+ * }
+ * ```
+ *
+ * `run`'s body stopped at `const r`, so it never saw `SCRIPT`, so `run()` did
+ * not reach the target and every test calling it read as vacuous. Nesting is the
+ * discriminator: the body runs to the next declaration at the same or shallower
+ * brace depth, which is the first one that is genuinely a SIBLING.
+ */
+function declEnd(code, matches, i, depth) {
+    if (depth === null)
+        return matches[i + 1]?.index ?? code.length;
+    const own = depth[matches[i].index] ?? 0;
+    for (let j = i + 1; j < matches.length; j += 1) {
+        const at = matches[j].index;
+        if ((depth[at] ?? 0) <= own)
+            return at;
+    }
+    return code.length;
+}
 /**
  * Grow `targets` with local declarations that themselves reach a target, to a
  * fixpoint.
@@ -245,13 +398,14 @@ function closeOverLocals(code, targets, python) {
     const re = python ? PY_LOCAL_DECL : JS_LOCAL_DECL;
     re.lastIndex = 0;
     const matches = [...code.matchAll(re)];
+    const depth = python ? null : braceDepths(code);
     for (let i = 0; i < matches.length; i += 1) {
         const m = matches[i];
         const names = boundNames(m, python);
         if (names.length === 0)
             continue;
         const start = m.index + m[0].length;
-        const end = matches[i + 1]?.index ?? code.length;
+        const end = declEnd(code, matches, i, depth);
         decls.push({ names, body: code.slice(start, end) });
     }
     if (!python) {
@@ -352,12 +506,12 @@ function pyTautology(line) {
         return false;
     return normalize(cmp[1]) === normalize(cmp[2]);
 }
-function scanBlock(code, block, file, python, reaching, annotated, skipped) {
+function scanBlock(code, block, file, python, reaching, annotated, skipped, outOfBand) {
     const lines = bodyLines(code, block).filter((l) => !isComment(l.text));
     const targets = annotated !== null ? new Set([annotated]) : reaching;
     return [
         ...tautologies(lines, block, file, python),
-        ...targetNeverInvoked(block, file, reaching, annotated),
+        ...targetNeverInvoked(block, file, reaching, annotated, outOfBand),
         ...absenceOnly(lines, block, file, python, targets, skipped),
     ];
 }
@@ -394,7 +548,7 @@ function tautologies(lines, block, file, python) {
         .map((l) => mk(file, l.line, 'VAC-001', 'critical', block.name, 'Assertion compares a value with itself; no implementation can fail it.', 'Assert the value the code under test should have produced, not the input.'));
 }
 /** VAC-002 -- the target is never referenced anywhere in the body. */
-function targetNeverInvoked(block, file, reaching, annotated) {
+function targetNeverInvoked(block, file, reaching, annotated, outOfBand) {
     if (annotated !== null) {
         if (mentionsAny(block.body, new Set([annotated])))
             return [];
@@ -402,6 +556,10 @@ function targetNeverInvoked(block, file, reaching, annotated) {
             mk(file, block.line, 'VAC-002', 'warning', block.name, `Declared target \`${annotated}\` is never referenced in this test.`, 'Invoke the target, or correct the @covers annotation to name what the test actually exercises.', 'annotated'),
         ];
     }
+    // The subprocess / bare-dynamic-import shapes reach first-party code without
+    // naming a symbol, so no target set can ever match them (#705).
+    if (outOfBand)
+        return [];
     if (reaching === null || mentionsAny(block.body, reaching))
         return [];
     return [
@@ -521,7 +679,7 @@ export function scanVacuity(path) {
     const blocks = enumerateTests(code, source, python);
     const reaching = resolveTargets(source, code, python);
     const skipped = [];
-    const findings = scanAllBlocks({ code, path, python, reaching }, blocks, skipped);
+    const findings = scanAllBlocks({ code, source, path, python, reaching }, blocks, skipped);
     const result = {
         checked: blocks.length,
         findings,
@@ -539,17 +697,29 @@ function scanAllBlocks(ctx, blocks, skipped) {
         const prev = blocks[i - 1];
         const floor = prev ? prev.bodyStart + prev.body.length : 0;
         const annotated = annotationFor(ctx.code, block, floor);
+        const outOfBand = !ctx.python &&
+            annotated === null &&
+            reachesOutOfBandTarget(ctx.source.slice(block.bodyStart, block.bodyStart + block.body.length));
         if (annotated === null && ctx.reaching === null) {
             // Both target-dependent rules go dark together, and both say so. VAC-003
             // asks "does any assertion observe the target", which is unanswerable
             // without a target -- so it abstains rather than falling back to the
             // 254-false-positive version of itself.
+            //
+            // An out-of-band reach answers VAC-002 (the test DOES invoke first-party
+            // code) but not VAC-003, which needs a SYMBOL to ask "did an assertion
+            // observe it". So the skip narrows rather than disappearing: reporting
+            // both as dark would overstate the gap, dropping it entirely would hide a
+            // real one, and #705 is explicit that a suppressed inference and a passing
+            // check must not look alike.
             skipped.push({
-                name: `VAC-002/VAC-003 (${block.name})`,
-                reason: 'target unresolvable: no @covers annotation and no first-party relative import to infer from',
+                name: `${outOfBand ? 'VAC-003' : 'VAC-002/VAC-003'} (${block.name})`,
+                reason: outOfBand
+                    ? 'target reached out of band (subprocess or bare dynamic import), so VAC-002 is answered but no symbol exists for absence-only to observe'
+                    : 'target unresolvable: no @covers annotation and no first-party relative import to infer from',
             });
         }
-        findings.push(...scanBlock(ctx.code, block, ctx.path, ctx.python, ctx.reaching, annotated, skipped));
+        findings.push(...scanBlock(ctx.code, block, ctx.path, ctx.python, ctx.reaching, annotated, skipped, outOfBand));
     }
     return findings;
 }

package/dist/engine/guardian/analysis-emit.js

@@ -36,13 +36,17 @@ import { mkdirSync, renameSync, statSync, unlinkSync, writeFileSync, } from 'nod
 import { dirname, join } from 'node:path';
 import { ensureAscii } from '../util/ensure-ascii.js';
 import { coverageDegradedNotice, coverageStatus, } from './coverage.js';
-import { combineNotices, renderFindings } from './pr-check.js';
+import { combineNotices, renderFindings, } from './pr-check.js';
 // 1.1 adds the additive `coverage` block (#554); readers of 1.0 are unaffected.
 // 1.2 adds the additive `skipped` list (#582). Additive again, and bumped again
 // for the reason recorded in #572: a reader that pins a version must be able to
 // tell which fields it can rely on being present, and silence about a new field
 // is indistinguishable from the field being absent for a real reason.
-export const SCHEMA_VERSION = '1.2';
+// 1.3 adds the additive `provenance` block (#761). Same additive rule. This is
+// the field an archived record needs most: when a run is questioned WEEKS later
+// from its uploaded artifact, `checked: 43` is only interpretable next to the
+// diff endpoints that produced it.
+export const SCHEMA_VERSION = '1.3';
 const ANALYSIS_SOURCE = 'canary-pr-guardian';
 const REF_SAFE = /[^A-Za-z0-9._-]/g;
 const REF_MAX = 100; // cap the sanitized ref so a long branch never hits ENAMETOOLONG
@@ -111,6 +115,7 @@ export function buildAnalysisRecord(findings, args) {
             ? null
             : { status: coverageStatus(coverage), ...coverage },
         skipped: args.skipped ?? [],
+        provenance: args.provenance ?? null,
         summary: {
             total: findings.length,
             unaddressed: active.length,