canary-test-cli 7.1.0 → 8.0.0

package/dist/engine/core/skill-dispatch.js

@@ -0,0 +1,115 @@
+/**
+ * Tier 2 of `canary skills run`: the dispatcher (#756).
+ *
+ * Canary shipped tier 1 (a skill declaring `cli:`/`entry:` is spawned) and
+ * tier 3 (a prose skill is unreachable), and nothing between. 14 of canary's 21
+ * skills carry no `cli:`, so no orchestrator, CI step, or sibling skill could
+ * invoke them at all -- a have/have-not split that costs far more here than the
+ * same split costs harness, where the dispatcher runs the CLI-less majority.
+ *
+ * ## What "running a prose skill" means here, honestly
+ *
+ * Canary is a CLI. It has no agent runtime, and it is not going to grow one to
+ * close this gap. So the dispatcher does the one thing a CLI can do faithfully:
+ * it RESOLVES the skill and hands back its executable contract -- identity,
+ * declared runtime requirements, and the workflow text an agent is to apply --
+ * with the tier and the determinism stated on the payload. The caller gets a
+ * resolved, machine-readable handle to a real skill instead of exit 2.
+ *
+ * What it deliberately does NOT do is apply the workflow and present the result
+ * as canary's. That would be canary claiming an answer it did not compute.
+ *
+ * ## Determinism labelling (issue design question 2)
+ *
+ * Every dispatch is stamped `determinism: 'agent-applied'`, against
+ * `'deterministic'` for a `cli:` skill. A consumer merging findings across
+ * skills must be able to tell a scanner's output from an agent's reading of a
+ * ruleset; without the label the two look interchangeable, which is exactly the
+ * confusion #755 documents about cassandra.
+ *
+ * ## Why no `--allow-executable-skills` equivalent (design question 3)
+ *
+ * That flag exists because a freshly cloned overlay can carry a `cli:` script,
+ * and invoking it runs someone else's code on the next CI run. Dispatch runs
+ * nothing: it reads a markdown file the registry already read at discovery and
+ * prints it. There is no new execution to gate, so gating it would be
+ * ceremony -- and ceremony that would keep the 14 skills unreachable in exactly
+ * the non-interactive contexts the issue is about. The trust boundary moves to
+ * whatever the caller does with the returned text, which is the caller's gate
+ * to own, and the payload labels itself so the caller can see what it holds.
+ *
+ * ## Failure mode (design question 4)
+ *
+ * A skill that could not be dispatched raises {@link SkillDispatchError}. An
+ * unreadable or bodyless SKILL.md is a failure, never an empty success -- a
+ * dispatcher that returned "nothing to do" for a skill it could not read would
+ * be indistinguishable from one that ran and found nothing.
+ */
+import { readFileSync } from 'node:fs';
+import { errnoCode } from './gate-result.js';
+// Written as an escape so this source stays ASCII, matching gate-result.ts.
+const EMDASH = '\u{2014}';
+/** A dispatch that could not be completed. Never degrades to an empty result. */
+export class SkillDispatchError extends Error {
+    skill;
+    constructor(skill, message) {
+        super(message);
+        this.name = 'SkillDispatchError';
+        this.skill = skill;
+    }
+}
+/**
+ * Strip a leading `---` frontmatter block, leaving the workflow prose.
+ *
+ * Mirrors the delimiter handling in `SkillRegistry.parseFrontmatter`: an
+ * unterminated block means the whole file was frontmatter, and there is no
+ * body to hand back.
+ */
+export function skillBody(text) {
+    if (!text.startsWith('---'))
+        return text.trim();
+    const rest = text.split('\n').slice(1);
+    const end = rest.findIndex((l) => l.trim() === '---');
+    return end === -1
+        ? ''
+        : rest
+            .slice(end + 1)
+            .join('\n')
+            .trim();
+}
+/**
+ * Resolve a prose skill into its dispatch payload.
+ *
+ * @throws {SkillDispatchError} when SKILL.md cannot be read, or holds no body.
+ */
+export function dispatchProseSkill(skill, args) {
+    let text;
+    try {
+        text = readFileSync(skill.path, 'utf-8');
+    }
+    catch (exc) {
+        const code = errnoCode(exc);
+        if (code === null)
+            throw exc;
+        throw new SkillDispatchError(skill.name, `cannot read ${skill.path} (${code}) ${EMDASH} the skill was ` +
+            'discovered but its workflow could not be loaded.');
+    }
+    const instructions = skillBody(text);
+    if (!instructions) {
+        throw new SkillDispatchError(skill.name, `${skill.path} carries frontmatter but no workflow body ${EMDASH} ` +
+            'there is nothing to dispatch. Reporting this as an empty run would ' +
+            'be indistinguishable from a skill that ran and found nothing.');
+    }
+    return {
+        skill: skill.name,
+        path: skill.path,
+        tier: 'dispatcher',
+        determinism: 'agent-applied',
+        requires_agent_runtime: true,
+        requires: skill.requires,
+        description: skill.description,
+        instructions,
+        args,
+    };
+}
+//# sourceMappingURL=skill-dispatch.js.map

package/dist/engine/core/skill-examples.js

@@ -59,6 +59,48 @@ const PLACEHOLDER = /[<>${}|`*\\]/;
 const HELP_FLAGS = new Set(['--help', '-h', '--version', '-V']);
 /** Non-mutating subcommands worth executing even without a help flag. */
 const READ_ONLY_COMMANDS = new Set(['canary skills list']);
+/**
+ * The author's declaration that a block is illustrative (#707).
+ *
+ * Placed on its own line immediately above the fence it governs:
+ *
+ *     <!-- canary:illustrative -->
+ *     ```bash
+ *     canary katana scan --since HEAD~1
+ *     ```
+ *
+ * Two facts land in the same "unverifiable" bucket and they are not the same
+ * fact: "nobody could run this" and "this was never meant to be run". The
+ * first is a gap in the corpus; the second is a deliberate authoring choice.
+ * Collapsing them is what let 88% of the corpus read as coverage debt when
+ * some of it was prose doing its job — and, worse, hid the real gaps inside
+ * the pile.
+ *
+ * Marking is NOT an escape hatch from the executable-example rule. It changes
+ * the reason on one block; a code-bearing skill still has to carry at least
+ * one example that actually runs (`no-executable-example`), so a skill cannot
+ * mark its way to green.
+ */
+const ILLUSTRATIVE_MARKER = /^\s*<!--\s*canary:illustrative\s*-->\s*$/;
+/**
+ * The reason carried by a declared-illustrative example.
+ *
+ * Exported because the summary line splits the unverifiable bucket on it
+ * (see {@link countDeclaredIllustrative}). A string literal compared in two
+ * files is a drift waiting to happen, and the drift would be silent: the
+ * split would quietly read 0 declared and the distinction this issue exists
+ * to draw would be gone with nothing red.
+ */
+export const ILLUSTRATIVE_REASON = 'declared illustrative by the author, so it is not run';
+/**
+ * How many of a gate's skipped examples were skipped BY DECLARATION.
+ *
+ * The rest are the honest gap: examples nobody could run and nobody said
+ * were prose.
+ */
+export function countDeclaredIllustrative(skipped) {
+    return skipped.filter((s) => s.reason === ILLUSTRATIVE_REASON).length;
+}
 /** How an example turned out. */
 export var ExampleVerdict;
 (function (ExampleVerdict) {
@@ -75,6 +117,14 @@ export var ExampleFindingKind;
     ExampleFindingKind["ExampleFailed"] = "example-failed";
     /** A code-bearing skill's doc offers no command to execute at all. */
     ExampleFindingKind["NoDocumentedExample"] = "no-documented-example";
+    /**
+     * A code-bearing skill documents commands, but not one of them can be run
+     * (#707). Distinct from {@link NoDocumentedExample}, and it was the larger
+     * hole: 5 of 9 `cli:` skills sat here while the corpus looked documented.
+     * A skill in this state can break in every documented way and CI stays
+     * green, which is the false-green shape the whole check exists to close.
+     */
+    ExampleFindingKind["NoExecutableExample"] = "no-executable-example";
 })(ExampleFindingKind || (ExampleFindingKind = {}));
 /**
  * Whether `line` closes the currently open fence.
@@ -95,6 +145,11 @@ function fencedShellLines(text) {
     const lines = text.split('\n');
     let fence = null;
     let shell = false;
+    let illustrative = false;
+    // The marker governs the NEXT fence, so it survives the blank line authors
+    // naturally leave between a comment and a block, and is spent by the fence
+    // it opens — a marker cannot leak onto a later, unrelated example.
+    let pendingMarker = false;
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
         const delimiter = /^\s*(`{3,}|~{3,})\s*([A-Za-z0-9_+-]*)/.exec(line);
@@ -104,16 +159,24 @@ function fencedShellLines(text) {
             if (delimiter) {
                 fence = delimiter[1];
                 shell = SHELL_FENCES.has((delimiter[2] ?? '').toLowerCase());
+                illustrative = pendingMarker;
+                pendingMarker = false;
+                continue;
             }
+            if (ILLUSTRATIVE_MARKER.test(line))
+                pendingMarker = true;
+            else if (line.trim() !== '')
+                pendingMarker = false;
             continue;
         }
         if (closesFence(line, delimiter, fence)) {
             fence = null;
             shell = false;
+            illustrative = false;
             continue;
         }
         if (shell)
-            out.push({ line: i + 1, raw: line });
+            out.push({ line: i + 1, raw: line, illustrative });
     }
     return out;
 }
@@ -146,7 +209,7 @@ function classify(command) {
  */
 export function extractExamples(text, skill, path) {
     const out = [];
-    for (const { line, raw } of fencedShellLines(text)) {
+    for (const { line, raw, illustrative } of fencedShellLines(text)) {
         // Strip a `$ ` or `> ` shell prompt; a doc that shows a prompt is still
         // documenting the command after it.
         const command = raw
@@ -157,8 +220,31 @@ export function extractExamples(text, skill, path) {
             continue;
         if (command !== 'canary' && !command.startsWith('canary '))
             continue;
+        // A declaration beats an inference. The author saying "this is prose"
+        // is a better fact than the classifier guessing why it could not run,
+        // and it is the fact a reader of the skipped list needs.
+        if (illustrative) {
+            out.push({
+                skill,
+                path,
+                command,
+                line,
+                executable: false,
+                declaredIllustrative: true,
+                reason: ILLUSTRATIVE_REASON,
+            });
+            continue;
+        }
         const { executable, reason } = classify(command);
-        out.push({ skill, path, command, line, executable, reason });
+        out.push({
+            skill,
+            path,
+            command,
+            line,
+            executable,
+            declaredIllustrative: false,
+            reason,
+        });
     }
     return out;
 }
@@ -239,6 +325,20 @@ export function checkExamples(surfaces, run, cwd) {
             }
             continue;
         }
+        // #707: documenting commands is not the same as documenting a RUNNABLE
+        // one. The cheapest fix is the skill's own `--help`, which needs no
+        // fixtures, credentials or network — and marking blocks illustrative
+        // cannot satisfy this, so the declaration stays honest.
+        if (codeBearing(decl) && !examples.some((e) => e.executable)) {
+            tally.findings.push({
+                kind: ExampleFindingKind.NoExecutableExample,
+                skill: decl.name,
+                path: decl.path,
+                detail: `declares a \`cli:\` and documents ${examples.length} command(s), ` +
+                    'but none is executable, so nothing in its doc has ever been run. ' +
+                    'Add one placeholder-free help-shaped example (its own `--help`).',
+            });
+        }
         tallyDeclaration(decl, runExamples(examples, run, cwd), tally);
     }
     return tally;

package/dist/engine/core/skill-registry.js

@@ -67,11 +67,25 @@ function codePointCompare(a, b) {
     }
     return ca.length - cb.length;
 }
-/** Bundled skills live at `<repo>/agents/skills`. Python: `_AGENTS_SKILLS_DIR`. */
+/**
+ * Bundled skills live at `<root>/agents/skills`, where `<root>` is three
+ * directories above this module. Python: `_AGENTS_SKILLS_DIR`.
+ *
+ * The "three levels up" is a PACKAGING CONTRACT, not an implementation detail
+ * (#757). It holds in the source tree (`ts/src/core`), in the compiled tree
+ * (`ts/dist/core`), and in the published npm package (`dist/engine/core`) --
+ * but only while whatever sits at that root actually ships an `agents/skills`.
+ * It did not: `canary-test-cli@7.1.0` published `bin/` and `dist/` only, so an
+ * installed CLI resolved this to a directory that has never existed and
+ * reported every bundled skill as absent, from any cwd. Exported so
+ * `ts/test/skill-packaging.test.ts` can pin both halves of the contract.
+ */
+export function bundledSkillsDirFrom(moduleDir) {
+    // moduleDir = <root>/<a>/<b>/core -> the root is three levels up.
+    return resolve(moduleDir, '..', '..', '..', 'agents', 'skills');
+}
 function defaultAgentsSkillsDir() {
-    const here = dirname(fileURLToPath(import.meta.url));
-    // here = ts/src/core -> repo root is three levels up (Python: parents[2]).
-    return resolve(here, '..', '..', '..', 'agents', 'skills');
+    return bundledSkillsDirFrom(dirname(fileURLToPath(import.meta.url)));
 }
 /**
  * A discovered skill (Python: `SkillInfo` dataclass).
@@ -150,6 +164,47 @@ export class SkillRegistry {
         }
         return [...skills.values()].sort((a, b) => codePointCompare(a.name, b.name));
     }
+    /**
+     * The directory tiers {@link discover} consults, in precedence order.
+     *
+     * Exists so an empty discovery can state its denominator (#757). "No skills
+     * found." is a claim about the world; what discovery can actually attest is
+     * "none of THESE four roots held one", and the two read very differently to
+     * someone standing in a directory full of SKILL.md files. The local tier
+     * walks cwd up to the git root, so it renders as the range it swept rather
+     * than one line per ancestor.
+     */
+    searchRoots(root) {
+        const searchRoot = resolve(root ?? process.cwd());
+        const ancestors = SkillRegistry.ancestorsToGitRoot(searchRoot);
+        const localDirs = ancestors.map((a) => join(a, '.canary', 'skills'));
+        const overlaysRoot = join(this.home, '.canary', 'overlays');
+        const globalDir = join(this.home, '.canary', 'skills');
+        return [
+            {
+                tier: 'bundled',
+                path: this.agentsSkillsDir,
+                exists: existsSync(this.agentsSkillsDir),
+            },
+            {
+                tier: 'overlay',
+                path: overlaysRoot,
+                exists: SkillRegistry.isDir(overlaysRoot),
+            },
+            {
+                tier: 'global',
+                path: globalDir,
+                exists: SkillRegistry.isDir(globalDir),
+            },
+            {
+                tier: 'local',
+                path: localDirs.length > 1
+                    ? `${localDirs[0]} (and ${localDirs.length - 1} ancestor(s) up to the git root)`
+                    : (localDirs[0] ?? join(searchRoot, '.canary', 'skills')),
+                exists: localDirs.some((d) => existsSync(d)),
+            },
+        ];
+    }
     /** Return the SkillInfo for `name` honoring precedence, or null. */
     find(name, root) {
         for (const skill of this.discover(root)) {

package/dist/engine/core/string-literals.js

@@ -59,7 +59,9 @@ export function blankStringContent(code, options = {}) {
     const spans = literalContentSpans(code, options.python === true);
     if (spans.length === 0)
         return code;
-    const out = [...code];
+    // `split('')`, not `[...code]`: spans are UTF-16 offsets, and spreading by
+    // code point would collapse each surrogate pair and shift every offset (#861).
+    const out = code.split('');
     for (const [start, end] of spans) {
         for (let i = start; i < end; i += 1) {
             // Newlines survive so line numbering is unchanged; everything else goes.

package/dist/engine/core/test-files.js

@@ -0,0 +1,77 @@
+/**
+ * The one answer to "which files in this tree are tests?" (#755).
+ *
+ * Extracted from `cli-commands.ts`, where it was private, because
+ * `canary-cassandra`'s skill CLI needs the SAME answer as `canary
+ * vacuity-check`. The four Tier-0 detectors are meant to be mergeable by a
+ * single consumer, and two collectors disagreeing about the denominator is the
+ * quietest way for that to stop being true: the same run would report a
+ * different `checked` depending on which door it came through.
+ *
+ * The walk's ignore set is load-bearing (#566): a dependency's own test suite
+ * is not the consumer's to fix. One downstream run before that fix produced 254
+ * of 256 findings inside `node_modules`, with the only `critical` in vendored
+ * code.
+ */
+import { readdirSync, statSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { JS_TEST_EXTENSIONS } from './static-linter.js';
+/** True when `p` is a readable directory. A missing path is not one. */
+export function isDir(p) {
+    try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/** Directories never worth walking; see the module docstring for why. */
+const IGNORED_DIRS = new Set([
+    'node_modules',
+    '.git',
+    '__pycache__',
+    '.venv',
+    'venv',
+    'dist',
+    'build',
+    '.next',
+    '.nuxt',
+]);
+function walkFiles(dir) {
+    const out = [];
+    let entries;
+    try {
+        entries = readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return out;
+    }
+    for (const e of entries) {
+        const full = join(dir, e.name);
+        if (e.isDirectory()) {
+            if (!IGNORED_DIRS.has(e.name))
+                out.push(...walkFiles(full));
+        }
+        else if (e.isFile())
+            out.push(full);
+    }
+    return out;
+}
+/**
+ * `test_*.py` plus `*.test.*` / `*.spec.*` over every extension the scanners
+ * can actually read -- `.mjs` and `.cjs` included, which is the half of #566
+ * that made a directory of ESM tests collect zero files.
+ */
+const JS_TEST_FILE_RE = new RegExp(`\\.(test|spec)\\.(${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join('|')})$`);
+/** Recursive test-file glob matching Python's `rglob` union, sorted by path. */
+export function collectTestFiles(dir) {
+    return walkFiles(dir)
+        .filter((p) => {
+        const b = basename(p);
+        return ((b.startsWith('test_') && b.endsWith('.py')) || JS_TEST_FILE_RE.test(b));
+    })
+        .sort();
+}
+/** Human-readable list of what {@link collectTestFiles} looks for. */
+export const SCANNABLE_DESC = `test_*.py, *.test|spec.{${JS_TEST_EXTENSIONS.map((e) => e.slice(1)).join(',')}}`;
+//# sourceMappingURL=test-files.js.map