canary-test-cli 7.0.0 → 7.2.0

package/dist/engine/core/skill-examples.js

@@ -0,0 +1,392 @@
+/**
+ * Execute the commands the docs promise (#487).
+ *
+ * #472 added `canary skills run canary-blackhawk -- --help` to a SKILL.md in
+ * the same PR that left the command broken: the CLI landed at mode 644, the
+ * spawn hit `EACCES`, and that became a bare exit 1 with no output. Nobody had
+ * ever run the command the doc documented. #480 answered part of it with a
+ * hand-written `spawnSync(cli, ['--help'])` per skill — real execution, which
+ * is why the exec-bit bug surfaced at all — but the block is duplicated across
+ * five test files, so a seventh skill is covered only when somebody remembers
+ * to add a sixth copy (#479).
+ *
+ * This module is the discovery-driven form. It consumes the surface inventory
+ * from {@link ./skill-surfaces.js} — deliberately, rather than walking the skill
+ * tree a second time: two walkers with two notions of what a skill is would
+ * disagree eventually, and that disagreement is the very bug class both checks
+ * exist to catch.
+ *
+ * ## Which examples are executable
+ *
+ * #487 left this as an open scope decision. The answer taken here is
+ * conservative and mechanical, because the alternative — a fenced-block
+ * annotation — asks every SKILL.md author to opt in, and an opt-in that is
+ * forgotten reads exactly like a skill with no examples:
+ *
+ *   - The command must live in a **shell-info fenced block** (` ```bash `,
+ *     `sh`, `shell`, `zsh`, `console`). Prose backticks are illustrative.
+ *   - It must be a **`canary` invocation**. Running arbitrary `npm`/`git` lines
+ *     out of a doc is a different and much larger blast radius.
+ *   - It must carry **no placeholder or shell metacharacter** (`<path>`, `$VAR`,
+ *     a pipe, a glob). A placeholder command was never meant to run verbatim.
+ *   - It must be **help-shaped** (`--help` / `-h` / `--version`, or a pure
+ *     listing command). This is what keeps a documented `katana scan` from
+ *     writing a ledger into whatever directory CI happens to be sitting in.
+ *
+ * Everything else is {@link ExampleVerdict.Unverifiable}, **with its reason
+ * recorded**. That is the load-bearing half of the design, and it is the same
+ * distinction `reachability.ts` draws between a dead link and a slow one: an
+ * outcome the checker is not entitled to assert on gets its own status instead
+ * of being folded into either pass or fail.
+ *
+ * ## Denominator
+ *
+ * `checked` counts the examples **actually executed** — never the examples
+ * found. Unverifiable examples travel in `GateResult.skipped`, so `gateOutcome`
+ * renders them in every summary line (D7) and an all-illustrative corpus
+ * ABSTAINS rather than reporting "all 0 examples passed" (#508).
+ */
+import { spawnSync } from 'node:child_process';
+import { SurfaceKind } from './skill-surfaces.js';
+/** Fence info strings whose contents are shell commands. */
+const SHELL_FENCES = new Set(['bash', 'sh', 'shell', 'zsh', 'console']);
+/**
+ * Characters that make a command line unsafe to run verbatim: placeholder
+ * brackets, variable expansion, redirection, pipes, subshells, globs.
+ */
+const PLACEHOLDER = /[<>${}|`*\\]/;
+/** Flags that make an invocation a pure read of the CLI's own surface. */
+const HELP_FLAGS = new Set(['--help', '-h', '--version', '-V']);
+/** Non-mutating subcommands worth executing even without a help flag. */
+const READ_ONLY_COMMANDS = new Set(['canary skills list']);
+/**
+ * The author's declaration that a block is illustrative (#707).
+ *
+ * Placed on its own line immediately above the fence it governs:
+ *
+ *     <!-- canary:illustrative -->
+ *     ```bash
+ *     canary katana scan --since HEAD~1
+ *     ```
+ *
+ * Two facts land in the same "unverifiable" bucket and they are not the same
+ * fact: "nobody could run this" and "this was never meant to be run". The
+ * first is a gap in the corpus; the second is a deliberate authoring choice.
+ * Collapsing them is what let 88% of the corpus read as coverage debt when
+ * some of it was prose doing its job — and, worse, hid the real gaps inside
+ * the pile.
+ *
+ * Marking is NOT an escape hatch from the executable-example rule. It changes
+ * the reason on one block; a code-bearing skill still has to carry at least
+ * one example that actually runs (`no-executable-example`), so a skill cannot
+ * mark its way to green.
+ */
+const ILLUSTRATIVE_MARKER = /^\s*<!--\s*canary:illustrative\s*-->\s*$/;
+/**
+ * The reason carried by a declared-illustrative example.
+ *
+ * Exported because the summary line splits the unverifiable bucket on it
+ * (see {@link countDeclaredIllustrative}). A string literal compared in two
+ * files is a drift waiting to happen, and the drift would be silent: the
+ * split would quietly read 0 declared and the distinction this issue exists
+ * to draw would be gone with nothing red.
+ */
+export const ILLUSTRATIVE_REASON = 'declared illustrative by the author, so it is not run';
+/**
+ * How many of a gate's skipped examples were skipped BY DECLARATION.
+ *
+ * The rest are the honest gap: examples nobody could run and nobody said
+ * were prose.
+ */
+export function countDeclaredIllustrative(skipped) {
+    return skipped.filter((s) => s.reason === ILLUSTRATIVE_REASON).length;
+}
+/** How an example turned out. */
+export var ExampleVerdict;
+(function (ExampleVerdict) {
+    /** Ran and exited 0 — the doc is proven. */
+    ExampleVerdict["Executed"] = "executed";
+    /** Ran and did not exit 0 — the doc promises something broken. */
+    ExampleVerdict["Failed"] = "failed";
+    /** Could not be run at all; the reason travels with it. */
+    ExampleVerdict["Unverifiable"] = "unverifiable";
+})(ExampleVerdict || (ExampleVerdict = {}));
+export var ExampleFindingKind;
+(function (ExampleFindingKind) {
+    /** A documented command was executed and failed. */
+    ExampleFindingKind["ExampleFailed"] = "example-failed";
+    /** A code-bearing skill's doc offers no command to execute at all. */
+    ExampleFindingKind["NoDocumentedExample"] = "no-documented-example";
+    /**
+     * A code-bearing skill documents commands, but not one of them can be run
+     * (#707). Distinct from {@link NoDocumentedExample}, and it was the larger
+     * hole: 5 of 9 `cli:` skills sat here while the corpus looked documented.
+     * A skill in this state can break in every documented way and CI stays
+     * green, which is the false-green shape the whole check exists to close.
+     */
+    ExampleFindingKind["NoExecutableExample"] = "no-executable-example";
+})(ExampleFindingKind || (ExampleFindingKind = {}));
+/**
+ * Whether `line` closes the currently open fence.
+ *
+ * A fence closes only on a BARE delimiter run at least as long as the opener,
+ * so a ```` block may legitimately contain ``` -- the same fence rule
+ * `scripts/check_doc_links.mjs` had to get right for #686.
+ */
+function closesFence(line, delimiter, fence) {
+    if (delimiter === null)
+        return false;
+    const run = delimiter[1];
+    return (run[0] === fence[0] && run.length >= fence.length && line.trim() === run);
+}
+/** Shell-fenced lines with their 1-based source line numbers. */
+function fencedShellLines(text) {
+    const out = [];
+    const lines = text.split('\n');
+    let fence = null;
+    let shell = false;
+    let illustrative = false;
+    // The marker governs the NEXT fence, so it survives the blank line authors
+    // naturally leave between a comment and a block, and is spent by the fence
+    // it opens — a marker cannot leak onto a later, unrelated example.
+    let pendingMarker = false;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const delimiter = /^\s*(`{3,}|~{3,})\s*([A-Za-z0-9_+-]*)/.exec(line);
+        if (fence === null) {
+            // A fence opens on a delimiter run; the info string decides whether the
+            // body is shell. Anything else (json, ts, text) is not a command.
+            if (delimiter) {
+                fence = delimiter[1];
+                shell = SHELL_FENCES.has((delimiter[2] ?? '').toLowerCase());
+                illustrative = pendingMarker;
+                pendingMarker = false;
+                continue;
+            }
+            if (ILLUSTRATIVE_MARKER.test(line))
+                pendingMarker = true;
+            else if (line.trim() !== '')
+                pendingMarker = false;
+            continue;
+        }
+        if (closesFence(line, delimiter, fence)) {
+            fence = null;
+            shell = false;
+            illustrative = false;
+            continue;
+        }
+        if (shell)
+            out.push({ line: i + 1, raw: line, illustrative });
+    }
+    return out;
+}
+/** Classify one command line: executable, or unverifiable with a reason. */
+function classify(command) {
+    if (PLACEHOLDER.test(command)) {
+        return {
+            executable: false,
+            reason: 'contains a placeholder or shell metacharacter, so it was never ' +
+                'meant to run verbatim',
+        };
+    }
+    if (READ_ONLY_COMMANDS.has(command))
+        return { executable: true, reason: null };
+    const tokens = command.split(/\s+/);
+    if (tokens.some((t) => HELP_FLAGS.has(t))) {
+        return { executable: true, reason: null };
+    }
+    return {
+        executable: false,
+        reason: 'not help-shaped, so running it could write files, need credentials, ' +
+            'or reach the network',
+    };
+}
+/**
+ * Extract the documented `canary` commands from one document.
+ *
+ * `text` is the raw file body (the inventory already read it), `skill` and
+ * `path` are carried through onto each example so a finding is attributable.
+ */
+export function extractExamples(text, skill, path) {
+    const out = [];
+    for (const { line, raw, illustrative } of fencedShellLines(text)) {
+        // Strip a `$ ` or `> ` shell prompt; a doc that shows a prompt is still
+        // documenting the command after it.
+        const command = raw
+            .trim()
+            .replace(/^[$>]\s+/, '')
+            .trim();
+        if (command === '' || command.startsWith('#'))
+            continue;
+        if (command !== 'canary' && !command.startsWith('canary '))
+            continue;
+        // A declaration beats an inference. The author saying "this is prose"
+        // is a better fact than the classifier guessing why it could not run,
+        // and it is the fact a reader of the skipped list needs.
+        if (illustrative) {
+            out.push({
+                skill,
+                path,
+                command,
+                line,
+                executable: false,
+                declaredIllustrative: true,
+                reason: ILLUSTRATIVE_REASON,
+            });
+            continue;
+        }
+        const { executable, reason } = classify(command);
+        out.push({
+            skill,
+            path,
+            command,
+            line,
+            executable,
+            declaredIllustrative: false,
+            reason,
+        });
+    }
+    return out;
+}
+/** Execute the executable examples; report a verdict for every example. */
+export function runExamples(examples, run, cwd) {
+    return examples.map((example) => {
+        if (!example.executable) {
+            return {
+                example,
+                verdict: ExampleVerdict.Unverifiable,
+                detail: example.reason ?? 'unverifiable',
+            };
+        }
+        const { status, output } = run(example.command, cwd);
+        if (status === 0) {
+            return { example, verdict: ExampleVerdict.Executed, detail: 'exit 0' };
+        }
+        return {
+            example,
+            verdict: ExampleVerdict.Failed,
+            detail: status === null
+                ? `the process never started: ${output.trim() || '(no output)'}`
+                : `exit ${status}: ${output.trim() || '(no output)'}`,
+        };
+    });
+}
+/** A skill declaration that ships code, and can therefore ship broken code. */
+function codeBearing(decl) {
+    return ((decl.kind === SurfaceKind.Skill || decl.kind === SurfaceKind.FlatSkill) &&
+        decl.cli !== null);
+}
+/** Fold one skill declaration's results into `tally`. */
+function tallyDeclaration(decl, results, tally) {
+    for (const result of results) {
+        // `<skill>:<line>` rather than the absolute path: a skill's SKILL.md is
+        // unambiguous from its name, and 29 absolute paths turned the D7 skip
+        // suffix into a summary line no reader would finish. Still fully
+        // attributable; `--json` carries the paths.
+        const where = `${decl.name}:${result.example.line}`;
+        if (result.verdict === ExampleVerdict.Unverifiable) {
+            // Never silently dropped: a skip renders in the summary line, so an
+            // example nobody can run stays visible instead of leaving the corpus.
+            tally.skipped.push({ name: where, reason: result.detail });
+            continue;
+        }
+        tally.checked += 1;
+        if (result.verdict === ExampleVerdict.Failed) {
+            tally.findings.push({
+                kind: ExampleFindingKind.ExampleFailed,
+                skill: decl.name,
+                path: decl.path,
+                detail: `\`${result.example.command}\` (line ${result.example.line}) ${result.detail}`,
+            });
+        }
+    }
+}
+export function checkExamples(surfaces, run, cwd) {
+    const tally = { checked: 0, findings: [], skipped: [] };
+    for (const decl of surfaces) {
+        if (decl.kind !== SurfaceKind.Skill &&
+            decl.kind !== SurfaceKind.FlatSkill) {
+            continue;
+        }
+        const examples = extractExamples(decl.text, decl.name, decl.path);
+        // #487 acceptance: a code-bearing skill with no runnable command in its
+        // doc is UNPROVEN, not clean. A markdown-only skill has no command that a
+        // mode bit could break, so it is not held to this.
+        if (examples.length === 0) {
+            if (codeBearing(decl)) {
+                tally.findings.push({
+                    kind: ExampleFindingKind.NoDocumentedExample,
+                    skill: decl.name,
+                    path: decl.path,
+                    detail: 'declares a `cli:` but its SKILL.md documents no command in a ' +
+                        'shell fence, so nothing about it has ever been executed from ' +
+                        'the doc',
+                });
+            }
+            continue;
+        }
+        // #707: documenting commands is not the same as documenting a RUNNABLE
+        // one. The cheapest fix is the skill's own `--help`, which needs no
+        // fixtures, credentials or network — and marking blocks illustrative
+        // cannot satisfy this, so the declaration stays honest.
+        if (codeBearing(decl) && !examples.some((e) => e.executable)) {
+            tally.findings.push({
+                kind: ExampleFindingKind.NoExecutableExample,
+                skill: decl.name,
+                path: decl.path,
+                detail: `declares a \`cli:\` and documents ${examples.length} command(s), ` +
+                    'but none is executable, so nothing in its doc has ever been run. ' +
+                    'Add one placeholder-free help-shaped example (its own `--help`).',
+            });
+        }
+        tallyDeclaration(decl, runExamples(examples, run, cwd), tally);
+    }
+    return tally;
+}
+/**
+ * The production runner: spawn the documented command against the repo's own
+ * built CLI, in `cwd`.
+ *
+ * Two deliberate substitutions, both narrow:
+ *
+ *   - The leading `canary` token becomes `node <canaryBin>`, because a doc
+ *     writes the installed name and CI has a checkout.
+ *   - `--allow-executable-skills` is inserted into a `skills run` invocation,
+ *     ahead of any `--` separator. `isExecutableSkillAllowed` refuses `cli:`
+ *     skills without a TTY, which a spawned process never has, so without the
+ *     flag every example would exit 3 and the check would measure the sandbox
+ *     rather than the doc. The flag is an execution-context opt-in and changes
+ *     nothing about the command's behaviour once it runs. See
+ *     {@link exampleArgv} for why the position matters.
+ */
+export function exampleArgv(command, canaryBin) {
+    const [, ...rest] = command.split(/\s+/);
+    if (rest[0] !== 'skills' || rest[1] !== 'run')
+        return [canaryBin, ...rest];
+    // The flag must land BEFORE `--`, or canary forwards it to the skill and the
+    // executable-skill guard still refuses. Appending it was the first bug this
+    // checker found, in itself: all four documented `skills run ... -- --help`
+    // examples reported exit 3, which measured the sandbox rather than the doc.
+    const sep = rest.indexOf('--');
+    const at = sep === -1 ? rest.length : sep;
+    return [
+        canaryBin,
+        ...rest.slice(0, at),
+        '--allow-executable-skills',
+        ...rest.slice(at),
+    ];
+}
+export function spawnRunner(canaryBin) {
+    return (command, cwd) => {
+        const res = spawnSync(process.execPath, exampleArgv(command, canaryBin), {
+            cwd,
+            encoding: 'utf-8',
+            timeout: 60_000,
+        });
+        return {
+            status: res.status,
+            output: `${res.stdout ?? ''}${res.stderr ?? ''}`,
+        };
+    };
+}
+//# sourceMappingURL=skill-examples.js.map

package/dist/engine/core/skill-registry.js

@@ -67,11 +67,25 @@ function codePointCompare(a, b) {
     }
     return ca.length - cb.length;
 }
-/** Bundled skills live at `<repo>/agents/skills`. Python: `_AGENTS_SKILLS_DIR`. */
+/**
+ * Bundled skills live at `<root>/agents/skills`, where `<root>` is three
+ * directories above this module. Python: `_AGENTS_SKILLS_DIR`.
+ *
+ * The "three levels up" is a PACKAGING CONTRACT, not an implementation detail
+ * (#757). It holds in the source tree (`ts/src/core`), in the compiled tree
+ * (`ts/dist/core`), and in the published npm package (`dist/engine/core`) --
+ * but only while whatever sits at that root actually ships an `agents/skills`.
+ * It did not: `canary-test-cli@7.1.0` published `bin/` and `dist/` only, so an
+ * installed CLI resolved this to a directory that has never existed and
+ * reported every bundled skill as absent, from any cwd. Exported so
+ * `ts/test/skill-packaging.test.ts` can pin both halves of the contract.
+ */
+export function bundledSkillsDirFrom(moduleDir) {
+    // moduleDir = <root>/<a>/<b>/core -> the root is three levels up.
+    return resolve(moduleDir, '..', '..', '..', 'agents', 'skills');
+}
 function defaultAgentsSkillsDir() {
-    const here = dirname(fileURLToPath(import.meta.url));
-    // here = ts/src/core -> repo root is three levels up (Python: parents[2]).
-    return resolve(here, '..', '..', '..', 'agents', 'skills');
+    return bundledSkillsDirFrom(dirname(fileURLToPath(import.meta.url)));
 }
 /**
  * A discovered skill (Python: `SkillInfo` dataclass).
@@ -150,6 +164,47 @@ export class SkillRegistry {
         }
         return [...skills.values()].sort((a, b) => codePointCompare(a.name, b.name));
     }
+    /**
+     * The directory tiers {@link discover} consults, in precedence order.
+     *
+     * Exists so an empty discovery can state its denominator (#757). "No skills
+     * found." is a claim about the world; what discovery can actually attest is
+     * "none of THESE four roots held one", and the two read very differently to
+     * someone standing in a directory full of SKILL.md files. The local tier
+     * walks cwd up to the git root, so it renders as the range it swept rather
+     * than one line per ancestor.
+     */
+    searchRoots(root) {
+        const searchRoot = resolve(root ?? process.cwd());
+        const ancestors = SkillRegistry.ancestorsToGitRoot(searchRoot);
+        const localDirs = ancestors.map((a) => join(a, '.canary', 'skills'));
+        const overlaysRoot = join(this.home, '.canary', 'overlays');
+        const globalDir = join(this.home, '.canary', 'skills');
+        return [
+            {
+                tier: 'bundled',
+                path: this.agentsSkillsDir,
+                exists: existsSync(this.agentsSkillsDir),
+            },
+            {
+                tier: 'overlay',
+                path: overlaysRoot,
+                exists: SkillRegistry.isDir(overlaysRoot),
+            },
+            {
+                tier: 'global',
+                path: globalDir,
+                exists: SkillRegistry.isDir(globalDir),
+            },
+            {
+                tier: 'local',
+                path: localDirs.length > 1
+                    ? `${localDirs[0]} (and ${localDirs.length - 1} ancestor(s) up to the git root)`
+                    : (localDirs[0] ?? join(searchRoot, '.canary', 'skills')),
+                exists: localDirs.some((d) => existsSync(d)),
+            },
+        ];
+    }
     /** Return the SkillInfo for `name` honoring precedence, or null. */
     find(name, root) {
         for (const skill of this.discover(root)) {