canary-test-cli 7.1.0 → 8.0.0

package/dist/engine/skills-cli.js

@@ -16,9 +16,10 @@ import { Command } from 'commander';
 import pc from 'picocolors';
 import { CliExitError, jsonIndent2, normalizeUsageExit } from './cli-common.js';
 import { gateOutcome } from './core/gate-result.js';
-import { checkExamples, spawnRunner, } from './core/skill-examples.js';
+import { checkExamples, countDeclaredIllustrative, spawnRunner, } from './core/skill-examples.js';
 import { SurfaceFindingKind, checkSurfaces, collectSurfaces, } from './core/skill-surfaces.js';
 import { isExecutableSkillAllowed, resolveCliPath, } from './core/skill-registry.js';
+import { SkillDispatchError, dispatchProseSkill, } from './core/skill-dispatch.js';
 import { CROSS, EM_DASH } from './main-deps.js';
 function overlayName(skill) {
     // Clone layout: ~/.canary/overlays/<overlay>/.canary/skills/<name>/SKILL.md
@@ -42,51 +43,145 @@ function formatSkill(skill, verbose) {
         line += `\n    ${pc.dim(skill.path)}`;
     return line;
 }
+/**
+ * An empty discovery, reported as the abstention it is (#757).
+ *
+ * `No skills found.` was a claim about the world made by a check that had
+ * verified nothing. All discovery can attest is that four specific roots held
+ * nothing -- so it says which four, and whether each one even existed. The
+ * distinction is what separates "this project has no skills" from "the tree I
+ * was pointed at is not there", and an installed CLI that shipped no bundled
+ * skills at all printed the first while meaning the second.
+ */
+function reportEmptyDiscovery(registry, deps) {
+    const roots = registry.searchRoots();
+    const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory', {
+        noun: 'skill(s)',
+    });
+    deps.out(pc.bold(pc.yellow(outcome.summaryLine)));
+    deps.out('No skill was discoverable. Searched:');
+    for (const root of roots) {
+        const state = root.exists ? 'present, no skill inside' : 'does not exist';
+        deps.out(`  ${root.tier.padEnd(8)} ${root.path} ${pc.dim(`(${state})`)}`);
+    }
+    const bundled = roots.find((r) => r.tier === 'bundled');
+    if (bundled && !bundled.exists) {
+        deps.out(`${pc.yellow(CROSS)} The bundled-skill root is missing from this ` +
+            `install ${EM_DASH} the CLI cannot see its own skills, so no cwd will ` +
+            'make them appear. Reinstall, or run from a Canary checkout.');
+    }
+}
+/**
+ * Print one source tier: a separating blank line only when an earlier tier
+ * printed, then the header, then its skills. Split out of `listCmd` to pay
+ * down its complexity; every helper here stays at or under the perf-ratchet
+ * warning threshold (10) so the paydown adds no new finding identity.
+ */
+function printTier(deps, header, skills, verbose, printedAbove) {
+    if (printedAbove)
+        deps.out('');
+    deps.out(header);
+    for (const skill of skills)
+        deps.out(formatSkill(skill, verbose));
+}
+function byOverlayName(a, b) {
+    return overlayName(a).localeCompare(overlayName(b));
+}
+/** Overlay skills print one header per overlay, groups sorted by name. */
+function printOverlayGroups(deps, overlay, verbose, printedAbove) {
+    let above = printedAbove;
+    let cur = null;
+    for (const skill of [...overlay].sort(byOverlayName)) {
+        const oname = overlayName(skill);
+        if (oname !== cur) {
+            if (above)
+                deps.out('');
+            deps.out(`${pc.bold('Overlay skills')} ${pc.dim(`(${oname} ${EM_DASH} override bundled):`)}`);
+            cur = oname;
+            above = true;
+        }
+        deps.out(formatSkill(skill, verbose));
+    }
+}
 function listCmd(opts, deps) {
-    const verbose = opts.verbose ?? false;
-    const skills = deps.makeSkillRegistry().discover();
+    const verbose = opts.verbose === true;
+    const registry = deps.makeSkillRegistry();
+    const skills = registry.discover();
     if (skills.length === 0) {
-        deps.out(pc.yellow('No skills found.'));
+        reportEmptyDiscovery(registry, deps);
         return;
     }
-    const bundled = skills.filter((s) => s.source === 'bundled');
-    const overlay = skills.filter((s) => s.source === 'overlay');
-    const globalSkills = skills.filter((s) => s.source === 'global');
-    const local = skills.filter((s) => s.source === 'local');
+    const bySource = (source) => skills.filter((s) => s.source === source);
+    const bundled = bySource('bundled');
+    const overlay = bySource('overlay');
+    const globalSkills = bySource('global');
+    const local = bySource('local');
+    let printed = false;
     if (bundled.length) {
-        deps.out(pc.bold('Bundled skills:'));
-        for (const skill of bundled)
-            deps.out(formatSkill(skill, verbose));
+        printTier(deps, pc.bold('Bundled skills:'), bundled, verbose, printed);
+        printed = true;
     }
     if (overlay.length) {
-        const sorted = [...overlay].sort((a, b) => overlayName(a).localeCompare(overlayName(b)));
-        let idx = 0;
-        let cur = null;
-        for (const skill of sorted) {
-            const oname = overlayName(skill);
-            if (oname !== cur) {
-                if (bundled.length || idx > 0)
-                    deps.out('');
-                deps.out(`${pc.bold('Overlay skills')} ${pc.dim(`(${oname} ${EM_DASH} override bundled):`)}`);
-                cur = oname;
-                idx += 1;
-            }
-            deps.out(formatSkill(skill, verbose));
-        }
+        printOverlayGroups(deps, overlay, verbose, printed);
+        printed = true;
     }
     if (globalSkills.length) {
-        if (bundled.length || overlay.length)
-            deps.out('');
-        deps.out(`${pc.bold('Global skills')} ${pc.dim(`(~/.canary/skills/ ${EM_DASH} override overlay):`)}`);
-        for (const skill of globalSkills)
-            deps.out(formatSkill(skill, verbose));
+        const header = `${pc.bold('Global skills')} ${pc.dim(`(~/.canary/skills/ ${EM_DASH} override overlay):`)}`;
+        printTier(deps, header, globalSkills, verbose, printed);
+        printed = true;
     }
     if (local.length) {
-        if (bundled.length || overlay.length || globalSkills.length)
-            deps.out('');
-        deps.out(`${pc.bold('Local overlay skills')} ${pc.dim('(override global):')}`);
-        for (const skill of local)
-            deps.out(formatSkill(skill, verbose));
+        const header = `${pc.bold('Local overlay skills')} ${pc.dim('(override global):')}`;
+        printTier(deps, header, local, verbose, printed);
+    }
+}
+/**
+ * Tier 2: render a dispatched prose skill (#756).
+ *
+ * The determinism label leads, in both modes. A consumer merging this with a
+ * `cli:` detector's findings has to be able to see that an agent produced it;
+ * a payload that only carried the workflow text would read exactly like a
+ * deterministic run's output.
+ */
+function renderDispatch(dispatch, json, deps) {
+    if (json) {
+        deps.out(jsonIndent2(dispatch));
+        return;
+    }
+    deps.out(`${pc.bold(dispatch.skill)} ${EM_DASH} dispatched as a prose skill ` +
+        `${pc.dim('(no cli:/entry: target)')}`);
+    deps.out(`${pc.yellow('determinism:')} ${pc.bold('agent-applied')} ${EM_DASH} canary ` +
+        'applied no judgment here. The workflow below must be executed by an ' +
+        'agent runtime, and its results must not be merged with a ' +
+        "deterministic detector's findings without carrying this label.");
+    if (dispatch.requires.length) {
+        deps.out(`${pc.dim('requires:')} ${dispatch.requires.join(', ')}`);
+    }
+    if (dispatch.args.length) {
+        deps.out(`${pc.dim('args (forwarded, uninterpreted):')} ${dispatch.args.join(' ')}`);
+    }
+    deps.out(`${pc.dim(`source: ${dispatch.path}`)}\n`);
+    deps.out(dispatch.instructions);
+}
+/**
+ * Dispatch a skill with no `cli:`/`entry:` target instead of refusing it.
+ *
+ * Deliberately reached BEFORE the `--allow-executable-skills` gate: that flag
+ * guards spawning someone else's code, and dispatch spawns nothing. Gating it
+ * would leave the 14 CLI-less skills unreachable in precisely the CI contexts
+ * #756 is about. See `core/skill-dispatch.ts` for the full reasoning.
+ */
+function runProseSkill(skill, args, opts, deps) {
+    try {
+        renderDispatch(dispatchProseSkill(skill, args), opts.json ?? false, deps);
+    }
+    catch (exc) {
+        if (!(exc instanceof SkillDispatchError))
+            throw exc;
+        // A dispatch that could not happen is reported as a failure, never as an
+        // empty run: exit 2 keeps the old "this skill is not runnable" contract.
+        deps.out(`${pc.red(CROSS)} Skill ${pc.bold(skill.name)}: ${exc.message}`);
+        throw new CliExitError(2);
     }
 }
 async function runCmd(name, args, opts, deps) {
@@ -99,9 +194,10 @@ async function runCmd(name, args, opts, deps) {
         deps.out(`${pc.red(CROSS)} Skill ${pc.bold(name)}: ${skill.error}`);
         throw new CliExitError(2);
     }
+    // Tier 2 (#756): a skill with no cli:/entry: is dispatched, not refused.
     if (!skill.isExecutable) {
-        deps.out(pc.yellow(`Skill ${pc.bold(name)} is markdown-only ${EM_DASH} no cli: or entry: field to run.`));
-        throw new CliExitError(2);
+        runProseSkill(skill, args, opts, deps);
+        return;
     }
     if (!isExecutableSkillAllowed(opts.allowExecutableSkills ?? false)) {
         deps.out(`${pc.red(CROSS)} Refusing to invoke executable skill in non-interactive context. Pass ${pc.bold('--allow-executable-skills')} to opt in (e.g. in trusted CI configurations).`);
@@ -215,6 +311,11 @@ function verifyJson(root, surfaceResult, exampleResult) {
             abstained: !(exampleResult.checked > 0),
             findings: exampleResult.findings,
             skipped: exampleResult.skipped ?? [],
+            // The skipped bucket, split (#707). Emitted rather than left for the
+            // consumer to derive: a CI step counting declared-illustrative examples
+            // by matching the reason string would be a second copy of a literal
+            // owned here, and it would drift to 0 silently.
+            illustrative: countDeclaredIllustrative(exampleResult.skipped ?? []),
         },
     });
 }
@@ -239,6 +340,15 @@ function verifyReport(root, surfaceResult, exampleResult, deps) {
         noun: 'documented example(s)',
     });
     deps.out(`examples: ${exampleOutcome.summaryLine}`);
+    // #707: the denominator, split. "29 unverifiable" reads as one fact and is
+    // two — blocks nobody could run, and blocks the author declared as prose.
+    // Only the first is a gap, and printing the total alone buries it.
+    const skipped = exampleResult.skipped ?? [];
+    const declared = countDeclaredIllustrative(skipped);
+    deps.out(`  denominator: executed=${exampleResult.checked} ` +
+        `illustrative=${declared} ` +
+        `unverifiable=${skipped.length - declared} ` +
+        `total=${exampleResult.checked + skipped.length}`);
     if (exampleOutcome.abstained) {
         deps.out(`  zero documented commands were executed ${EM_DASH} nothing here is ` +
             'proven. Add a help-shaped example in a shell fence to a SKILL.md, or ' +
@@ -263,28 +373,22 @@ function partition(s, sep) {
         return [s, '', ''];
     return [s.slice(0, i), sep, s.slice(i + sep.length)];
 }
-/** Build the `skills` sub-app wired to `deps`. */
-export function buildSkillsCommand(deps) {
-    const program = new Command('skills');
-    program
-        .description('List and invoke discoverable Canary skills.')
-        .exitOverride(normalizeUsageExit);
-    program
-        .command('list')
-        .description('List every skill discoverable from the current directory.')
-        .option('-v, --verbose', 'Also print the SKILL.md path for each skill.')
-        .action((opts) => {
-        listCmd(opts, deps);
-    });
+/** Register `skills run` -- the tier-1 target path plus the tier-2 dispatcher. */
+function registerRun(program, deps) {
     program
         .command('run')
-        .description("Invoke a code-bearing skill's declared cli or entry target.")
+        .description("Invoke a skill: a code-bearing skill's cli/entry target, or a prose " +
+        "skill's workflow via the dispatcher.")
         .argument('<name>', 'Name of the skill to invoke.')
         .argument('[args...]', "Arguments forwarded to the skill's cli/entry.")
         .option('--allow-executable-skills', 'Opt-in to invoking cli:/entry: skills in non-interactive (CI) contexts.')
+        .option('--json', 'For a dispatched prose skill, emit the labelled payload as JSON.')
         .action(async (name, args, opts) => {
         await runCmd(name, args ?? [], opts, deps);
     });
+}
+/** Register `skills verify` -- the two-denominator cross-surface check. */
+function registerVerify(program, deps) {
     program
         .command('verify')
         .description('Check that skill surfaces agree and that documented examples still run.')
@@ -295,6 +399,22 @@ export function buildSkillsCommand(deps) {
         .action((opts) => {
         verifyCmd(opts, deps);
     });
+}
+/** Build the `skills` sub-app wired to `deps`. */
+export function buildSkillsCommand(deps) {
+    const program = new Command('skills');
+    program
+        .description('List and invoke discoverable Canary skills.')
+        .exitOverride(normalizeUsageExit);
+    program
+        .command('list')
+        .description('List every skill discoverable from the current directory.')
+        .option('-v, --verbose', 'Also print the SKILL.md path for each skill.')
+        .action((opts) => {
+        listCmd(opts, deps);
+    });
+    registerRun(program, deps);
+    registerVerify(program, deps);
     for (const sub of program.commands) {
         sub.exitOverride(normalizeUsageExit);
     }

package/dist/engine/workflow-cli.js

@@ -79,29 +79,94 @@ async function discoverCmd(opts, deps) {
         throw new CliExitError(1);
     }
 }
-function showCmd(opts, deps) {
-    const wd = deps.makeWorkflowDiscovery();
+/**
+ * Which project keys `show` should report on.
+ *
+ * `--project` names one; otherwise every cached `.canary/workflow-*.json` is
+ * listed. An unreadable `.canary` yields no keys rather than throwing: a
+ * missing cache is the ordinary first-run state, not an error.
+ */
+function resolveProjectKeys(opts, deps) {
+    if (opts.project)
+        return [opts.project];
     let keys = [];
-    if (opts.project) {
-        keys = [opts.project];
+    const canaryDir = join(deps.cwd(), '.canary');
+    if (existsSync(canaryDir)) {
+        try {
+            keys = readdirSync(canaryDir)
+                .filter((f) => f.startsWith('workflow-') && f.endsWith('.json'))
+                .map((f) => f.slice('workflow-'.length, -'.json'.length));
+        }
+        catch {
+            keys = [];
+        }
     }
-    else {
-        const canaryDir = join(deps.cwd(), '.canary');
-        if (existsSync(canaryDir)) {
-            try {
-                keys = readdirSync(canaryDir)
-                    .filter((f) => f.startsWith('workflow-') && f.endsWith('.json'))
-                    .map((f) => f.slice('workflow-'.length, -'.json'.length));
-            }
-            catch {
-                keys = [];
-            }
+    if (keys.length === 0) {
+        deps.out(pc.yellow('No cached workflow mappings found.'));
+        throw new CliExitError(0);
+    }
+    return keys;
+}
+/** One mapping as JSON: the whole document, or just its semantic roles. */
+function mappingAsJson(mapping, rolesOnly) {
+    if (!rolesOnly)
+        return mapping.toJson();
+    const rolesDict = {};
+    for (const [r, sr] of Object.entries(mapping.semantic_roles)) {
+        rolesDict[r] = { status_name: sr.status_name, issue_type: sr.issue_type };
+    }
+    return jsonIndent2(rolesDict);
+}
+/** `role -> status` lines, indented under a heading. Shared by both modes. */
+function printSemanticRoles(roles, lead, deps) {
+    deps.out(`${lead}${pc.bold('Semantic roles:')}`);
+    for (const [role, sr] of Object.entries(roles)) {
+        deps.out(`    ${role.padEnd(20)} ${'\u{2192}'} '${sr.status_name}'  ${pc.dim(`(${sr.issue_type})`)}`);
+    }
+}
+/** Each issue type, its statuses, and its transitions. */
+function printIssueTypes(issueTypes, deps) {
+    for (const it of issueTypes) {
+        deps.out(`\n  ${pc.bold(it.name)}`);
+        for (const s of it.statuses) {
+            deps.out(`    [${s.category}] ${s.name}`);
         }
-        if (keys.length === 0) {
-            deps.out(pc.yellow('No cached workflow mappings found.'));
-            throw new CliExitError(0);
+        if (it.transitions.length === 0)
+            continue;
+        deps.out('    Transitions:');
+        for (const t of it.transitions) {
+            deps.out(`      ${t.from_status} ${'\u{2192}'} ${t.to_status}  ${pc.dim(`(${t.name})`)}`);
         }
     }
+}
+/**
+ * One cached mapping, rendered for a human.
+ *
+ * Extracted from `showCmd`, which had grown to hold key resolution, JSON
+ * emission and this rendering at once -- and carried the semantic-roles block
+ * TWICE, once per mode, which is why the two had already drifted in their
+ * indentation.
+ */
+function printMapping(key, mapping, rolesOnly, deps) {
+    const confirmedTag = mapping.role_annotations_confirmed
+        ? pc.green('confirmed')
+        : pc.yellow('unconfirmed');
+    deps.out(`\n${pc.bold(key)}  ${pc.dim(`source=${mapping.source}  discovered=${mapping.discovered_at}  roles=${confirmedTag}`)}`);
+    const hasRoles = Object.keys(mapping.semantic_roles).length > 0;
+    if (rolesOnly) {
+        if (hasRoles)
+            printSemanticRoles(mapping.semantic_roles, '  ', deps);
+        else
+            deps.out(`  ${pc.yellow('No semantic roles resolved yet.')}`);
+        return;
+    }
+    printIssueTypes(mapping.issue_types, deps);
+    if (hasRoles)
+        printSemanticRoles(mapping.semantic_roles, '\n  ', deps);
+}
+function showCmd(opts, deps) {
+    const wd = deps.makeWorkflowDiscovery();
+    const keys = resolveProjectKeys(opts, deps);
     let anyFound = false;
     for (const key of keys) {
         const mapping = wd.show(key);
@@ -111,55 +176,10 @@ function showCmd(opts, deps) {
         }
         anyFound = true;
         if (opts.json) {
-            if (opts.rolesOnly) {
-                const rolesDict = {};
-                for (const [r, sr] of Object.entries(mapping.semantic_roles)) {
-                    rolesDict[r] = {
-                        status_name: sr.status_name,
-                        issue_type: sr.issue_type,
-                    };
-                }
-                deps.out(jsonIndent2(rolesDict));
-            }
-            else {
-                deps.out(mapping.toJson());
-            }
-            continue;
-        }
-        const confirmedTag = mapping.role_annotations_confirmed
-            ? pc.green('confirmed')
-            : pc.yellow('unconfirmed');
-        deps.out(`\n${pc.bold(key)}  ${pc.dim(`source=${mapping.source}  discovered=${mapping.discovered_at}  roles=${confirmedTag}`)}`);
-        if (opts.rolesOnly) {
-            if (Object.keys(mapping.semantic_roles).length) {
-                deps.out(`  ${pc.bold('Semantic roles:')}`);
-                for (const [role, sr] of Object.entries(mapping.semantic_roles)) {
-                    deps.out(`    ${role.padEnd(20)} ${'\u{2192}'} '${sr.status_name}'  ${pc.dim(`(${sr.issue_type})`)}`);
-                }
-            }
-            else {
-                deps.out(`  ${pc.yellow('No semantic roles resolved yet.')}`);
-            }
+            deps.out(mappingAsJson(mapping, opts.rolesOnly === true));
             continue;
         }
-        for (const it of mapping.issue_types) {
-            deps.out(`\n  ${pc.bold(it.name)}`);
-            for (const s of it.statuses) {
-                deps.out(`    [${s.category}] ${s.name}`);
-            }
-            if (it.transitions.length) {
-                deps.out('    Transitions:');
-                for (const t of it.transitions) {
-                    deps.out(`      ${t.from_status} ${'\u{2192}'} ${t.to_status}  ${pc.dim(`(${t.name})`)}`);
-                }
-            }
-        }
-        if (Object.keys(mapping.semantic_roles).length) {
-            deps.out(`\n  ${pc.bold('Semantic roles:')}`);
-            for (const [role, sr] of Object.entries(mapping.semantic_roles)) {
-                deps.out(`    ${role.padEnd(20)} ${'\u{2192}'} '${sr.status_name}'  ${pc.dim(`(${sr.issue_type})`)}`);
-            }
-        }
+        printMapping(key, mapping, opts.rolesOnly === true, deps);
     }
     if (!anyFound) {
         throw new CliExitError(1);

package/dist/reporters/testtracker.d.ts

@@ -82,7 +82,7 @@ export declare function runStatus(totals: {
  * `test.id`, which is genuinely unique, but a title is not: a Playwright
  * `dependencies:` setup project runs in full in EVERY shard (dependencies are
  * not sharded), so a `merge-reports` payload over a sharded matrix legitimately
- * carries the same setup title once per shard. Capwell's `capwell-web` suite
+ * carries the same setup title once per shard. One consumer’s sharded suite
  * had every nightly rejected this way and never ingested a single run.
  *
  * Collapse rule keeps the merged row honest: worst status wins, the first real

package/dist/reporters/testtracker.js

@@ -109,7 +109,7 @@ const STATUS_SEVERITY = {
  * `test.id`, which is genuinely unique, but a title is not: a Playwright
  * `dependencies:` setup project runs in full in EVERY shard (dependencies are
  * not sharded), so a `merge-reports` payload over a sharded matrix legitimately
- * carries the same setup title once per shard. Capwell's `capwell-web` suite
+ * carries the same setup title once per shard. One consumer’s sharded suite
  * had every nightly rejected this way and never ingested a single run.
  *
  * Collapse rule keeps the merged row honest: worst status wins, the first real

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "canary-test-cli",
-  "version": "7.1.0",
+  "version": "8.0.0",
   "description": "Canary — AI-powered test automation agent",
   "license": "MIT",
   "repository": {
@@ -39,7 +39,8 @@
   "files": [
     "bin/canary.js",
     "bin/canary-mcp.js",
-    "dist/"
+    "dist/",
+    "agents/"
   ],
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.30.0",