canary-test-cli 7.0.0 → 7.2.0

package/dist/engine/core/promotion-verdict.js

@@ -0,0 +1,261 @@
+/**
+ * The structured verdict `canary-promote-test` gates on (#477).
+ *
+ * ## Why this exists now, when the issue says it is blocked
+ *
+ * #477 was parked on the emit side: `harness:test-craft` runs an 8-axis per-test
+ * LLM critique with no machine-readable output, so there was nothing to consume
+ * and building a consumer against an unspecified shape meant building it twice.
+ * That is still true of the LLM critique. It is no longer true of the gate:
+ * #605 (soundness) and #612 (vacuity) emit structured, per-test, DETERMINISTIC
+ * verdicts, and those are the ones that were ever going to be allowed to block.
+ *
+ * ## The three decisions #477 asked for
+ *
+ * **Which axes gate.** Deterministic defects gate; style reports.
+ *
+ * | Axis              | Rules                        | Gates | Why                                                              |
+ * | ----------------- | ---------------------------- | ----- | ---------------------------------------------------------------- |
+ * | `soundness`       | `SOUND-001/002/003`          | yes   | Pins a value no correct implementation must produce              |
+ * | `assertions`      | `LINT-006`                   | yes   | A test that asserts nothing always passes                        |
+ * | `flakiness`       | `FLAKE-001/002`              | yes   | The issue's own named blocker; both are `critical`               |
+ * | `vacuity`         | `VAC-001/003`, annotated 002 | yes   | Deterministic, or the author declared the target themselves      |
+ * | `selectors`       | `LINT-001/002/003`           | no    | Brittle, not wrong; a reviewer's call                            |
+ * | `maintainability` | `LINT-005`, `FLAKE-003/004`  | no    | Style and softer signals                                        |
+ *
+ * Gating on all of them would block nearly every promotion, which is exactly the
+ * outcome #477 predicted for a naive 8-axis gate.
+ *
+ * **What happens with no verdict.** `abstain`, exit 3, and say so. Promotion
+ * falls back to today's manual review. It must never become silently stricter
+ * (a `block` nobody can act on) or silently looser (a `promote` over a file the
+ * scanner could not read) -- and two distinct zeros are guarded: a file no
+ * ruleset parses, and a parseable file holding no tests.
+ *
+ * **Whether an LLM judgement may block.** No. Everything that gates in this repo
+ * is deterministic, and this change does not spend that. The decision is
+ * structural rather than documentary: {@link VerdictSource} admits one value, so
+ * there is no field an LLM verdict can arrive in and quietly acquire authority.
+ * `harness:test-craft` stays what `canary-promote-test` already calls it -- an
+ * optional deeper audit for a human.
+ *
+ * ## The fidelity ladder
+ *
+ * `VAC-002` is inference, and #477's real anxiety was making a heuristic
+ * load-bearing on a promotion gate. So the rung decides the authority, mirroring
+ * the guardian's `coverage-verified > graph-verified > heuristic`:
+ * `annotated` (the author named the target) blocks; `import-inferred` reports.
+ */
+import { EXIT_ABSTAINED, errnoCode, gateOutcome, } from './gate-result.js';
+import { StaticLinter, UnsupportedTestFileError, frameworkForPath, } from './static-linter.js';
+import { scanVacuity, } from './vacuity-scanner.js';
+/**
+ * Which rules land on which axis, and whether that axis gates.
+ *
+ * A rule matching NO row here is invisible to the verdict: not gating, not
+ * advisory, not printed. `LINT-004` -- an unawaited Playwright action, the
+ * linter's other `critical` and the canonical false-green defect -- was omitted
+ * from the first cut and walked straight through the gate built to stop it.
+ * `test-signal-review-findings.test.ts` now asserts that every rule a real lint
+ * produces lands somewhere, because the omission is otherwise silent.
+ */
+const AXES = [
+    { axis: 'soundness', gating: true, rules: /^SOUND-/ },
+    { axis: 'assertions', gating: true, rules: /^LINT-006$/ },
+    // LINT-004 sits here rather than in its own axis because an unawaited action
+    // IS the classic race: the assertion runs before the action lands.
+    { axis: 'flakiness', gating: true, rules: /^FLAKE-00[12]$|^LINT-004$/ },
+    { axis: 'vacuity', gating: true, rules: /^VAC-/ },
+    { axis: 'selectors', gating: false, rules: /^LINT-00[123]$/ },
+    {
+        axis: 'maintainability',
+        gating: false,
+        rules: /^LINT-005$|^FLAKE-00[34]$/,
+    },
+];
+/**
+ * Whether a finding on a gating axis may actually block.
+ *
+ * The one place the fidelity ladder is spent. `VAC-002` at `import-inferred` is
+ * an inference about which symbol a test was meant to exercise; blocking a
+ * promotion on that would make a heuristic load-bearing, which is the specific
+ * thing #477 asked not to do by accident. At `annotated` fidelity the author
+ * wrote the target down, so the finding is a contradiction of a stated contract
+ * and blocks.
+ */
+function mayBlock(f) {
+    if (f.rule !== 'VAC-002')
+        return true;
+    return f.fidelity === 'annotated';
+}
+function normalizeLint(f) {
+    return {
+        rule: f.rule,
+        line: f.line,
+        severity: f.severity,
+        message: f.message,
+        suggestion: f.suggestion,
+    };
+}
+function normalizeVacuity(f) {
+    const out = {
+        rule: f.rule,
+        line: f.line,
+        severity: f.severity,
+        message: `${f.test}: ${f.message}`,
+        suggestion: f.suggestion,
+    };
+    if (f.fidelity)
+        out.fidelity = f.fidelity;
+    return out;
+}
+const ABSTAIN_REMEDY = 'No verdict could be produced, so promotion falls back to manual review ' +
+    '(canary-promote-test Phase 1) -- it has NOT been approved. ' +
+    'Point at a single generated test file that contains at least one test.';
+function abstained(file, skipped, axes) {
+    const outcome = gateOutcome({ checked: 0, findings: [], skipped }, 'gate');
+    return {
+        file,
+        decision: 'abstain',
+        source: 'deterministic',
+        checked: 0,
+        axes,
+        blocked: [],
+        skipped,
+        summaryLine: outcome.summaryLine,
+        remedy: ABSTAIN_REMEDY,
+        exitCode: EXIT_ABSTAINED,
+    };
+}
+function emptyAxes() {
+    return AXES.map((a) => ({ axis: a.axis, gating: a.gating, findings: [] }));
+}
+/**
+ * Produce the promotion verdict for one generated test file.
+ *
+ * Deliberately single-file: promotion is a per-file decision, and a directory
+ * roll-up would let one clean file's verdict read as cover for a sibling's.
+ */
+export function promotionVerdict(path) {
+    if (frameworkForPath(path) === null) {
+        return abstained(path, [
+            {
+                name: path,
+                reason: 'no ruleset parses this extension, so a clean result would be meaningless',
+            },
+        ], emptyAxes());
+    }
+    const lint = lintOrAbstain(path);
+    if (!Array.isArray(lint))
+        return lint;
+    const vacuity = scanVacuity(path);
+    const axes = groupIntoAxes([
+        ...lint.map(normalizeLint),
+        ...vacuity.findings.map(normalizeVacuity),
+    ]);
+    const skipped = vacuity.skipped ?? [];
+    // A parseable file with no tests is another zero, and it must not read as a
+    // pass: promotion would let an empty file into the committed suite. Note that
+    // `lint` can still be non-empty here (a stray `Date.now()` outside any test),
+    // so the check is on the TEST count, not on the finding count.
+    if (vacuity.checked === 0) {
+        return abstained(path, [
+            ...skipped,
+            {
+                name: path,
+                reason: 'file holds no test declarations, so there is nothing to promote',
+            },
+        ], axes);
+    }
+    return decide(path, axes, skipped, vacuity.checked);
+}
+/**
+ * The lint findings, or a ready-made abstention when the linter refused.
+ *
+ * Two distinct refusals, and BOTH have to become an abstention rather than an
+ * exception. Letting a read error propagate meant the CLI printed a raw ENOENT
+ * stack and exited 0 -- a promotion gate that could not open the draft,
+ * reporting success. Anything without an `errno`-style `code` still throws,
+ * because swallowing an unknown fault is how a scanner learns to go quiet.
+ */
+function lintOrAbstain(path) {
+    try {
+        return new StaticLinter().lint(path);
+    }
+    catch (e) {
+        if (e instanceof UnsupportedTestFileError) {
+            return abstained(path, [{ name: path, reason: e.message }], emptyAxes());
+        }
+        // ERRNO-shaped only. Keying off "has a string `code`" swept in every Node
+        // PROGRAMMER error too -- `ERR_INVALID_ARG_TYPE`, `ERR_STRING_TOO_LONG` --
+        // so a genuine defect inside the linter was reported as a clean ABSTAIN with
+        // a misleading reason instead of surfacing. An unknown fault must still
+        // throw; that is the difference between degrading honestly and going quiet.
+        const code = errnoCode(e);
+        if (code === null)
+            throw e;
+        return abstained(path, [{ name: path, reason: `could not be read (${code})` }], emptyAxes());
+    }
+}
+/** Every axis, in registry order, each carrying its findings sorted by line. */
+function groupIntoAxes(all) {
+    return AXES.map((spec) => ({
+        axis: spec.axis,
+        gating: spec.gating,
+        findings: all
+            .filter((f) => spec.rules.test(f.rule))
+            .sort((a, b) => a.line - b.line),
+    }));
+}
+/** Turn evaluated axes into the promote/block decision and its copy. */
+function decide(path, axes, skipped, checked) {
+    const blocked = [
+        ...new Set(axes
+            .filter((a) => a.gating)
+            .flatMap((a) => a.findings)
+            .filter(mayBlock)
+            .map((f) => f.rule)),
+    ];
+    // `gateOutcome` owns the summary line so promotion reports its denominator in
+    // the same shape as every other gate, rather than inventing a private format.
+    const outcome = gateOutcome({ checked, findings: blocked, skipped }, 'gate', {
+        noun: 'test(s)',
+    });
+    const decision = blocked.length > 0 ? 'block' : 'promote';
+    const advisoryCount = axes
+        .filter((a) => !a.gating)
+        .reduce((n, a) => n + a.findings.length, 0);
+    return {
+        file: path,
+        decision,
+        source: 'deterministic',
+        checked,
+        axes,
+        blocked,
+        skipped,
+        summaryLine: outcome.summaryLine,
+        remedy: remedyFor(decision, blocked, advisoryCount, skipped.length),
+        exitCode: decision === 'block' ? 1 : 0,
+    };
+}
+function remedyFor(decision, blocked, advisoryCount, skipCount) {
+    if (decision === 'block') {
+        return (`Blocked on ${blocked.join(', ')}. Fix the test or regenerate it ` +
+            'with a sharper prompt -- do not hand-patch a generated draft ' +
+            '(canary-promote-test Phase 1).');
+    }
+    const parts = ['Promotable.'];
+    if (advisoryCount > 0) {
+        parts.push(`${advisoryCount} advisory finding(s) are a reviewer's call, not a blocker.`);
+    }
+    // A `promote` whose rules went dark must not read as an unqualified pass.
+    // `checked` legitimately counts the tests VAC-001 did run on, so the
+    // denominator is right -- but a reader seeing only "promotable" would never
+    // learn that two of the three vacuity rules could not be evaluated at all.
+    if (skipCount > 0) {
+        parts.push(`${skipCount} check(s) could not run on this file -- see the skip list; ` +
+            'those rules did NOT pass, they abstained.');
+    }
+    return advisoryCount > 0 || skipCount > 0 ? parts.join(' ') : '';
+}
+//# sourceMappingURL=promotion-verdict.js.map

package/dist/engine/core/quality-scorer.js

@@ -19,10 +19,23 @@ const ASSERTIONS = {
     // nothing. (`\bassert\b` alone does NOT match `assert_valid`: `_` is a word
     // char, so the `\b` after `assert` fails there.)
     pytest: /\bassert\b|\bpytest\.raises\b|\bself\.assert\w+\b|\bassert\w*\s*\(/g,
-    playwright: /\bexpect\s*\(|\btoBeVisible\b|\btoHaveText\b|\btoHaveTitle\b|\btoHaveURL\b|\btoBeEnabled\b|\btoBeDisabled\b|\btoBeChecked\b|\btoHaveValue\b|\btoHaveCount\b/g,
+    // `\bexpect\w*\s*\(` is the JS/TS half of that same rationale (#738). The
+    // naming convention for a custom assertion helper is `expect*` here, not
+    // `assert*`, and it is the pattern Playwright's own docs recommend: a suite
+    // that routes its checks through `expectRouteTestId(page, id)` so the
+    // assertion semantics live in one documented place had EVERY added test
+    // flagged `added test asserts nothing`. `\w*` is zero-width-matchable, so
+    // plain `expect(` still matches and the change is purely additive.
+    //
+    // The precision cost is the mirror image of the accepted pytest one: a
+    // non-asserting function whose name happens to start with `expect` now
+    // counts. That is the right side to err on for an advisory finding that
+    // never gates — a false "asserts nothing" on a correct test is what teaches
+    // a reader to ignore the finding.
+    playwright: /\bexpect\w*\s*\(|\btoBeVisible\b|\btoHaveText\b|\btoHaveTitle\b|\btoHaveURL\b|\btoBeEnabled\b|\btoBeDisabled\b|\btoBeChecked\b|\btoHaveValue\b|\btoHaveCount\b/g,
     // Plus non-`expect` assertion styles common in JS/TS: node:assert / vitest
     // `assert(...)` / `assert.equal(...)`, and chai BDD `x.should.equal`.
-    vitest: /\bexpect\s*\(|\btoBe\s*\(|\btoEqual\s*\(|\btoThrow\b|\btoContain\s*\(|\btoBeNull\b|\btoBeUndefined\b|\btoMatchObject\b|\bassert\s*\(|\bassert\.\w+|\.should\b/g,
+    vitest: /\bexpect\w*\s*\(|\btoBe\s*\(|\btoEqual\s*\(|\btoThrow\b|\btoContain\s*\(|\btoBeNull\b|\btoBeUndefined\b|\btoMatchObject\b|\bassert\s*\(|\bassert\.\w+|\.should\b/g,
     k6: /\bcheck\s*\(|'[^']+'\s*:\s*\([^)]*\)\s*=>/g,
 };
 const NEGATIVE_KW = /\b(error|invalid|empty|null|undefined|throws|raises|exception|fail|missing|negative|reject|4\d{2}|5\d{2}|boundary|edge)\b/i;

package/dist/engine/core/reporter.js

@@ -25,6 +25,7 @@
  */
 import { mkdirSync, writeFileSync } from 'node:fs';
 import { dirname } from 'node:path';
+import { ensureAscii } from '../util/ensure-ascii.js';
 const SARIF_SCHEMA = 'https://json.schemastore.org/sarif-2.1.0.json';
 const TOOL_NAME = 'Canary';
 const TOOL_VERSION = '0.1.0';
@@ -70,15 +71,6 @@ function pyOr(value, fallback) {
 function pyGet(obj, key, fallback) {
     return Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : fallback;
 }
-/**
- * Reproduce Python's `json.dumps(..., ensure_ascii=True)` (the library default)
- * on `JSON.stringify` output: escape every code point >= 0x80 as `\uXXXX`. Only
- * touches the >= 0x80 range, so the ASCII escapes `JSON.stringify` already
- * produced are left intact. (Same helper as `guardian/pr-check.ts`.)
- */
-function ensureAscii(json) {
-    return json.replace(/[€-￿]/g, (ch) => '\\u' + ch.charCodeAt(0).toString(16).padStart(4, '0'));
-}
 /**
  * `json.dumps(default=str)` replacer. Values the encoder can't natively handle
  * are coerced via `str()`; in JS the only common such value that would

package/dist/engine/core/skill-dispatch.js

@@ -0,0 +1,115 @@
+/**
+ * Tier 2 of `canary skills run`: the dispatcher (#756).
+ *
+ * Canary shipped tier 1 (a skill declaring `cli:`/`entry:` is spawned) and
+ * tier 3 (a prose skill is unreachable), and nothing between. 14 of canary's 21
+ * skills carry no `cli:`, so no orchestrator, CI step, or sibling skill could
+ * invoke them at all -- a have/have-not split that costs far more here than the
+ * same split costs harness, where the dispatcher runs the CLI-less majority.
+ *
+ * ## What "running a prose skill" means here, honestly
+ *
+ * Canary is a CLI. It has no agent runtime, and it is not going to grow one to
+ * close this gap. So the dispatcher does the one thing a CLI can do faithfully:
+ * it RESOLVES the skill and hands back its executable contract -- identity,
+ * declared runtime requirements, and the workflow text an agent is to apply --
+ * with the tier and the determinism stated on the payload. The caller gets a
+ * resolved, machine-readable handle to a real skill instead of exit 2.
+ *
+ * What it deliberately does NOT do is apply the workflow and present the result
+ * as canary's. That would be canary claiming an answer it did not compute.
+ *
+ * ## Determinism labelling (issue design question 2)
+ *
+ * Every dispatch is stamped `determinism: 'agent-applied'`, against
+ * `'deterministic'` for a `cli:` skill. A consumer merging findings across
+ * skills must be able to tell a scanner's output from an agent's reading of a
+ * ruleset; without the label the two look interchangeable, which is exactly the
+ * confusion #755 documents about cassandra.
+ *
+ * ## Why no `--allow-executable-skills` equivalent (design question 3)
+ *
+ * That flag exists because a freshly cloned overlay can carry a `cli:` script,
+ * and invoking it runs someone else's code on the next CI run. Dispatch runs
+ * nothing: it reads a markdown file the registry already read at discovery and
+ * prints it. There is no new execution to gate, so gating it would be
+ * ceremony -- and ceremony that would keep the 14 skills unreachable in exactly
+ * the non-interactive contexts the issue is about. The trust boundary moves to
+ * whatever the caller does with the returned text, which is the caller's gate
+ * to own, and the payload labels itself so the caller can see what it holds.
+ *
+ * ## Failure mode (design question 4)
+ *
+ * A skill that could not be dispatched raises {@link SkillDispatchError}. An
+ * unreadable or bodyless SKILL.md is a failure, never an empty success -- a
+ * dispatcher that returned "nothing to do" for a skill it could not read would
+ * be indistinguishable from one that ran and found nothing.
+ */
+import { readFileSync } from 'node:fs';
+import { errnoCode } from './gate-result.js';
+// Written as an escape so this source stays ASCII, matching gate-result.ts.
+const EMDASH = '\u{2014}';
+/** A dispatch that could not be completed. Never degrades to an empty result. */
+export class SkillDispatchError extends Error {
+    skill;
+    constructor(skill, message) {
+        super(message);
+        this.name = 'SkillDispatchError';
+        this.skill = skill;
+    }
+}
+/**
+ * Strip a leading `---` frontmatter block, leaving the workflow prose.
+ *
+ * Mirrors the delimiter handling in `SkillRegistry.parseFrontmatter`: an
+ * unterminated block means the whole file was frontmatter, and there is no
+ * body to hand back.
+ */
+export function skillBody(text) {
+    if (!text.startsWith('---'))
+        return text.trim();
+    const rest = text.split('\n').slice(1);
+    const end = rest.findIndex((l) => l.trim() === '---');
+    return end === -1
+        ? ''
+        : rest
+            .slice(end + 1)
+            .join('\n')
+            .trim();
+}
+/**
+ * Resolve a prose skill into its dispatch payload.
+ *
+ * @throws {SkillDispatchError} when SKILL.md cannot be read, or holds no body.
+ */
+export function dispatchProseSkill(skill, args) {
+    let text;
+    try {
+        text = readFileSync(skill.path, 'utf-8');
+    }
+    catch (exc) {
+        const code = errnoCode(exc);
+        if (code === null)
+            throw exc;
+        throw new SkillDispatchError(skill.name, `cannot read ${skill.path} (${code}) ${EMDASH} the skill was ` +
+            'discovered but its workflow could not be loaded.');
+    }
+    const instructions = skillBody(text);
+    if (!instructions) {
+        throw new SkillDispatchError(skill.name, `${skill.path} carries frontmatter but no workflow body ${EMDASH} ` +
+            'there is nothing to dispatch. Reporting this as an empty run would ' +
+            'be indistinguishable from a skill that ran and found nothing.');
+    }
+    return {
+        skill: skill.name,
+        path: skill.path,
+        tier: 'dispatcher',
+        determinism: 'agent-applied',
+        requires_agent_runtime: true,
+        requires: skill.requires,
+        description: skill.description,
+        instructions,
+        args,
+    };
+}
+//# sourceMappingURL=skill-dispatch.js.map