canary-test-cli 7.0.0 → 7.2.0

package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+// canary-fail-fast -- surface test failures fast and loud. Ported behavior-for-
+// behavior from the Python original.
+//
+// Two halves:
+//   --config <playwright.config.*>  audit fail-fast knobs (maxFailures/forbidOnly/
+//                                   retries); print recommendations (read-only).
+//   --results <playwright.json>     print a loud, categorized failure digest to
+//                                   the CI log + ::error annotations; exit non-
+//                                   zero on any failure so the step fails.
+//
+// At least one of --config / --results is required. Self-contained -- no
+// external skill dependency.
+//
+// Invoked via `canary skills run canary-fail-fast -- --results <json> [--config <path>]`.
+
+import fs from 'node:fs';
+
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+import { parseFailures } from './parse.mjs';
+import { buildDigest } from './digest.mjs';
+import { checkConfig } from './fastfail_check.mjs';
+
+const PREFIX = 'canary-fail-fast:';
+const DASH = '\u2014'; // em dash (see digest.mjs)
+
+const USAGE =
+  'usage: canary-fail-fast [-h] [--results PATH] [--config PATH]\n' +
+  '\n' +
+  'Fail-fast config audit + loud run-end failure digest.';
+
+/**
+ * Both value flags are optional here -- main enforces "at least one of
+ * --results / --config" itself, because argparse has no vocabulary for that.
+ * The four shared invariants live in the shared parser (#479).
+ */
+export const CLI_SPEC = {
+  prog: 'canary-fail-fast',
+  values: {
+    '--results': { key: 'results' },
+    '--config': { key: 'config' },
+  },
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  if (!args.results && !args.config) {
+    console.error(
+      `${PREFIX} nothing to do ${DASH} pass --results and/or --config.`,
+    );
+    return 1;
+  }
+
+  // ---- config audit -----------------------------------------------------
+  if (args.config) {
+    let text;
+    try {
+      text = fs.readFileSync(args.config, 'utf8');
+    } catch (exc) {
+      console.error(`${PREFIX} cannot read config: ${exc.message}`);
+      return 1;
+    }
+    const recs = checkConfig(text);
+    if (recs.length) {
+      console.log('Fail-fast config recommendations:');
+      for (const r of recs) console.log(`  - ${r}`);
+    } else {
+      console.log('Fail-fast config OK.');
+    }
+  }
+
+  // ---- failure digest ---------------------------------------------------
+  let exitCode = 0;
+  if (args.results) {
+    if (!fs.existsSync(args.results)) {
+      console.error(`${PREFIX} results file not found: ${args.results}`);
+      return 1;
+    }
+    let failures;
+    try {
+      failures = parseFailures(args.results);
+    } catch (exc) {
+      console.error(`${PREFIX} ${exc.message}`);
+      return 1;
+    }
+    const d = buildDigest(failures);
+    console.log(d.text);
+    for (const ann of d.annotations) console.log(ann);
+    exitCode = d.exitCode;
+  }
+
+  return exitCode;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
+// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated JSON that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs

@@ -0,0 +1,69 @@
+// digest -- loud, categorized failure digest (pure). Ported behavior-for-
+// behavior from the Python original.
+//
+// Turns a list of failures into a terse CI-log digest + `::error` workflow
+// annotations + a non-zero exit code, so an engineer triages from the run log
+// without opening the HTML report.
+
+import { FAILURE_CATEGORIES, categorizeFailure } from './failures.mjs';
+
+// Output glyphs kept as \u escapes so the source stays ASCII while the emitted
+// text is byte-identical to the Python original.
+const CHECK = '\u2705'; // white heavy check mark
+const CROSS = '\u274c'; // cross mark
+const DASH = '\u2014'; // em dash
+
+/**
+ * @typedef {{text: string, annotations: string[], exitCode: number}} Digest
+ */
+
+function firstLine(error, limit = 160) {
+  if (!error) return '(no error message)';
+  for (const raw of error.split(/\r\n|\r|\n/)) {
+    const line = raw.trim();
+    if (line) return line.slice(0, limit);
+  }
+  return '(no error message)';
+}
+
+/** Build the digest text, `::error` annotations, and exit code from failures. */
+export function buildDigest(failures) {
+  if (!failures.length) {
+    return { text: `${CHECK} 0 failing tests.`, annotations: [], exitCode: 0 };
+  }
+
+  const n = failures.length;
+  const byCat = new Map();
+  for (const f of failures) {
+    const cat = categorizeFailure(f.error);
+    if (!byCat.has(cat)) byCat.set(cat, []);
+    byCat.get(cat).push(f);
+  }
+
+  const lines = [
+    `${CROSS} ${n} failing test${n !== 1 ? 's' : ''} ${DASH} triage by category:`,
+    '',
+  ];
+  for (const cat of FAILURE_CATEGORIES) {
+    const bucket = byCat.get(cat);
+    if (!bucket || !bucket.length) continue;
+    lines.push(`  ${cat} (${bucket.length}):`);
+    for (const f of bucket) {
+      lines.push(`    - ${f.title} ${DASH} ${firstLine(f.error)}`);
+    }
+  }
+  const text = lines.join('\n');
+
+  const annotations = [];
+  for (const f of failures) {
+    const cat = categorizeFailure(f.error);
+    let loc = '';
+    if (f.file) loc += `file=${f.file},`;
+    if (f.line !== null && f.line !== undefined) loc += `line=${f.line},`;
+    annotations.push(
+      `::error ${loc}title=Test failure::${f.title} ${DASH} ${cat}: ${firstLine(f.error)}`,
+    );
+  }
+
+  return { text, annotations, exitCode: 1 };
+}

package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs

@@ -0,0 +1,60 @@
+// failures -- heuristic failure categorization (self-contained, pure). Ported
+// behavior-for-behavior from the Python original.
+//
+// Order matters -- the most distinctive signals are checked first so 4xx/5xx
+// status-code patterns don't swallow schema errors that happen to mention a
+// status code in a response preview.
+
+export const FAILURE_CATEGORIES = [
+  'schema',
+  'auth',
+  'server',
+  'client',
+  'timeout',
+  'network',
+  'other',
+];
+
+// [category, pattern] pairs in match-priority order (distinct from the display
+// order in FAILURE_CATEGORIES). Patterns are case-insensitive and stateless
+// (no `g` flag), mirroring Python's re.search with re.IGNORECASE.
+//
+// Intentional deviation (exotic input only): the `\b` word boundaries use
+// JavaScript's ASCII-only word-character definition, whereas Python's `re` is
+// Unicode-aware by default. An accented/non-ASCII letter glued directly to a
+// status code (a "u"-with-umlaut immediately before `401`, say) can therefore
+// categorize differently than the Python original. This is negligible for real
+// Playwright error messages, which are
+// ASCII around status codes; documented so the behavior reads as deliberate.
+const RULES = [
+  [
+    'schema',
+    /ZodError|invalid[_ ]type|unrecognized key|expected .+ received|at path "|\bzod\b/i,
+  ],
+  [
+    'auth',
+    /\b401\b|unauthorized|\b403\b|forbidden|invalid(?: auth)? token|token expired/i,
+  ],
+  ['timeout', /timeout|timed out|etimedout|deadline exceeded/i],
+  [
+    'network',
+    /econnrefused|enotfound|econnreset|socket hang up|getaddrinfo|network request failed/i,
+  ],
+  [
+    'server',
+    /\b5\d{2}\b|internal server error|bad gateway|service unavailable|gateway timeout/i,
+  ],
+  [
+    'client',
+    /\b4(?:0[045-9]|1\d|2\d)\b|bad request|not found|unprocessable|conflict/i,
+  ],
+];
+
+/** Return the category of a failure error message ('other' when unknown). */
+export function categorizeFailure(error) {
+  if (!error) return 'other';
+  for (const [category, pattern] of RULES) {
+    if (pattern.test(error)) return category;
+  }
+  return 'other';
+}

package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs

@@ -0,0 +1,43 @@
+// fastfail_check -- fail-fast config audit for a Playwright config (pure, read-
+// only). Ported behavior-for-behavior from the Python original.
+//
+// Scans a `playwright.config.*` for the fail-fast knobs a suite should set so a
+// broken CI run aborts early instead of burning the whole matrix. Never edits
+// the config -- it recommends, and CANONICAL is the block to paste in.
+
+const DASH = '\u2014'; // em dash (see digest.mjs)
+
+// The block to paste into playwright.config.ts. Exported for reference; the CLI
+// does not emit it.
+export const CANONICAL = `export default defineConfig({
+  // Fail fast in CI: abort once enough has clearly broken, never on local runs.
+  forbidOnly: !!process.env.CI,             // a stray test.only fails the build
+  maxFailures: process.env.CI ? 10 : 0,     // stop the run after 10 failures in CI
+  retries: process.env.CI ? 2 : 0,          // absorb flakes in CI; surface them locally
+  // ...your existing config
+});
+`;
+
+// knob name -> why it matters, in the Python dict's insertion order.
+const KNOBS = [
+  ['forbidOnly', 'a stray `test.only` silently skips the rest of the suite'],
+  [
+    'maxFailures',
+    'a broken run keeps burning the matrix instead of aborting early',
+  ],
+  [
+    'retries',
+    'flakes either fail the build or hide locally without a CI retry policy',
+  ],
+];
+
+/**
+ * Return one recommendation per fail-fast knob missing from the config text.
+ * Empty array means all knobs are present. Substring scan -- good enough to flag
+ * absence; it does not validate the knob's value.
+ */
+export function checkConfig(text) {
+  return KNOBS.filter(([knob]) => !text.includes(knob)).map(
+    ([knob, why]) => `Add \`${knob}\` ${DASH} without it, ${why}.`,
+  );
+}

package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs

@@ -0,0 +1,149 @@
+// parse -- minimal Playwright JSON parser (failing tests only, self-contained).
+// Ported behavior-for-behavior from the Python original.
+//
+// Walks the Playwright JSON reporter's nested suites/specs/tests and returns the
+// real failures. A failed/unexpected test with a passing retry is flaky and
+// excluded; without one it is a failure. Leading non-JSON banners are stripped
+// before parsing (matches the reporter's defensive indexOf('{')).
+
+import fs from 'node:fs';
+
+/**
+ * @typedef {{title: string, status: string, file: (string|null),
+ *            line: (number|null), error: (string|null)}} Failure
+ */
+
+/** Build a Failure with the Python dataclass defaults (absent -> null). */
+export function Failure(fields) {
+  return {
+    title: fields.title,
+    status: fields.status,
+    file: fields.file ?? null,
+    line: fields.line ?? null,
+    error: fields.error ?? null,
+  };
+}
+
+function isObject(x) {
+  return x !== null && typeof x === 'object' && !Array.isArray(x);
+}
+
+// Mirror Python dict.get: `.get` on a non-dict (str/list/number/None) raises
+// AttributeError, which parseFailures converts into a structured error. Here a
+// non-object access throws TypeError, caught by the same conversion.
+function dget(obj, key, dflt) {
+  if (!isObject(obj)) throw new TypeError('value is not an object');
+  return key in obj ? obj[key] : dflt;
+}
+
+// Mirror Python's `container or []` for a for-loop target. Python truthiness
+// differs from JS: an empty object `{}` and empty string `""` are FALSY in
+// Python but TRUTHY in JS. A naive `x || []` would turn `{"suites": {}}` into a
+// thrown "unexpected structure" (JS iterates the truthy `{}`) where Python
+// quietly iterates nothing -- flipping a CI pass into a fail. So:
+//   - any Python-falsy value (null/undefined/false/0/""/[]/{})  -> [] (iterate nothing)
+//   - an Array                                                  -> the array
+//   - any other truthy value (non-empty object/string/number)   -> returned as-is,
+//     so `for...of` reproduces Python's iterate-then-`.get`-fails path (a string
+//     iterates its chars; a non-empty object/number is non-iterable and throws),
+//     which surfaces as the same "unexpected structure" error.
+function asItems(v) {
+  if (Array.isArray(v)) return v;
+  if (v && typeof v === 'object') return Object.keys(v).length ? v : [];
+  if (v && typeof v !== 'object') return v; // non-empty string / truthy number
+  return []; // null/undefined/false/0/""
+}
+
+/** Parse a Playwright JSON results file, returning only the real failures. */
+export function parseFailures(resultsPath) {
+  if (!fs.existsSync(resultsPath)) return [];
+
+  let text = fs.readFileSync(resultsPath, 'utf8');
+  const braceAt = text.indexOf('{');
+  if (braceAt > 0) text = text.slice(braceAt);
+
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch (exc) {
+    throw new Error(`results file is not valid JSON: ${exc.message}`);
+  }
+
+  if (!isObject(data)) {
+    throw new Error("results file's top-level value must be an object");
+  }
+
+  const failures = [];
+  try {
+    for (const suite of asItems(data.suites)) {
+      processSuite(suite, failures, '', '');
+    }
+  } catch (exc) {
+    if (exc instanceof TypeError) {
+      throw new Error(
+        `results file has an unexpected structure: ${exc.message}`,
+      );
+    }
+    throw exc;
+  }
+  return failures;
+}
+
+function processSuite(suite, failures, parentPath, suiteFile) {
+  const suiteTitle = dget(suite, 'title', '');
+  const suitePath = parentPath ? `${parentPath} > ${suiteTitle}` : suiteTitle;
+  const currentFile = dget(suite, 'file', undefined) || suiteFile;
+
+  for (const child of asItems(dget(suite, 'suites', []))) {
+    processSuite(child, failures, suitePath, currentFile);
+  }
+
+  for (const spec of asItems(dget(suite, 'specs', []))) {
+    const specPath = `${suitePath} > ${dget(spec, 'title', '')}`;
+    const specLocation = dget(spec, 'location', undefined) || {};
+    for (const test of asItems(dget(spec, 'tests', []))) {
+      const testTitle =
+        dget(test, 'title', undefined) || dget(spec, 'title', '');
+      const testLocation = dget(test, 'location', undefined) || {};
+      const results = asItems(dget(test, 'results', undefined));
+
+      const status = dget(test, 'status', 'unknown');
+      if (status !== 'unexpected' && status !== 'failed') continue;
+
+      const hasPassingRetry = results.some((r) => {
+        const s = dget(r, 'status', undefined);
+        return s === 'passed' || s === 'expected';
+      });
+      if (hasPassingRetry) continue; // flaky -- excluded from the failure count
+
+      let error = null;
+      if (results.length) {
+        const last = results[results.length - 1];
+        const err = dget(last, 'error', undefined) || {};
+        error = dget(err, 'message', undefined);
+        if (error === undefined || error === null) {
+          const errs = dget(last, 'errors', undefined) || [];
+          if (errs.length) {
+            error = dget(errs[0], 'message', undefined);
+          }
+        }
+      }
+
+      failures.push(
+        Failure({
+          title: `${specPath} > ${testTitle}`,
+          status,
+          file:
+            dget(testLocation, 'file', undefined) ||
+            dget(specLocation, 'file', undefined) ||
+            currentFile,
+          line:
+            dget(testLocation, 'line', undefined) ||
+            dget(specLocation, 'line', undefined) ||
+            null,
+          error,
+        }),
+      );
+    }
+  }
+}

package/agents/skills/claude-code/canary-failure-impact/SKILL.md

@@ -0,0 +1,153 @@
+---
+name: canary-failure-impact
+description: >
+  For a given test, function, or code path, traces downstream effects and
+  produces a severity label. Investigates config/auth failures using the
+  consuming repo's declared user_catalog_skill. Optionally focuses on critical
+  paths when critical-areas.json is present.
+---
+
+# Canary: Failure Impact
+
+Answers "what actually breaks if this code fails and no test catches it?"
+Produces a severity label and a concrete description of downstream effects to
+help prioritise where to invest test coverage.
+
+## When to Use
+
+- Before deciding which gap to close first: "which of these matters most?"
+
+- When a test fails and you need to understand the blast radius
+
+- As Phase 3 of `/canary-test-pipeline`
+
+- When asked "what's the impact if this breaks?"
+
+## This skill vs. `canary guardian analyze`
+
+This skill discovers downstream dependents with harness's `compute_blast_radius`
+primitive when the MCP is present (degrading to plain `grep -r` when it is not),
+then applies a **domain-keyword heuristic** (Steps 3–4 below) to turn that
+dependent set into a severity label. The severity labeling is the heuristic part
+— keyword matching over dependent file and function names.
+`canary guardian analyze` (`ts/src/guardian/`, wired to
+`canary guardian analyze` in `ts/src/cli.ts`) is a **real OpenAPI-diff
+blast-radius engine** — it diffs two OpenAPI specs (`--spec-before` /
+`--spec-after`), extracts the actual added/removed/changed endpoints, and maps
+each to coverage gaps against a `coverage-report.json`. For the class of change
+it covers, guardian is strictly higher-fidelity than the heuristics here.
+
+- **Use `canary guardian analyze`** when the change is an API/schema change and
+  you have (or can generate) before/after OpenAPI specs — it gives exact
+  endpoint-level impact and coverage-gap data instead of a keyword guess.
+- **Use this skill** for everything guardian doesn't cover: non-API code paths
+  (services, UI components, internal functions), impact tracing where no OpenAPI
+  spec exists, or when you need the broader billing/auth/compliance
+  domain-severity labeling in Step 3 rather than a strict API diff.
+
+They are complementary, not competing — do not duplicate guardian's spec-diff
+logic here if an OpenAPI change is in scope; delegate to
+`canary guardian analyze` instead.
+
+## Input
+
+Provide one of:
+
+- A test file path: `tests/loyalty/points.spec.ts`
+
+- A function name: `accruePoints`
+
+- A code path: `src/loyalty/points.service.ts`
+
+If `.canary/critical-areas.json` is present, focus tracing on paths with
+`risk_score ≥ 0.7`.
+
+## Tracing Logic
+
+### Step 1 — Identify the code path
+
+Resolve the input to a specific file and function. If ambiguous, ask before
+proceeding.
+
+### Step 2 — Walk downstream dependents
+
+**With harness MCP available:** call `compute_blast_radius` for the target file
+(`file`, `mode: "detailed"`). It simulates cascading failure with a
+probability-weighted BFS and returns each affected node with a cumulative
+failure probability — this is the purpose-built blast-radius primitive, so use
+it instead of hand-walking `get_relationships` hop-by-hop. Feed the returned
+node set into Step 3, and let the cumulative probability weight the severity
+(high-probability nodes dominate). When you additionally need the affected set
+grouped by kind (tests vs docs vs code), call `get_impact` for the same target.
+
+**Fallback:** use `grep -r` to find files that import or call the target. Limit
+to direct dependents (1 hop) when MCP is unavailable.
+
+### Step 3 — Classify each dependent by domain
+
+Apply these heuristics to the dependent paths and function names:
+
+| Domain signal                           | Severity modifier                   |
+| --------------------------------------- | ----------------------------------- |
+| billing / payment / charge / invoice    | +2 (financial impact)               |
+| auth / session / token / permission     | +2 (security/access)                |
+| compliance / audit / PHI / PII / HIPAA  | +2 (regulatory)                     |
+| data / persist / write / store / commit | +1 (data integrity)                 |
+| UI / render / display / format / label  | −1 (user-facing only, no data risk) |
+
+### Step 4 — Aggregate to severity label
+
+Base score starts at 2 (Medium). Sum modifiers from step 3. Cap at 4 (Critical).
+
+| Score | Label    |
+| ----- | -------- |
+| 5+    | Critical |
+| 3–4   | High     |
+| 2     | Medium   |
+| 0–1   | Low      |
+
+### Step 5 — User catalog investigation
+
+When a test failure in the target path involves an auth, permission, or
+configuration error:
+
+1. Read `user_catalog_skill` from `.canary/company.json`
+2. If present: invoke `canary skills run <user_catalog_skill>` with the required
+   attributes from the error context
+3. If a matching user/config is found: surface it as a suggestion
+4. If absent or no match: present constructively —
+
+   > "This failure may be a test user or test data configuration issue. Check
+   > your user catalog if you have one, or set up the required test data before
+   > re-running."
+
+## Output Format
+
+```text
+Failure impact — src/loyalty/points.service.ts::accruePoints
+
+  Severity: HIGH
+
+  If this breaks undetected:
+  · Members see incorrect balance in the partner portal  (user-facing)
+  · Points journal diverges from the ledger  (data integrity)
+  · Downstream: redemption.service.ts · tier-upgrade.service.ts ·
+    reporting.service.ts  (3 dependents)
+
+  Priority: write failure-path tests before next release
+  Suggested: /canary-write-test "test failure paths for accruePoints"
+```
+
+## Related skills
+
+- `/canary-critical-areas` — produces `critical-areas.json` used for focus
+
+- `/canary-ci-ready` — uses the same user-catalog investigation pattern
+
+- `/canary-write-test` — generates tests for the identified high-impact gaps
+
+- `/canary-test-pipeline` — Phase 3
+
+- `canary guardian analyze` (CLI, `ts/src/guardian/`) — higher-fidelity
+  OpenAPI-diff blast-radius engine; use instead of this skill's heuristics when
+  the change is an API/schema change with before/after specs available

package/agents/skills/claude-code/canary-failure-impact/skill.yaml

@@ -0,0 +1,15 @@
+name: canary-failure-impact
+version: '1.0.0'
+description:
+  Trace the downstream blast radius of an undetected failure for a given test,
+  function, or code path and assign a severity label.
+stability: static
+triggers:
+  - manual
+platforms:
+  - claude-code
+type: rigid
+tools: []
+tier: 1
+depends_on:
+  - canary-ci-ready