canary-test-cli 7.0.0 → 7.2.0

package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+// canary-test-reporter -- Playwright JSON -> Markdown + JSON report. Ported
+// behavior-for-behavior from the Python original.
+//
+//   --results <path>        required: Playwright JSON results file
+//   --markdown-out <path>   write Markdown report to file (stdout if neither
+//                           --markdown-out nor --json-out is given)
+//   --json-out <path>       write JSON report to file
+//
+// Exit code: 1 when any test failed, else 0.
+//
+// Invoked via `canary skills run canary-test-reporter -- --results <json>`.
+
+import fs from 'node:fs';
+
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+import { parseResults } from './parse.mjs';
+import { renderMarkdown } from './render.mjs';
+import { renderJson } from './json_report.mjs';
+
+const PREFIX = 'canary-test-reporter:';
+
+const USAGE =
+  'usage: canary-test-reporter [-h] --results PATH [--markdown-out PATH]\n' +
+  '                            [--json-out PATH]\n' +
+  '\n' +
+  'Playwright JSON results -> Markdown + JSON test report.';
+
+/**
+ * `--results` is required; the optional writers default to null so main can
+ * tell "not asked for" from "asked for, empty". The four shared invariants live
+ * in the shared parser (#479).
+ */
+export const CLI_SPEC = {
+  prog: 'canary-test-reporter',
+  values: {
+    '--results': { key: 'results' },
+    '--markdown-out': { key: 'markdownOut' },
+    '--json-out': { key: 'jsonOut' },
+  },
+  required: ['--results'],
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  if (!fs.existsSync(args.results)) {
+    console.error(`${PREFIX} results file not found: ${args.results}`);
+    return 1;
+  }
+
+  let data;
+  try {
+    data = parseResults(args.results);
+  } catch (exc) {
+    console.error(`${PREFIX} ${exc.message}`);
+    return 1;
+  }
+
+  const markdown = renderMarkdown(data);
+
+  if (args.markdownOut) {
+    fs.writeFileSync(args.markdownOut, markdown, 'utf8');
+  } else if (!args.jsonOut) {
+    // Python `print(markdown, end="")` -- no trailing newline is added.
+    process.stdout.write(markdown);
+  }
+
+  if (args.jsonOut) {
+    fs.writeFileSync(args.jsonOut, renderJson(data), 'utf8');
+  }
+
+  return data.failed > 0 ? 1 : 0;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
+// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated JSON that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs

@@ -0,0 +1,58 @@
+// json_report -- JSON serializer for Playwright test results (self-contained,
+// pure). Ported behavior-for-behavior from the Python original.
+//
+// The field NAMES and ORDER are a downstream contract (a flat, TCM-row-ish
+// shape); do not rename or reorder them.
+//
+// Accepted numeric-repr limitation (input-shape-dependent, consistent with the
+// canary-instrument port): Python json.dumps renders a whole-number FLOAT with a
+// trailing ".0" (300.0 -> "300.0"), but JS Number cannot distinguish 300 from
+// 300.0, so a whole-number float duration serializes without the ".0". This is
+// unreachable in practice -- Playwright's reporter emits integer millisecond
+// durations, and genuinely fractional floats (12.5) already agree byte-for-byte.
+
+/**
+ * Serialize like Python's json.dumps(obj, indent=2) with the default
+ * ensure_ascii=True. JSON.stringify already matches the 2-space indentation and
+ * the empty-container ("[]"/"{}") forms, but leaves non-ASCII raw -- so escape
+ * every code unit above 0x7F to a \uXXXX sequence (astral chars become their two
+ * UTF-16 surrogate escapes, exactly as Python emits them). This keeps the JSON
+ * artifact byte-identical when a title/error/file carries an emoji, accented, or
+ * CJK character (or an em-dash inside a Playwright diff).
+ */
+function dumpsIndent2Ascii(obj) {
+  const nonAscii = /[\u0080-\uffff]/g;
+  return JSON.stringify(obj, null, 2).replace(
+    nonAscii,
+    (ch) => `\\u${ch.charCodeAt(0).toString(16).padStart(4, '0')}`,
+  );
+}
+
+/** Serialize a ReportData to the pinned JSON report string (indent=2). */
+export function renderJson(data) {
+  // Python: datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -- second
+  // precision, no milliseconds. toISOString() carries `.mmmZ`; strip it so the
+  // emitted bytes match the Python format exactly.
+  const generatedAt = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
+  const output = {
+    version: 1,
+    generated_at: generatedAt,
+    summary: {
+      total: data.total,
+      passed: data.passed,
+      failed: data.failed,
+      flaky: data.flaky,
+      skipped: data.skipped,
+      duration_ms: data.duration_ms,
+    },
+    results: data.results.map((r) => ({
+      title: r.title,
+      status: r.status,
+      file: r.file,
+      line: r.line,
+      duration_ms: r.duration_ms,
+      error: r.error,
+    })),
+  };
+  return dumpsIndent2Ascii(output);
+}

package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs

@@ -0,0 +1,216 @@
+// parse -- full-fidelity Playwright JSON parser (self-contained). Ported
+// behavior-for-behavior from the Python original.
+//
+// Walks the Playwright JSON reporter's nested suites/specs/tests and classifies
+// each test as passed, failed, flaky, or skipped. A failed/unexpected test with
+// a passing retry is flaky and carries no error. Leading non-JSON banners are
+// stripped before parsing (matches the reporter's defensive indexOf('{')).
+
+import fs from 'node:fs';
+
+/**
+ * @typedef {{title: string, status: string, file: (string|null),
+ *            line: (number|null), duration_ms: (number|null),
+ *            error: (string|null)}} TestResult
+ */
+
+/** Build a TestResult with the Python dataclass defaults (absent -> null). */
+export function TestResult(fields) {
+  return {
+    title: fields.title,
+    status: fields.status,
+    file: fields.file ?? null,
+    line: fields.line ?? null,
+    duration_ms: fields.duration_ms ?? null,
+    error: fields.error ?? null,
+  };
+}
+
+/**
+ * @typedef {{total: number, passed: number, failed: number, flaky: number,
+ *            skipped: number, duration_ms: number,
+ *            results: TestResult[]}} ReportData
+ */
+
+/** Build a ReportData (mirrors the Python dataclass; all fields required). */
+export function ReportData(fields) {
+  return {
+    total: fields.total,
+    passed: fields.passed,
+    failed: fields.failed,
+    flaky: fields.flaky,
+    skipped: fields.skipped,
+    duration_ms: fields.duration_ms,
+    results: fields.results,
+  };
+}
+
+function isObject(x) {
+  return x !== null && typeof x === 'object' && !Array.isArray(x);
+}
+
+// Mirror Python dict.get: `.get` on a non-dict (str/list/number/None) raises
+// AttributeError, which parseResults converts into a structured error. Here a
+// non-object access throws TypeError, caught by the same conversion.
+function dget(obj, key, dflt) {
+  if (!isObject(obj)) throw new TypeError('value is not an object');
+  return key in obj ? obj[key] : dflt;
+}
+
+// Mirror Python's `container or []` for a for-loop target. Python truthiness
+// differs from JS: an empty object `{}` and empty string `""` are FALSY in
+// Python but TRUTHY in JS. A naive `x || []` would turn `{"suites": {}}` into a
+// thrown "unexpected structure" (JS iterates the truthy `{}`) where Python
+// quietly iterates nothing -- flipping a CI pass into a fail. So:
+//   - any Python-falsy value (null/undefined/false/0/""/[]/{})  -> [] (iterate nothing)
+//   - an Array                                                  -> the array
+//   - any other truthy value (non-empty object/string/number)   -> returned as-is,
+//     so `for...of` / `.some` reproduces Python's iterate-then-`.get`-fails path
+//     (a string iterates its chars; a non-empty object/number is non-iterable and
+//     throws), which surfaces as the same "unexpected structure" error.
+function asItems(v) {
+  if (Array.isArray(v)) return v;
+  if (v && typeof v === 'object') return Object.keys(v).length ? v : [];
+  if (v && typeof v !== 'object') return v; // non-empty string / truthy number
+  return []; // null/undefined/false/0/""
+}
+
+/** Classify a raw Playwright status into passed/failed/flaky/skipped. */
+function classify(rawStatus, testResults) {
+  if (rawStatus === 'skipped' || rawStatus === 'pending') return 'skipped';
+  if (rawStatus === 'passed' || rawStatus === 'expected') return 'passed';
+  if (rawStatus === 'flaky') return 'flaky';
+  if (rawStatus === 'failed' || rawStatus === 'unexpected') {
+    const hasPassingRetry = testResults.some((r) => {
+      const s = dget(r, 'status', undefined);
+      return s === 'passed' || s === 'expected';
+    });
+    return hasPassingRetry ? 'flaky' : 'failed';
+  }
+  return 'failed';
+}
+
+/** Parse a Playwright JSON results file into a full ReportData. */
+export function parseResults(resultsPath) {
+  if (!fs.existsSync(resultsPath)) {
+    return ReportData({
+      total: 0,
+      passed: 0,
+      failed: 0,
+      flaky: 0,
+      skipped: 0,
+      duration_ms: 0,
+      results: [],
+    });
+  }
+
+  let text = fs.readFileSync(resultsPath, 'utf8');
+  const braceAt = text.indexOf('{');
+  if (braceAt > 0) text = text.slice(braceAt);
+
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch (exc) {
+    throw new Error(`results file is not valid JSON: ${exc.message}`);
+  }
+
+  if (!isObject(data)) {
+    throw new Error("results file's top-level value must be an object");
+  }
+
+  const results = [];
+  try {
+    for (const suite of asItems(data.suites)) {
+      processSuite(suite, results, '', '');
+    }
+  } catch (exc) {
+    if (exc instanceof TypeError) {
+      throw new Error(
+        `results file has an unexpected structure: ${exc.message}`,
+      );
+    }
+    throw exc;
+  }
+
+  const passed = results.filter((r) => r.status === 'passed').length;
+  const failed = results.filter((r) => r.status === 'failed').length;
+  const flaky = results.filter((r) => r.status === 'flaky').length;
+  const skipped = results.filter((r) => r.status === 'skipped').length;
+  const totalMs = results.reduce((acc, r) => acc + (r.duration_ms || 0), 0);
+
+  return ReportData({
+    total: results.length,
+    passed,
+    failed,
+    flaky,
+    skipped,
+    duration_ms: totalMs,
+    results,
+  });
+}
+
+function processSuite(suite, results, parentPath, suiteFile) {
+  const suiteTitle = dget(suite, 'title', '');
+  const suitePath = parentPath ? `${parentPath} > ${suiteTitle}` : suiteTitle;
+  const currentFile = dget(suite, 'file', undefined) || suiteFile;
+
+  for (const child of asItems(dget(suite, 'suites', []))) {
+    processSuite(child, results, suitePath, currentFile);
+  }
+
+  for (const spec of asItems(dget(suite, 'specs', []))) {
+    const specTitle = dget(spec, 'title', '');
+    const specPath = `${suitePath} > ${specTitle}`;
+    const specLocation = dget(spec, 'location', undefined) || {};
+    for (const test of asItems(dget(spec, 'tests', []))) {
+      const rawTestTitle = dget(test, 'title', null);
+      const testLocation = dget(test, 'location', undefined) || {};
+      const testResults = asItems(dget(test, 'results', undefined));
+      const rawStatus = dget(test, 'status', 'unknown');
+
+      const status = classify(rawStatus, testResults);
+
+      let error = null;
+      let duration = null;
+      if (testResults.length) {
+        const last = testResults[testResults.length - 1];
+        duration = dget(last, 'duration', null);
+        if (status === 'failed') {
+          const err = dget(last, 'error', undefined) || {};
+          error = dget(err, 'message', null);
+          if (error === null || error === undefined) {
+            const errs = dget(last, 'errors', undefined) || [];
+            if (errs.length) error = dget(errs[0], 'message', null);
+          }
+        }
+      }
+
+      const title =
+        rawTestTitle && rawTestTitle !== specTitle
+          ? `${specPath} > ${rawTestTitle}`
+          : specPath;
+
+      const testLine = dget(testLocation, 'line', null);
+      const line =
+        testLine !== null && testLine !== undefined
+          ? testLine
+          : dget(specLocation, 'line', null);
+
+      results.push(
+        TestResult({
+          title,
+          status,
+          file:
+            dget(testLocation, 'file', undefined) ||
+            dget(specLocation, 'file', undefined) ||
+            currentFile ||
+            null,
+          line,
+          duration_ms: duration,
+          error,
+        }),
+      );
+    }
+  }
+}

package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs

@@ -0,0 +1,114 @@
+// render -- Markdown renderer for Playwright test results (self-contained,
+// pure). Ported behavior-for-behavior from the Python original.
+//
+// Output glyphs are kept as \u escapes so the source stays ASCII while the
+// emitted Markdown bytes are byte-identical to the Python original:
+//   MIDDOT   U+00B7 MIDDLE DOT         -- status-line separator
+//   ELLIPSIS U+2026 HORIZONTAL ELLIPSIS -- error-truncation marker
+//   EMDASH   U+2014 EM DASH            -- flaky-line separator
+const MIDDOT = '\u00b7';
+const ELLIPSIS = '\u2026';
+const EMDASH = '\u2014';
+
+const ERROR_LINE_LIMIT = 10;
+
+// Reproduce Python str.splitlines(): it splits on \n, \r, \r\n, \v, \f, and the
+// \x1c-\x1e / \x85 / \u2028 / \u2029 boundaries, and yields NO trailing empty
+// element for a trailing boundary. A naive "a\n".split("\n") would add a
+// spurious blank line inside the fenced error block, so match splitlines.
+function splitlines(s) {
+  const re = /\r\n|[\n\r\v\f\x1c\x1d\x1e\x85\u2028\u2029]/g;
+  const parts = [];
+  let last = 0;
+  let m;
+  while ((m = re.exec(s)) !== null) {
+    parts.push(s.slice(last, m.index));
+    last = m.index + m[0].length;
+  }
+  if (last < s.length) parts.push(s.slice(last));
+  return parts;
+}
+
+// Match Python's f"{ms/1000:.1f}" byte-for-byte, including round-half-to-EVEN
+// on exact ties. JS `(ms/1000).toFixed(1)` rounds half-UP, so it diverges
+// wherever ms/1000 lands on an exactly-representable .5-tenths tie -- which for
+// integer milliseconds is precisely ms == 250 (mod 500) (x.25 / x.75, the only
+// dyadic half-tenths): 250ms -> Python "0.2s" vs toFixed "0.3s". At those ties
+// the two candidate tenths are floor(ms/100) and +1; Python picks the even one.
+// Non-tie values are not exact halves, so toFixed already rounds them to the
+// same nearest tenth Python does. Assumes the integer-ms contract (a fractional
+// duration is rounded by toFixed, matching Python for all non-tie inputs).
+function formatDurationSeconds(ms) {
+  if (ms % 500 === 250) {
+    const lowerTenths = Math.floor(ms / 100);
+    const tenths = lowerTenths % 2 === 0 ? lowerTenths : lowerTenths + 1;
+    return (tenths / 10).toFixed(1);
+  }
+  return (ms / 1000).toFixed(1);
+}
+
+function formatLocation(r) {
+  if (!r.file) return '';
+  return r.line ? '`' + r.file + ':' + r.line + '`' : '`' + r.file + '`';
+}
+
+/** Render a ReportData to the themed Markdown report. */
+export function renderMarkdown(data) {
+  const parts = ['# Test Report\n\n'];
+
+  // Status line, e.g.:
+  //   **2 failed** <MIDDOT> **1 flaky** <MIDDOT> **14 passed** <MIDDOT>
+  //   **1 skipped** <MIDDOT> 18 tests <MIDDOT> 12.4s
+  const chips = [];
+  if (data.failed) chips.push(`**${data.failed} failed**`);
+  if (data.flaky) chips.push(`**${data.flaky} flaky**`);
+  if (data.passed) chips.push(`**${data.passed} passed**`);
+  if (data.skipped) chips.push(`**${data.skipped} skipped**`);
+  const durationS = formatDurationSeconds(data.duration_ms);
+  let statusLine = `${data.total} tests ${MIDDOT} ${durationS}s\n`;
+  if (chips.length) {
+    statusLine = chips.join(` ${MIDDOT} `) + ` ${MIDDOT} ` + statusLine;
+  }
+  parts.push(statusLine);
+
+  // Failed section
+  const failed = data.results.filter((r) => r.status === 'failed');
+  if (failed.length) {
+    parts.push(`\n## Failed (${failed.length})\n`);
+    for (const r of failed) {
+      parts.push(`\n### ${r.title}\n`);
+      const loc = formatLocation(r);
+      if (loc) parts.push(`\n${loc}\n`);
+      if (r.error) {
+        let lines = splitlines(r.error);
+        if (lines.length > ERROR_LINE_LIMIT) {
+          lines = lines
+            .slice(0, ERROR_LINE_LIMIT)
+            .concat([`${ELLIPSIS} (truncated)`]);
+        }
+        parts.push(`\n\`\`\`\n${lines.join('\n')}\n\`\`\`\n`);
+      }
+    }
+  }
+
+  // Flaky section
+  const flaky = data.results.filter((r) => r.status === 'flaky');
+  if (flaky.length) {
+    parts.push(`\n## Flaky (${flaky.length})\n`);
+    for (const r of flaky) {
+      const loc = formatLocation(r);
+      parts.push(`\n- ${loc ? loc + ` ${EMDASH} ` : ''}${r.title}\n`);
+    }
+  }
+
+  // Summary table
+  parts.push('\n## Summary\n');
+  parts.push('\n| Status | Count |\n| --- | --- |\n');
+  parts.push(`| Passed | ${data.passed} |\n`);
+  parts.push(`| Failed | ${data.failed} |\n`);
+  parts.push(`| Flaky | ${data.flaky} |\n`);
+  parts.push(`| Skipped | ${data.skipped} |\n`);
+  parts.push(`| **Total** | **${data.total}** |\n`);
+
+  return parts.join('');
+}