canary-test-cli 7.1.0 → 8.0.0

package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs

@@ -0,0 +1,187 @@
+// span_reader -- merge/correlate OTel span JSONL files into a Trace (pure,
+// read-only). Ported from Python span_reader.py, behavior-for-behavior.
+//
+// Reads one or more `otel-spans.<worker>.jsonl` files (one JSON span object
+// per line, written by otel_bootstrap/instrument.mjs), groups spans by
+// `traceId`, resolves each trace's root span (the one carrying a `test.id`
+// attribute -- set by otel_bootstrap/playwright-fixture.ts's root-span
+// fixture), and attaches that trace's HTTP child spans to the resolved test.
+// Traces with no `test.id`-attributed root bucket their HTTP spans under the
+// synthetic test id "__setup__" (traffic outside any test, e.g. global setup).
+//
+// Assumed span envelope: {traceId, spanId, parentSpanId, name, startTime,
+// duration_ms, attributes{}} -- matches exactly what instrument.mjs's
+// JsonlFileSpanExporter writes; no `endTime` key is emitted (duration_ms +
+// startTime cover it). HTTP attributes keyed http.method/http.request.method,
+// http.url, http.route, http.status_code.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { RequestSpan, TestTrace, Trace } from './run_types.mjs';
+
+const SETUP_TEST_ID = '__setup__';
+
+// Mirrors Python's Path.glob("otel-spans.*.jsonl"): the `*` matches any run of
+// characters (including none) within a single path segment.
+const SPAN_FILE_RE = /^otel-spans\..*\.jsonl$/;
+
+/**
+ * Read every `otel-spans.*.jsonl` file under `spansDir`, correlate spans to
+ * their test roots, and return the Trace. A missing or non-directory path
+ * yields an empty trace (never throws) -- the same as the Python version.
+ * @param {string} spansDir
+ */
+export function readTraces(spansDir) {
+  const byTrace = new Map();
+
+  if (isDir(spansDir)) {
+    const files = fs
+      .readdirSync(spansDir)
+      .filter((name) => SPAN_FILE_RE.test(name))
+      .sort();
+    for (const name of files) {
+      for (const span of readJsonl(path.join(spansDir, name))) {
+        const traceId = span.traceId;
+        if (!traceId) continue;
+        if (!byTrace.has(traceId)) byTrace.set(traceId, []);
+        byTrace.get(traceId).push(span);
+      }
+    }
+  }
+
+  const byTest = [];
+  const setupRequests = [];
+  let spansTotal = 0;
+
+  for (const [traceId, spans] of byTrace) {
+    const root = spans.find(isTestRoot) ?? null;
+    const httpSpans = spans.filter((s) => s !== root && isHttpSpan(s));
+    const requests = httpSpans.map(toRequestSpan);
+    spansTotal += requests.length;
+
+    if (root === null) {
+      setupRequests.push(...requests);
+      continue;
+    }
+
+    const attrs = root.attributes ?? {};
+    byTest.push(
+      TestTrace({
+        test_id: attrs['test.id'] ?? '',
+        test_title: attrs['test.title'] ?? '',
+        test_file: attrs['test.file'] ?? '',
+        trace_id: traceId,
+        outcome: attrs['test.outcome'] ?? '',
+        requests,
+      }),
+    );
+  }
+
+  if (setupRequests.length) {
+    byTest.push(
+      TestTrace({
+        test_id: SETUP_TEST_ID,
+        test_title: '',
+        test_file: '',
+        trace_id: '',
+        outcome: '',
+        requests: setupRequests,
+      }),
+    );
+  }
+
+  return Trace({ spans_total: spansTotal, by_test: byTest });
+}
+
+function isDir(p) {
+  try {
+    return fs.statSync(p).isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+function readJsonl(filePath) {
+  const spans = [];
+  // NDJSON records are \n- or \r\n-delimited. We intentionally do NOT split on
+  // the exotic Unicode line separators that Python's str.splitlines() also
+  // breaks on (U+2028, U+2029, U+0085, ...): instrument.mjs writes those RAW
+  // inside JSON string values, so splitting on them would fragment a valid
+  // record and misattribute its request to __setup__.
+  for (const raw of fs.readFileSync(filePath, 'utf8').split(/\r\n|\n/)) {
+    const line = raw.trim();
+    if (!line) continue;
+    try {
+      spans.push(JSON.parse(line));
+    } catch {
+      continue; // malformed/torn line (e.g. a crashed worker's last write)
+    }
+  }
+  return spans;
+}
+
+function isTestRoot(span) {
+  return 'test.id' in (span.attributes ?? {});
+}
+
+function isHttpSpan(span) {
+  const attrs = span.attributes ?? {};
+  return 'http.method' in attrs || 'http.request.method' in attrs;
+}
+
+/**
+ * Reconstruct the request URL across both OTel HTTP semantic conventions.
+ *
+ * The old convention put the whole thing in one attribute, `http.url`. The
+ * current one (stable since semconv 1.23) splits it: `url.full` on some
+ * instrumentations, and otherwise the pieces -- `url.scheme`, `server.address`,
+ * `server.port`, `url.path`. `@opentelemetry/auto-instrumentations-node`, which
+ * `otel_bootstrap/instrument.mjs` tells consumers to install, emits the SPLIT
+ * form and no `http.url` at all.
+ *
+ * Reading only `http.url` therefore produced `url: ""` on every request while
+ * the reader still reported a full span count and exit 0 -- the artifact looked
+ * written and was empty of the one field a coverage consumer needs. Port is
+ * omitted when it is the scheme default, so the URL matches what a spec or a
+ * HAR would say.
+ */
+function requestUrl(attrs) {
+  const direct = attrs['url.full'] ?? attrs['http.url'];
+  if (direct) return direct;
+
+  const scheme = attrs['url.scheme'] ?? attrs['http.scheme'];
+  const host =
+    attrs['server.address'] ?? attrs['net.peer.name'] ?? attrs['http.host'];
+  const path = attrs['url.path'] ?? attrs['http.target'] ?? '';
+  if (!scheme || !host) return typeof path === 'string' ? path : '';
+
+  const port = attrs['server.port'] ?? attrs['net.peer.port'];
+  const isDefaultPort =
+    port == null ||
+    (scheme === 'http' && +port === 80) ||
+    (scheme === 'https' && +port === 443);
+  const authority = isDefaultPort ? host : `${host}:${port}`;
+  const query = attrs['url.query'] ? `?${attrs['url.query']}` : '';
+  return `${scheme}://${authority}${path}${query}`;
+}
+
+function toRequestSpan(span) {
+  const attrs = span.attributes ?? {};
+  const method = attrs['http.method'] || attrs['http.request.method'] || '';
+  return RequestSpan({
+    method,
+    url: requestUrl(attrs),
+    // `http.route` is a SERVER-side attribute and is never present on the client
+    // spans this reader consumes. Kept for producers that do supply it (a future
+    // server-side producer under the same v1 contract), but its absence here is
+    // normal, not a gap.
+    route: attrs['http.route'] ?? null,
+    // Renamed in the current convention. Old name kept as a fallback.
+    status:
+      attrs['http.response.status_code'] ?? attrs['http.status_code'] ?? null,
+    duration_ms: span.duration_ms ?? 0,
+    span_id: span.spanId ?? '',
+    started_at: span.startTime ?? '',
+  });
+}

package/agents/skills/claude-code/canary-katana/SKILL.md

@@ -0,0 +1,243 @@
+---
+name: canary-katana
+description:
+  Quarantines deleted and newly-skipped tests instead of letting them vanish.
+  Captures every removed or skipped test with provenance (who, when, which
+  commit, why) into an append-only ledger, and alarms in exactly one case — the
+  deletion removed the last coverage of a symbol critical-areas.json marks
+  high-risk. Silent by default, degrades to recording-only when critical-area
+  data is missing. Self-contained, deterministic, advisory by default.
+cli: scripts/cli.mjs
+requires: [node>=20]
+---
+
+# Canary Katana
+
+Named for Tatsu Yamashiro's Soultaker — the blade that captures the soul of
+whatever it cuts. A deleted test is coverage that leaves without a trace: the
+suite still goes green, the gap is invisible, and nobody notices until the bug
+it caught ships. Katana catches every test as it is removed or muted, records
+who took it and why, and raises its voice only when the cut was the last thing
+guarding a critical path.
+
+Tier-0 deterministic analysis: no LLM, no network, no secrets, no dependency on
+any other skill at runtime.
+
+## What it captures
+
+| Event     | Detected from a diff                                                                                                                                                      |
+| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `removed` | a `def test_*` / `async def test_*` (Python) or `describe`/`it`/`test('…')` (JS/TS) that left on a `-` line **and did not come back on the `+` side**                     |
+| `skipped` | a `+`-side skip/mute marker: `@pytest.mark.skip` / `skipif` / `xfail`, or `it.skip` / `test.skip` / `describe.skip`, `it.only` / `test.only`, `xit` / `xdescribe` / `fit` |
+
+A test flipped in place from `it('x')` to `it.skip('x')` is **one** event, not
+two: the skip supersedes the removal so the ledger never double-counts a
+mute-in-place as both a deletion and a skip.
+
+The general form of that rule is the emphasis above: a `(file, title)` that
+reappears on the `+` side was **modified, not removed**. Without it, any rewrite
+of a declaration line recorded a deletion of a test that is still in the tree —
+a prettier reflow of a long signature, or a `.skip` lifted in place, was enough
+(#783). The ledger is append-only, so such a row is permanent and cannot be
+corrected without the hand-edit the ledger exists to prevent; and a consumer
+that attributes on the newest matching row would hand every test under a phantom
+removal of a `describe` the wrong ticket. A **rename** is still a removal — the
+old title's coverage really is gone.
+
+## The one thing it alarms on
+
+Most test deletions are legitimate — dead feature removal, genuine dedup — so
+alarming on every one is nag fatigue within a week, and a gate people mute is
+worse than no gate. Katana is **silent by default** and alarms only when a
+removed test was the **last coverage** of a symbol listed in
+`critical-areas.json` (produced by `canary-critical-areas`).
+
+- **name-matched** — the removed test's name matches an area symbol and no other
+  test still covers it. Severity `critical` when the area's `risk_score` is high
+  (≥ 0.7), otherwise `high`.
+- **heuristic** — only the test's _directory_ maps to the area (no name match).
+  Always severity `medium`, and flagged as lower fidelity.
+
+### Degradation is loud and safe
+
+When `critical-areas.json` is missing or malformed, katana records everything
+but alarms on nothing, printing:
+
+```text
+critical-area data unavailable, recording only, not alarming
+```
+
+Degradation never manufactures a failure — even under `--strict`, a degraded run
+exits `0`.
+
+## The ledger
+
+Append-only JSON at `.canary/quarantine.json` (override with `--ledger`). Each
+row carries full provenance so a vanished test leaves a trail:
+
+```json
+{
+  "schema_version": 2,
+  "entries": [
+    {
+      "test": "test_points_service_earns",
+      "file": "tests/test_points.py",
+      "kind": "removed",
+      "marker": "",
+      "commit": "…40 hex…",
+      "author": "Ada Lovelace",
+      "date": "2026-07-20T10:00:00+00:00",
+      "reason": "chore: drop points coverage",
+      "cause": "",
+      "issue": "",
+      "expiry": ""
+    }
+  ]
+}
+```
+
+Re-running on the same change adds nothing (entries de-duplicate); a corrupt
+ledger is a hard error, never silently overwritten.
+
+### Schema v2: why a row is out, not just how it left (#771)
+
+`cause`, `issue` and `expiry` are written by the quarantine producer, not by
+katana. Katana records what it can observe from a diff — a test was removed or
+skipped — and leaves `cause` empty, because "someone deleted this in commit
+abc123" is provenance, not a judgement about why the test is out of the suite.
+
+`reason` and `cause` are deliberately separate. `reason` is **derived** (the
+commit subject). `cause` is **asserted** — one of `flaky`, `product-defect`,
+`blocked-data`, `obsolete`. Collapsing them would dress an auto-derived string
+up as a claim someone stands behind.
+
+**One row per `(test, file)` may state a cause, and a caused row wins.** A row
+with a cause supersedes a causeless row for the same pair, and a causeless row
+is dropped when a caused row already exists. This is the one place the ledger is
+not purely append-only, and it exists because the alternative is worse: katana
+recording `{kind: 'skipped', cause: ''}` and a quarantine producer recording
+`{kind: 'skipped', cause: 'product-defect', issue: …}` differ in every-field
+identity, so **both** would persist — and a consumer that fails on an unlinked
+quarantine (`canary-ci-ready` does) would fail on the causeless row while the
+linked row sat beside it. The ledger would be contradicting itself about one
+test.
+
+History survives that rule: only rows differing in cause-bearing state collapse.
+Two caused rows, or two causeless rows, keep the full-field identity and both
+remain.
+
+### Where `issue` comes from, and why the trailer keeps its own name
+
+`issue` is the bug the quarantine is waiting on. It has two sources, and both
+land in the same field:
+
+- **A `Ticket:` commit trailer**, read by katana at capture time (`Bug:` and
+  `Tracked:` are accepted spellings). This is the low-friction path: the person
+  switching the test off names the bug in the commit that does it.
+- **A quarantine producer**, writing a caused row directly.
+
+v1 called this field `ticket` (#781). It is folded into `issue` here rather than
+kept alongside, because two fields answering "what is this waiting on" is how a
+consumer ends up reading the empty one — and the consumer is specific:
+`canary-ci-ready` fails a quarantine with no **linked issue**, in either Jira or
+GitHub. The schema now uses the consumer's word. A v1 row's `ticket` migrates
+onto `issue` on load, so no recorded link is lost.
+
+`Ticket:` survives as the name of the **trailer**, which is a mechanism rather
+than a schema: it is what you type in a commit message, and renaming it would
+invalidate the trailers already written without teaching anyone anything.
+
+Empty is a real and important state, not a gap to paper over. A test switched
+off with nothing to chase is the worst thing this ledger can record, and it can
+only be seen if it is recorded honestly.
+
+A v1 file is normalized on load, so every row comes back carrying the v2 fields
+(empty where unrecorded). That is what makes writing `schema_version: 2` honest
+— the version claims these rows have these fields, and after load they do.
+Stamping the version over un-migrated rows would make it a promise the file does
+not keep.
+
+## Invocation
+
+```bash
+# Diff the current branch against its merge-base, record, advise (exit 0):
+canary skills run canary-katana
+
+# Feed an explicit diff and a critical-areas map:
+canary skills run canary-katana -- \
+  --diff-file changes.diff --critical-areas .canary/critical-areas.json
+
+# Machine-readable:
+canary skills run canary-katana -- --json
+
+# Fail the step only when a critical path loses its last coverage:
+canary skills run canary-katana -- --strict
+
+# Usage and options (exits 0, and writes nothing to the ledger):
+canary skills run canary-katana -- --help
+```
+
+Value flags (`--repo`, `--diff-file`, `--ledger`, `--critical-areas`) accept
+both `--repo <path>` and `--repo=<path>`, matching `canary-instrument` and
+`canary-fail-fast`.
+
+An unknown flag is rejected with `unrecognized arguments: <flag>` and exit 2,
+and a value flag left without a usable value is
+`argument <flag>: expected one argument` (exit 2). That covers all three ways
+the value can go missing: the flag is last, the next token is another flag, or
+the value is empty — in either the `--repo=` spelling or, the one shells
+actually produce, `--repo "$UNSET_VAR"`. Empty is rejected rather than accepted
+because `--repo ''` would resolve the ledger to `path.join('', '.canary', ...)`
+and write it into the process CWD instead of the target repo.
+
+All of these are decided before any diff is read or ledger entry is appended, so
+a usage request or a typo never mutates the working tree.
+
+`--json` shape:
+
+```json
+{
+  "schema_version": 2,
+  "captured": [
+    { "name": "…", "file": "…", "kind": "removed", "line": 3, "marker": "" }
+  ],
+  "findings": [
+    {
+      "kind": "last-coverage-removed",
+      "test": "…",
+      "file": "…",
+      "area": "src/loyalty/points.service.ts",
+      "fidelity": "name-matched",
+      "severity": "critical",
+      "evidence": "…"
+    }
+  ],
+  "ledger": ".canary/quarantine.json"
+}
+```
+
+A degraded run adds a top-level `"degraded_notice"` and an empty `findings`.
+
+## CI wiring (GitHub Actions)
+
+Advisory first, then promote to blocking once the ledger is trusted — the same
+path every canary gate takes.
+
+```yaml
+- name: Quarantine deleted tests (advisory)
+  run:
+    canary skills run canary-katana -- --critical-areas
+    .canary/critical-areas.json
+# Once trusted, add --strict so a last-coverage loss fails the PR:
+# run: canary skills run canary-katana -- --critical-areas .canary/critical-areas.json --strict
+```
+
+## Fidelity limits (regex/diff-lite, on purpose)
+
+- **Line-scoped diff parsing.** A declaration split across lines can be missed;
+  katana errs toward recording the clear cases.
+- **Name/dir coverage is heuristic.** "Last coverage" is inferred from test
+  names and directory layout, not a real coverage run — treat `heuristic`
+  findings as prompts to look, not verdicts.
+- **Provenance needs git.** Fed a `--diff-file` outside a git repo, author and
+  commit are recorded as `unknown` / empty rather than guessed.