canary-test-cli 7.1.0 → 8.0.0

package/agents/skills/claude-code/canary-instrument/SKILL.md

@@ -0,0 +1,157 @@
+---
+name: canary-instrument
+description:
+  Instrument a Playwright run with OpenTelemetry and emit a run.json artifact
+  correlating every test to the outbound HTTP requests it made — "which test
+  made which request?" — with zero manual bookkeeping in test code. Trace-only
+  v1 contract, additive-safe for future pytest/k6/node producers. Self-contained
+  (bundles its own run-type factories and span reader).
+cli: scripts/cli.mjs
+requires: [node>=20]
+---
+
+# Canary Instrument
+
+Correlate every outbound HTTP request in a Playwright run to the test that made
+it, using OTel span parent/child relationships. Zero required external
+dependencies to produce output — default file-based span export, no OTel
+collector needed.
+
+## Setup (two manual steps, once per suite)
+
+This skill ships fixture _files_ you wire into your own suite — it does not
+vendor OTel as a dependency of the `canary` package itself. Install these first:
+
+```bash
+npm install --save-dev \
+  @opentelemetry/sdk-node @opentelemetry/api \
+  @opentelemetry/auto-instrumentations-node \
+  @opentelemetry/exporter-trace-otlp-http
+```
+
+**1. Bootstrap the OTel SDK before Playwright starts** — add `NODE_OPTIONS` to
+your test command (or `playwright.config.ts`'s `webServer`/CI step):
+
+```bash
+NODE_OPTIONS="--import ./node_modules/canary/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs" \
+  npx playwright test
+```
+
+Copy `otel_bootstrap/instrument.mjs` into your repo (e.g. `otel/instrument.mjs`)
+if you'd rather not reference the path inside `node_modules`.
+
+**2. Merge the root-span fixture into your `fixtures.ts`:**
+
+```ts
+import { test as base } from '@playwright/test';
+import { withTestSpan } from './otel_bootstrap/playwright-fixture';
+
+export const test = withTestSpan(base);
+```
+
+Every test using this `test` export now opens a root span carrying
+`test.id`/`test.title`/`test.file`, and every HTTP call the test makes nests
+under it automatically — no manual span code in individual tests.
+
+## Invocation
+
+```bash
+canary skills run canary-instrument -- \
+  --spans test-results/trace --output test-results \
+  [--suite-type e2e_ui]
+
+# Usage and options (exits 0; --spans/--output are not required for --help):
+canary skills run canary-instrument -- --help
+```
+
+An unknown flag, a missing required flag, and a value-flag left without its
+value are all usage errors (exit 2). `--help` short-circuits ahead of the
+required-argument check, so it never complains about `--spans`/`--output`.
+
+Writes `test-results/run.json`. Creates `--output` if it doesn't exist.
+Missing/empty `--spans` produces `trace: {spans_total: 0, by_test: []}`, not a
+failure. `--suite-type` is a free-form string (no enum) — pass whatever label
+describes your suite.
+
+## `run.json` v1 contract (trace-only)
+
+```jsonc
+{
+  "schema_version": 1,
+  "suite_type": "",
+  "generated_at": "2026-07-15T18:00:00+00:00",
+  "trace": {
+    "spans_total": 124,
+    "by_test": [
+      {
+        "test_id": "users-spec:1", // "__setup__" for orphan traffic
+        "test_title": "lists users",
+        "test_file": "tests/users.spec.ts",
+        "trace_id": "abc123...",
+        "outcome": "passed",
+        "requests": [
+          {
+            "method": "GET",
+            "url": "http://localhost:3000/users/1",
+            "route": "/users/:id",
+            "status": 200,
+            "duration_ms": 12.4,
+            "span_id": "def456...",
+            "started_at": "2026-07-15T18:00:01+00:00",
+          },
+        ],
+      },
+    ],
+  },
+}
+```
+
+No `coverage` key, no `canary_run_id` key — cut for v1 (see
+`docs/knowledge/decisions/0006-otel-test-side-tracing.md` and
+`docs/changes/canary-instrument/proposal.md`). Additive-only evolution: new
+optional fields may appear later; existing fields never change meaning.
+
+## Sending spans to a collector (optional)
+
+Set `OTEL_EXPORTER_OTLP_ENDPOINT` before the test run and spans are
+_additionally_ streamed there — the file exporter still writes
+`test-results/trace/otel-spans.*.jsonl` either way, so `canary-instrument`'s own
+correlation is never dependent on a collector being up. If your org's endpoint
+is recorded in company-knowledge, export it first:
+
+```bash
+export OTEL_EXPORTER_OTLP_ENDPOINT="$(canary company-knowledge show --json | jq -r '.otel_exporter_endpoint')"
+```
+
+See `docs/guides/company-knowledge.md` for the `otel_exporter_endpoint` field.
+
+## CI wiring (GitHub Actions)
+
+```yaml
+- name: Run Playwright (instrumented)
+  env:
+    NODE_OPTIONS: '--import ./otel/instrument.mjs'
+  run:
+    npx playwright test --reporter=json --output-file=test-results/results.json
+
+- name: Correlate tests to HTTP spans
+  if: always()
+  run: |
+    canary skills run canary-instrument -- \
+      --spans test-results/trace --output test-results
+
+- name: Upload run.json
+  if: always()
+  uses: actions/upload-artifact@v4
+  with:
+    name: run-trace
+    path: test-results/run.json
+```
+
+## Related skills
+
+- `canary-test-reporter` — Markdown/JSON test summary; `run.json`'s `by_test[]`
+  rows are structurally similar to its `TestResult` shape (title/status as join
+  keys) — a future consumer can read both artifacts with one join key.
+- `canary-fail-fast` — aborts a broken run early; use alongside this skill for
+  complete CI coverage.

package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs

@@ -0,0 +1,178 @@
+#!/usr/bin/env node
+// canary-instrument -- correlate a Playwright run's tests to their outbound
+// HTTP spans. Ported from cli.py, behavior-preserving.
+//
+// Reads OTel span JSONL files written by otel_bootstrap/instrument.mjs (see
+// SKILL.md for the two manual wiring steps), resolves each Playwright test's
+// root span (set by otel_bootstrap/playwright-fixture.ts), attaches HTTP child
+// spans, and writes a run.json v1 artifact (trace-only; see run_types.mjs for
+// the contract).
+//
+// Invoked via:
+//   canary skills run canary-instrument -- \
+//     --spans test-results/trace --output test-results [--suite-type e2e_ui]
+//
+// Missing/empty --spans is not a failure -- it produces an empty trace block.
+// Self-contained -- no external skill dependency.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+
+import { RunArtifact } from './run_types.mjs';
+import { readTraces } from './span_reader.mjs';
+
+const USAGE =
+  'usage: canary-instrument [-h] --spans PATH --output PATH\n' +
+  '                         [--suite-type TYPE]\n' +
+  '\n' +
+  "Correlate a Playwright run's tests to their outbound HTTP spans and write a\n" +
+  'run.json v1 artifact.\n' +
+  '\n' +
+  'options:\n' +
+  '  -h, --help         show this help message and exit\n' +
+  '  --spans PATH       directory of OTel span JSONL files to read\n' +
+  '  --output PATH      directory to write run.json into (created if missing)\n' +
+  '  --suite-type TYPE  suite label recorded in the artifact (e.g. e2e_ui)';
+
+/**
+ * Both value flags are required, and `--suite-type` is a free-form label. Two
+ * deliberate divergences from argparse survive the move to the shared parser
+ * (#479): flags must be spelled in full (no prefix abbreviation, so `--out` is
+ * an unknown flag rather than `--output`), and an empty value is now REJECTED
+ * rather than passed through -- `--spans=` used to yield an empty trace, which
+ * is the missing-value case wearing a disguise.
+ */
+export const CLI_SPEC = {
+  prog: 'canary-instrument',
+  values: {
+    '--spans': { key: 'spans' },
+    '--output': { key: 'output' },
+    '--suite-type': { key: 'suiteType' },
+  },
+  defaults: { suiteType: '' },
+  required: ['--spans', '--output'],
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+/**
+ * Serialize like Python's json.dumps(obj, indent=2): 2-space indent AND
+ * ensure_ascii=True. JSON.stringify already matches the indentation and empty
+ * container ("[]"/"{}") forms, but leaves non-ASCII raw -- so escape every code
+ * unit above 0x7F to a \uXXXX sequence (astral chars become their two UTF-16
+ * surrogate escapes, exactly as Python emits them). No trailing newline,
+ * matching cli.py's write_text.
+ */
+function dumpsIndent2Ascii(obj) {
+  const nonAscii = /[\u0080-\uffff]/g;
+  return JSON.stringify(obj, null, 2).replace(
+    nonAscii,
+    (ch) => `\\u${ch.charCodeAt(0).toString(16).padStart(4, '0')}`,
+  );
+}
+
+/**
+ * Python's datetime.now(timezone.utc).isoformat() renders a "+00:00" offset;
+ * JS toISOString() renders "Z". Normalize the suffix so generated_at keeps the
+ * documented "+00:00" convention. (Python emits microseconds, JS milliseconds;
+ * generated_at is a wall-clock stamp and is never asserted against.)
+ */
+function nowIso() {
+  return new Date().toISOString().replace('Z', '+00:00');
+}
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  const spansDir = args.spans;
+  if (existsButNotDir(spansDir)) {
+    console.error(`canary-instrument: --spans is not a directory: ${spansDir}`);
+    return 1;
+  }
+
+  const trace = readTraces(spansDir);
+  const artifact = RunArtifact({
+    schema_version: 1,
+    suite_type: args.suiteType,
+    generated_at: nowIso(),
+    trace,
+  });
+
+  // A run.json whose requests carry no URL is worse than no run.json. The
+  // artifact's whole purpose is answering "which test hit which endpoint", so a
+  // consumer that reads URL-less requests computes zero endpoints matched and
+  // renders a clean 0% as though it were a measurement. Refuse to write it.
+  //
+  // Note this is NOT the empty-spans case, which is deliberately fine (see the
+  // header): zero spans yields an empty trace and exit 0, and "0 spans, 0
+  // buckets" describes itself honestly. What fails here is the drift signature --
+  // spans WERE captured and correlated, and every one of them lost its URL,
+  // which is what an attribute-name mismatch looks like from the outside.
+  const requests = trace.by_test.flatMap((t) => t.requests ?? []);
+  const withUrl = requests.filter((r) => r.url);
+  if (requests.length > 0 && withUrl.length === 0) {
+    console.error(
+      `canary-instrument: correlated ${requests.length} request(s) across ` +
+        `${trace.spans_total} span(s) and NONE carried a URL. Refusing to write ` +
+        `an artifact that would read as "no endpoints exercised".`,
+    );
+    console.error(
+      'This is what an OTel semantic-convention mismatch looks like: the spans ' +
+        'are present and attributed, but the URL attribute this reader expects is ' +
+        'not the one the instrumentation emitted. Check the attribute keys on an ' +
+        'HTTP span in --spans against requestUrl() in span_reader.mjs.',
+    );
+    return 1;
+  }
+
+  const outputDir = args.output;
+  fs.mkdirSync(outputDir, { recursive: true });
+  const outPath = path.join(outputDir, 'run.json');
+  fs.writeFileSync(outPath, dumpsIndent2Ascii(artifact), 'utf8');
+
+  console.log(
+    `canary-instrument: wrote ${outPath} ` +
+      `(${trace.spans_total} spans, ${trace.by_test.length} test buckets, ` +
+      `${withUrl.length}/${requests.length} request(s) with a URL)`,
+  );
+  // Partial loss is advisory: a mixed run is still usable, but silence would let
+  // a growing blind spot pass for a shrinking one.
+  if (withUrl.length < requests.length) {
+    console.warn(
+      `canary-instrument: ${requests.length - withUrl.length} request(s) had no ` +
+        'resolvable URL and will not match any endpoint.',
+    );
+  }
+  return 0;
+}
+
+function existsButNotDir(p) {
+  try {
+    return !fs.statSync(p).isDirectory();
+  } catch {
+    return false; // missing path is fine -- yields an empty trace
+  }
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
+// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated JSON that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs

@@ -0,0 +1,96 @@
+// otel_bootstrap/instrument.mjs
+//
+// Node OTel SDK bootstrap for canary-instrument. Import via:
+//   NODE_OPTIONS="--import ./otel_bootstrap/instrument.mjs" npx playwright test
+//
+// Default: writes one JSON span per line to
+//   test-results/trace/otel-spans.<TEST_WORKER_INDEX>.jsonl
+// (no collector required — this is what scripts/span_reader.mjs reads).
+// When OTEL_EXPORTER_OTLP_ENDPOINT is set, spans are *additionally*
+// streamed to that collector via OTLPTraceExporter; the file path above is
+// unaffected either way.
+//
+// Auto-instruments HTTP/undici only — fs instrumentation is disabled so
+// Playwright's own file I/O doesn't show up as noise spans.
+//
+// Consumer-supplied dependencies (not vendored by this skill):
+//   @opentelemetry/sdk-node @opentelemetry/api
+//   @opentelemetry/auto-instrumentations-node
+//   @opentelemetry/exporter-trace-otlp-http
+
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const workerIndex = process.env.TEST_WORKER_INDEX ?? '0';
+const outDir = path.join(process.cwd(), 'test-results', 'trace');
+fs.mkdirSync(outDir, { recursive: true });
+// Synchronous fd, not fs.createWriteStream: Playwright's worker process
+// calls process.exit() once its assigned tests finish (it does not wait for
+// the event loop to drain), which cuts off any pending async stream writes
+// before they reach disk. fs.writeSync makes each export() call durable
+// immediately, so span data survives that hard exit — see
+// docs/changes/canary-instrument/proposal.md's ADR reference for why this
+// skill can't assume a graceful shutdown hook will run.
+const outFd = fs.openSync(
+  path.join(outDir, `otel-spans.${workerIndex}.jsonl`),
+  'a',
+);
+
+/** Minimal file exporter — one JSON span per line, matches span_reader.mjs. */
+class JsonlFileSpanExporter {
+  export(spans, resultCallback) {
+    for (const span of spans) {
+      const [startSec, startNs] = span.startTime;
+      const [durSec, durNs] = span.duration;
+      fs.writeSync(outFd, JSON.stringify({
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+        // ReadableSpan.parentSpanId was replaced by parentSpanContext in
+        // the current @opentelemetry/sdk-trace* line this skill installs
+        // (span.parentSpanId is undefined there, so JSON.stringify silently
+        // dropped the key entirely) — read the id off parentSpanContext.
+        parentSpanId: span.parentSpanContext?.spanId ?? null,
+        name: span.name,
+        startTime: new Date(startSec * 1000 + startNs / 1e6).toISOString(),
+        duration_ms: durSec * 1000 + durNs / 1e6,
+        attributes: span.attributes,
+      }) + '\n');
+    }
+    resultCallback({ code: 0 });
+  }
+
+  shutdown() {
+    try {
+      fs.closeSync(outFd);
+    } catch {
+      // already closed (e.g. shutdown() invoked twice) — fine.
+    }
+    return Promise.resolve();
+  }
+}
+
+const spanProcessors = [new SimpleSpanProcessor(new JsonlFileSpanExporter())];
+
+if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) {
+  spanProcessors.push(
+    new SimpleSpanProcessor(
+      new OTLPTraceExporter({ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT }),
+    ),
+  );
+}
+
+const sdk = new NodeSDK({
+  spanProcessors,
+  instrumentations: [
+    getNodeAutoInstrumentations({
+      '@opentelemetry/instrumentation-fs': { enabled: false },
+    }),
+  ],
+});
+
+sdk.start();
+process.on('exit', () => sdk.shutdown());

package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts

@@ -0,0 +1,44 @@
+// otel_bootstrap/playwright-fixture.ts
+//
+// withTestSpan(base) wraps a Playwright `test` object with an `auto`
+// fixture that opens one root span per test (test.id/test.title/test.file
+// attributes), activates it as the OTel active context so the test's HTTP
+// calls nest as child spans (the whole correlation trick this skill relies
+// on), and closes it in teardown with test.outcome set from
+// testInfo.status. Merge into your own fixtures.ts:
+//
+//   import { test as base } from '@playwright/test';
+//   import { withTestSpan } from './otel_bootstrap/playwright-fixture';
+//   export const test = withTestSpan(base);
+//
+// Root-span-via-fixture (not a custom reporter) is deliberate — reporters
+// run in Playwright's main process and can't establish the OTel active
+// context the HTTP auto-instrumentation needs to nest child spans. See
+// docs/knowledge/decisions/0006-otel-test-side-tracing.md.
+
+import type { TestType } from '@playwright/test';
+import { trace, context } from '@opentelemetry/api';
+
+const tracer = trace.getTracer('canary-instrument');
+
+export function withTestSpan<T extends TestType<any, any>>(base: T): T {
+  return base.extend({
+    _rootSpan: [
+      async ({}, use, testInfo) => {
+        const span = tracer.startSpan(testInfo.title, {
+          attributes: {
+            'test.id': testInfo.titlePath.join(':'),
+            'test.title': testInfo.title,
+            'test.file': testInfo.file,
+          },
+        });
+        await context.with(trace.setSpan(context.active(), span), async () => {
+          await use();
+        });
+        span.setAttribute('test.outcome', testInfo.status ?? 'unknown');
+        span.end();
+      },
+      { auto: true },
+    ] as any,
+  }) as T;
+}

package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs

@@ -0,0 +1,81 @@
+// run_types -- the run.json v1 contract (trace-only), ported from the Python
+// dataclasses one-for-one as the skill moves to JS.
+//
+// Each factory returns a plain object whose keys are declared in exactly the
+// order the Python dataclass declared its fields, so JSON.stringify emits the
+// same key order the old dataclasses.asdict() did. `toDict()` is therefore an
+// identity pass -- the object already IS the on-disk dict shape.
+//
+// No `coverage` key and no `canary_run_id` key exist anywhere in this module:
+// both were cut for v1 (coverage is a separate future skill; canary_run_id has
+// no consumer yet). Additive-only evolution: new optional fields may be
+// appended later; existing fields never change meaning.
+
+/**
+ * @typedef {{method: string, url: string, route: (string|null),
+ *   status: (number|null), duration_ms: number, span_id: string,
+ *   started_at: string}} RequestSpanRow
+ */
+
+/**
+ * @typedef {{test_id: string, test_title: string, test_file: string,
+ *   trace_id: string, outcome: string, requests: RequestSpanRow[]}} TestTraceRow
+ */
+
+/** One outbound HTTP request span. Field order matches the Python dataclass. */
+export function RequestSpan({
+  method,
+  url,
+  route,
+  status,
+  duration_ms,
+  span_id,
+  started_at,
+}) {
+  return { method, url, route, status, duration_ms, span_id, started_at };
+}
+
+/**
+ * One test's trace bucket. `test_id` is "__setup__" for orphan (rootless)
+ * traffic. `requests` defaults to a fresh empty array (mirrors the Python
+ * `field(default_factory=list)`).
+ * @param {{test_id: string, test_title: string, test_file: string,
+ *   trace_id: string, outcome: string, requests?: RequestSpanRow[]}} f
+ */
+export function TestTrace({
+  test_id,
+  test_title,
+  test_file,
+  trace_id,
+  outcome,
+  requests = [],
+}) {
+  return { test_id, test_title, test_file, trace_id, outcome, requests };
+}
+
+/**
+ * The trace block: total request spans + per-test buckets.
+ * @param {{spans_total: number, by_test?: TestTraceRow[]}} f
+ */
+export function Trace({ spans_total, by_test = [] }) {
+  return { spans_total, by_test };
+}
+
+/** The top-level run.json artifact. */
+export function RunArtifact({
+  schema_version,
+  suite_type,
+  generated_at,
+  trace,
+}) {
+  return { schema_version, suite_type, generated_at, trace };
+}
+
+/**
+ * Mirror of the Python `RunArtifact.to_dict()` (a plain `asdict()`). Because
+ * the factories already build plain nested objects in field order, this is an
+ * identity function -- kept for API parity with the Python contract.
+ */
+export function toDict(artifact) {
+  return artifact;
+}