canary-test-cli 7.0.0 → 7.2.0

package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs

@@ -0,0 +1,178 @@
+#!/usr/bin/env node
+// canary-instrument -- correlate a Playwright run's tests to their outbound
+// HTTP spans. Ported from cli.py, behavior-preserving.
+//
+// Reads OTel span JSONL files written by otel_bootstrap/instrument.mjs (see
+// SKILL.md for the two manual wiring steps), resolves each Playwright test's
+// root span (set by otel_bootstrap/playwright-fixture.ts), attaches HTTP child
+// spans, and writes a run.json v1 artifact (trace-only; see run_types.mjs for
+// the contract).
+//
+// Invoked via:
+//   canary skills run canary-instrument -- \
+//     --spans test-results/trace --output test-results [--suite-type e2e_ui]
+//
+// Missing/empty --spans is not a failure -- it produces an empty trace block.
+// Self-contained -- no external skill dependency.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+
+import { RunArtifact } from './run_types.mjs';
+import { readTraces } from './span_reader.mjs';
+
+const USAGE =
+  'usage: canary-instrument [-h] --spans PATH --output PATH\n' +
+  '                         [--suite-type TYPE]\n' +
+  '\n' +
+  "Correlate a Playwright run's tests to their outbound HTTP spans and write a\n" +
+  'run.json v1 artifact.\n' +
+  '\n' +
+  'options:\n' +
+  '  -h, --help         show this help message and exit\n' +
+  '  --spans PATH       directory of OTel span JSONL files to read\n' +
+  '  --output PATH      directory to write run.json into (created if missing)\n' +
+  '  --suite-type TYPE  suite label recorded in the artifact (e.g. e2e_ui)';
+
+/**
+ * Both value flags are required, and `--suite-type` is a free-form label. Two
+ * deliberate divergences from argparse survive the move to the shared parser
+ * (#479): flags must be spelled in full (no prefix abbreviation, so `--out` is
+ * an unknown flag rather than `--output`), and an empty value is now REJECTED
+ * rather than passed through -- `--spans=` used to yield an empty trace, which
+ * is the missing-value case wearing a disguise.
+ */
+export const CLI_SPEC = {
+  prog: 'canary-instrument',
+  values: {
+    '--spans': { key: 'spans' },
+    '--output': { key: 'output' },
+    '--suite-type': { key: 'suiteType' },
+  },
+  defaults: { suiteType: '' },
+  required: ['--spans', '--output'],
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+/**
+ * Serialize like Python's json.dumps(obj, indent=2): 2-space indent AND
+ * ensure_ascii=True. JSON.stringify already matches the indentation and empty
+ * container ("[]"/"{}") forms, but leaves non-ASCII raw -- so escape every code
+ * unit above 0x7F to a \uXXXX sequence (astral chars become their two UTF-16
+ * surrogate escapes, exactly as Python emits them). No trailing newline,
+ * matching cli.py's write_text.
+ */
+function dumpsIndent2Ascii(obj) {
+  const nonAscii = /[\u0080-\uffff]/g;
+  return JSON.stringify(obj, null, 2).replace(
+    nonAscii,
+    (ch) => `\\u${ch.charCodeAt(0).toString(16).padStart(4, '0')}`,
+  );
+}
+
+/**
+ * Python's datetime.now(timezone.utc).isoformat() renders a "+00:00" offset;
+ * JS toISOString() renders "Z". Normalize the suffix so generated_at keeps the
+ * documented "+00:00" convention. (Python emits microseconds, JS milliseconds;
+ * generated_at is a wall-clock stamp and is never asserted against.)
+ */
+function nowIso() {
+  return new Date().toISOString().replace('Z', '+00:00');
+}
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  const spansDir = args.spans;
+  if (existsButNotDir(spansDir)) {
+    console.error(`canary-instrument: --spans is not a directory: ${spansDir}`);
+    return 1;
+  }
+
+  const trace = readTraces(spansDir);
+  const artifact = RunArtifact({
+    schema_version: 1,
+    suite_type: args.suiteType,
+    generated_at: nowIso(),
+    trace,
+  });
+
+  // A run.json whose requests carry no URL is worse than no run.json. The
+  // artifact's whole purpose is answering "which test hit which endpoint", so a
+  // consumer that reads URL-less requests computes zero endpoints matched and
+  // renders a clean 0% as though it were a measurement. Refuse to write it.
+  //
+  // Note this is NOT the empty-spans case, which is deliberately fine (see the
+  // header): zero spans yields an empty trace and exit 0, and "0 spans, 0
+  // buckets" describes itself honestly. What fails here is the drift signature --
+  // spans WERE captured and correlated, and every one of them lost its URL,
+  // which is what an attribute-name mismatch looks like from the outside.
+  const requests = trace.by_test.flatMap((t) => t.requests ?? []);
+  const withUrl = requests.filter((r) => r.url);
+  if (requests.length > 0 && withUrl.length === 0) {
+    console.error(
+      `canary-instrument: correlated ${requests.length} request(s) across ` +
+        `${trace.spans_total} span(s) and NONE carried a URL. Refusing to write ` +
+        `an artifact that would read as "no endpoints exercised".`,
+    );
+    console.error(
+      'This is what an OTel semantic-convention mismatch looks like: the spans ' +
+        'are present and attributed, but the URL attribute this reader expects is ' +
+        'not the one the instrumentation emitted. Check the attribute keys on an ' +
+        'HTTP span in --spans against requestUrl() in span_reader.mjs.',
+    );
+    return 1;
+  }
+
+  const outputDir = args.output;
+  fs.mkdirSync(outputDir, { recursive: true });
+  const outPath = path.join(outputDir, 'run.json');
+  fs.writeFileSync(outPath, dumpsIndent2Ascii(artifact), 'utf8');
+
+  console.log(
+    `canary-instrument: wrote ${outPath} ` +
+      `(${trace.spans_total} spans, ${trace.by_test.length} test buckets, ` +
+      `${withUrl.length}/${requests.length} request(s) with a URL)`,
+  );
+  // Partial loss is advisory: a mixed run is still usable, but silence would let
+  // a growing blind spot pass for a shrinking one.
+  if (withUrl.length < requests.length) {
+    console.warn(
+      `canary-instrument: ${requests.length - withUrl.length} request(s) had no ` +
+        'resolvable URL and will not match any endpoint.',
+    );
+  }
+  return 0;
+}
+
+function existsButNotDir(p) {
+  try {
+    return !fs.statSync(p).isDirectory();
+  } catch {
+    return false; // missing path is fine -- yields an empty trace
+  }
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
+// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated JSON that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs

@@ -0,0 +1,96 @@
+// otel_bootstrap/instrument.mjs
+//
+// Node OTel SDK bootstrap for canary-instrument. Import via:
+//   NODE_OPTIONS="--import ./otel_bootstrap/instrument.mjs" npx playwright test
+//
+// Default: writes one JSON span per line to
+//   test-results/trace/otel-spans.<TEST_WORKER_INDEX>.jsonl
+// (no collector required — this is what scripts/span_reader.mjs reads).
+// When OTEL_EXPORTER_OTLP_ENDPOINT is set, spans are *additionally*
+// streamed to that collector via OTLPTraceExporter; the file path above is
+// unaffected either way.
+//
+// Auto-instruments HTTP/undici only — fs instrumentation is disabled so
+// Playwright's own file I/O doesn't show up as noise spans.
+//
+// Consumer-supplied dependencies (not vendored by this skill):
+//   @opentelemetry/sdk-node @opentelemetry/api
+//   @opentelemetry/auto-instrumentations-node
+//   @opentelemetry/exporter-trace-otlp-http
+
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const workerIndex = process.env.TEST_WORKER_INDEX ?? '0';
+const outDir = path.join(process.cwd(), 'test-results', 'trace');
+fs.mkdirSync(outDir, { recursive: true });
+// Synchronous fd, not fs.createWriteStream: Playwright's worker process
+// calls process.exit() once its assigned tests finish (it does not wait for
+// the event loop to drain), which cuts off any pending async stream writes
+// before they reach disk. fs.writeSync makes each export() call durable
+// immediately, so span data survives that hard exit — see
+// docs/changes/canary-instrument/proposal.md's ADR reference for why this
+// skill can't assume a graceful shutdown hook will run.
+const outFd = fs.openSync(
+  path.join(outDir, `otel-spans.${workerIndex}.jsonl`),
+  'a',
+);
+
+/** Minimal file exporter — one JSON span per line, matches span_reader.mjs. */
+class JsonlFileSpanExporter {
+  export(spans, resultCallback) {
+    for (const span of spans) {
+      const [startSec, startNs] = span.startTime;
+      const [durSec, durNs] = span.duration;
+      fs.writeSync(outFd, JSON.stringify({
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+        // ReadableSpan.parentSpanId was replaced by parentSpanContext in
+        // the current @opentelemetry/sdk-trace* line this skill installs
+        // (span.parentSpanId is undefined there, so JSON.stringify silently
+        // dropped the key entirely) — read the id off parentSpanContext.
+        parentSpanId: span.parentSpanContext?.spanId ?? null,
+        name: span.name,
+        startTime: new Date(startSec * 1000 + startNs / 1e6).toISOString(),
+        duration_ms: durSec * 1000 + durNs / 1e6,
+        attributes: span.attributes,
+      }) + '\n');
+    }
+    resultCallback({ code: 0 });
+  }
+
+  shutdown() {
+    try {
+      fs.closeSync(outFd);
+    } catch {
+      // already closed (e.g. shutdown() invoked twice) — fine.
+    }
+    return Promise.resolve();
+  }
+}
+
+const spanProcessors = [new SimpleSpanProcessor(new JsonlFileSpanExporter())];
+
+if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) {
+  spanProcessors.push(
+    new SimpleSpanProcessor(
+      new OTLPTraceExporter({ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT }),
+    ),
+  );
+}
+
+const sdk = new NodeSDK({
+  spanProcessors,
+  instrumentations: [
+    getNodeAutoInstrumentations({
+      '@opentelemetry/instrumentation-fs': { enabled: false },
+    }),
+  ],
+});
+
+sdk.start();
+process.on('exit', () => sdk.shutdown());

package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts

@@ -0,0 +1,44 @@
+// otel_bootstrap/playwright-fixture.ts
+//
+// withTestSpan(base) wraps a Playwright `test` object with an `auto`
+// fixture that opens one root span per test (test.id/test.title/test.file
+// attributes), activates it as the OTel active context so the test's HTTP
+// calls nest as child spans (the whole correlation trick this skill relies
+// on), and closes it in teardown with test.outcome set from
+// testInfo.status. Merge into your own fixtures.ts:
+//
+//   import { test as base } from '@playwright/test';
+//   import { withTestSpan } from './otel_bootstrap/playwright-fixture';
+//   export const test = withTestSpan(base);
+//
+// Root-span-via-fixture (not a custom reporter) is deliberate — reporters
+// run in Playwright's main process and can't establish the OTel active
+// context the HTTP auto-instrumentation needs to nest child spans. See
+// docs/knowledge/decisions/0006-otel-test-side-tracing.md.
+
+import type { TestType } from '@playwright/test';
+import { trace, context } from '@opentelemetry/api';
+
+const tracer = trace.getTracer('canary-instrument');
+
+export function withTestSpan<T extends TestType<any, any>>(base: T): T {
+  return base.extend({
+    _rootSpan: [
+      async ({}, use, testInfo) => {
+        const span = tracer.startSpan(testInfo.title, {
+          attributes: {
+            'test.id': testInfo.titlePath.join(':'),
+            'test.title': testInfo.title,
+            'test.file': testInfo.file,
+          },
+        });
+        await context.with(trace.setSpan(context.active(), span), async () => {
+          await use();
+        });
+        span.setAttribute('test.outcome', testInfo.status ?? 'unknown');
+        span.end();
+      },
+      { auto: true },
+    ] as any,
+  }) as T;
+}

package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs

@@ -0,0 +1,81 @@
+// run_types -- the run.json v1 contract (trace-only), ported from the Python
+// dataclasses one-for-one as the skill moves to JS.
+//
+// Each factory returns a plain object whose keys are declared in exactly the
+// order the Python dataclass declared its fields, so JSON.stringify emits the
+// same key order the old dataclasses.asdict() did. `toDict()` is therefore an
+// identity pass -- the object already IS the on-disk dict shape.
+//
+// No `coverage` key and no `canary_run_id` key exist anywhere in this module:
+// both were cut for v1 (coverage is a separate future skill; canary_run_id has
+// no consumer yet). Additive-only evolution: new optional fields may be
+// appended later; existing fields never change meaning.
+
+/**
+ * @typedef {{method: string, url: string, route: (string|null),
+ *   status: (number|null), duration_ms: number, span_id: string,
+ *   started_at: string}} RequestSpanRow
+ */
+
+/**
+ * @typedef {{test_id: string, test_title: string, test_file: string,
+ *   trace_id: string, outcome: string, requests: RequestSpanRow[]}} TestTraceRow
+ */
+
+/** One outbound HTTP request span. Field order matches the Python dataclass. */
+export function RequestSpan({
+  method,
+  url,
+  route,
+  status,
+  duration_ms,
+  span_id,
+  started_at,
+}) {
+  return { method, url, route, status, duration_ms, span_id, started_at };
+}
+
+/**
+ * One test's trace bucket. `test_id` is "__setup__" for orphan (rootless)
+ * traffic. `requests` defaults to a fresh empty array (mirrors the Python
+ * `field(default_factory=list)`).
+ * @param {{test_id: string, test_title: string, test_file: string,
+ *   trace_id: string, outcome: string, requests?: RequestSpanRow[]}} f
+ */
+export function TestTrace({
+  test_id,
+  test_title,
+  test_file,
+  trace_id,
+  outcome,
+  requests = [],
+}) {
+  return { test_id, test_title, test_file, trace_id, outcome, requests };
+}
+
+/**
+ * The trace block: total request spans + per-test buckets.
+ * @param {{spans_total: number, by_test?: TestTraceRow[]}} f
+ */
+export function Trace({ spans_total, by_test = [] }) {
+  return { spans_total, by_test };
+}
+
+/** The top-level run.json artifact. */
+export function RunArtifact({
+  schema_version,
+  suite_type,
+  generated_at,
+  trace,
+}) {
+  return { schema_version, suite_type, generated_at, trace };
+}
+
+/**
+ * Mirror of the Python `RunArtifact.to_dict()` (a plain `asdict()`). Because
+ * the factories already build plain nested objects in field order, this is an
+ * identity function -- kept for API parity with the Python contract.
+ */
+export function toDict(artifact) {
+  return artifact;
+}

package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs

@@ -0,0 +1,187 @@
+// span_reader -- merge/correlate OTel span JSONL files into a Trace (pure,
+// read-only). Ported from Python span_reader.py, behavior-for-behavior.
+//
+// Reads one or more `otel-spans.<worker>.jsonl` files (one JSON span object
+// per line, written by otel_bootstrap/instrument.mjs), groups spans by
+// `traceId`, resolves each trace's root span (the one carrying a `test.id`
+// attribute -- set by otel_bootstrap/playwright-fixture.ts's root-span
+// fixture), and attaches that trace's HTTP child spans to the resolved test.
+// Traces with no `test.id`-attributed root bucket their HTTP spans under the
+// synthetic test id "__setup__" (traffic outside any test, e.g. global setup).
+//
+// Assumed span envelope: {traceId, spanId, parentSpanId, name, startTime,
+// duration_ms, attributes{}} -- matches exactly what instrument.mjs's
+// JsonlFileSpanExporter writes; no `endTime` key is emitted (duration_ms +
+// startTime cover it). HTTP attributes keyed http.method/http.request.method,
+// http.url, http.route, http.status_code.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { RequestSpan, TestTrace, Trace } from './run_types.mjs';
+
+const SETUP_TEST_ID = '__setup__';
+
+// Mirrors Python's Path.glob("otel-spans.*.jsonl"): the `*` matches any run of
+// characters (including none) within a single path segment.
+const SPAN_FILE_RE = /^otel-spans\..*\.jsonl$/;
+
+/**
+ * Read every `otel-spans.*.jsonl` file under `spansDir`, correlate spans to
+ * their test roots, and return the Trace. A missing or non-directory path
+ * yields an empty trace (never throws) -- the same as the Python version.
+ * @param {string} spansDir
+ */
+export function readTraces(spansDir) {
+  const byTrace = new Map();
+
+  if (isDir(spansDir)) {
+    const files = fs
+      .readdirSync(spansDir)
+      .filter((name) => SPAN_FILE_RE.test(name))
+      .sort();
+    for (const name of files) {
+      for (const span of readJsonl(path.join(spansDir, name))) {
+        const traceId = span.traceId;
+        if (!traceId) continue;
+        if (!byTrace.has(traceId)) byTrace.set(traceId, []);
+        byTrace.get(traceId).push(span);
+      }
+    }
+  }
+
+  const byTest = [];
+  const setupRequests = [];
+  let spansTotal = 0;
+
+  for (const [traceId, spans] of byTrace) {
+    const root = spans.find(isTestRoot) ?? null;
+    const httpSpans = spans.filter((s) => s !== root && isHttpSpan(s));
+    const requests = httpSpans.map(toRequestSpan);
+    spansTotal += requests.length;
+
+    if (root === null) {
+      setupRequests.push(...requests);
+      continue;
+    }
+
+    const attrs = root.attributes ?? {};
+    byTest.push(
+      TestTrace({
+        test_id: attrs['test.id'] ?? '',
+        test_title: attrs['test.title'] ?? '',
+        test_file: attrs['test.file'] ?? '',
+        trace_id: traceId,
+        outcome: attrs['test.outcome'] ?? '',
+        requests,
+      }),
+    );
+  }
+
+  if (setupRequests.length) {
+    byTest.push(
+      TestTrace({
+        test_id: SETUP_TEST_ID,
+        test_title: '',
+        test_file: '',
+        trace_id: '',
+        outcome: '',
+        requests: setupRequests,
+      }),
+    );
+  }
+
+  return Trace({ spans_total: spansTotal, by_test: byTest });
+}
+
+function isDir(p) {
+  try {
+    return fs.statSync(p).isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+function readJsonl(filePath) {
+  const spans = [];
+  // NDJSON records are \n- or \r\n-delimited. We intentionally do NOT split on
+  // the exotic Unicode line separators that Python's str.splitlines() also
+  // breaks on (U+2028, U+2029, U+0085, ...): instrument.mjs writes those RAW
+  // inside JSON string values, so splitting on them would fragment a valid
+  // record and misattribute its request to __setup__.
+  for (const raw of fs.readFileSync(filePath, 'utf8').split(/\r\n|\n/)) {
+    const line = raw.trim();
+    if (!line) continue;
+    try {
+      spans.push(JSON.parse(line));
+    } catch {
+      continue; // malformed/torn line (e.g. a crashed worker's last write)
+    }
+  }
+  return spans;
+}
+
+function isTestRoot(span) {
+  return 'test.id' in (span.attributes ?? {});
+}
+
+function isHttpSpan(span) {
+  const attrs = span.attributes ?? {};
+  return 'http.method' in attrs || 'http.request.method' in attrs;
+}
+
+/**
+ * Reconstruct the request URL across both OTel HTTP semantic conventions.
+ *
+ * The old convention put the whole thing in one attribute, `http.url`. The
+ * current one (stable since semconv 1.23) splits it: `url.full` on some
+ * instrumentations, and otherwise the pieces -- `url.scheme`, `server.address`,
+ * `server.port`, `url.path`. `@opentelemetry/auto-instrumentations-node`, which
+ * `otel_bootstrap/instrument.mjs` tells consumers to install, emits the SPLIT
+ * form and no `http.url` at all.
+ *
+ * Reading only `http.url` therefore produced `url: ""` on every request while
+ * the reader still reported a full span count and exit 0 -- the artifact looked
+ * written and was empty of the one field a coverage consumer needs. Port is
+ * omitted when it is the scheme default, so the URL matches what a spec or a
+ * HAR would say.
+ */
+function requestUrl(attrs) {
+  const direct = attrs['url.full'] ?? attrs['http.url'];
+  if (direct) return direct;
+
+  const scheme = attrs['url.scheme'] ?? attrs['http.scheme'];
+  const host =
+    attrs['server.address'] ?? attrs['net.peer.name'] ?? attrs['http.host'];
+  const path = attrs['url.path'] ?? attrs['http.target'] ?? '';
+  if (!scheme || !host) return typeof path === 'string' ? path : '';
+
+  const port = attrs['server.port'] ?? attrs['net.peer.port'];
+  const isDefaultPort =
+    port == null ||
+    (scheme === 'http' && +port === 80) ||
+    (scheme === 'https' && +port === 443);
+  const authority = isDefaultPort ? host : `${host}:${port}`;
+  const query = attrs['url.query'] ? `?${attrs['url.query']}` : '';
+  return `${scheme}://${authority}${path}${query}`;
+}
+
+function toRequestSpan(span) {
+  const attrs = span.attributes ?? {};
+  const method = attrs['http.method'] || attrs['http.request.method'] || '';
+  return RequestSpan({
+    method,
+    url: requestUrl(attrs),
+    // `http.route` is a SERVER-side attribute and is never present on the client
+    // spans this reader consumes. Kept for producers that do supply it (a future
+    // server-side producer under the same v1 contract), but its absence here is
+    // normal, not a gap.
+    route: attrs['http.route'] ?? null,
+    // Renamed in the current convention. Old name kept as a fallback.
+    status:
+      attrs['http.response.status_code'] ?? attrs['http.status_code'] ?? null,
+    duration_ms: span.duration_ms ?? 0,
+    span_id: span.spanId ?? '',
+    started_at: span.startTime ?? '',
+  });
+}