canary-test-cli 5.15.0 → 6.0.0

package/dist/overlay-conflicts.d.ts

@@ -0,0 +1,33 @@
+import type { OverlayEntry, OverlayRegistry } from './overlays-registry.js';
+/** One overlay contending for a skill name, with its effective precedence. */
+export interface Contender {
+    overlay: string;
+    /** Declared precedence, or 0 when the entry left it null/absent. */
+    precedence: number;
+}
+/** A skill name shipped by two or more registered overlays. */
+export interface SkillConflict {
+    skill: string;
+    /** All contending overlays, ordered by precedence desc then name asc. */
+    contenders: Contender[];
+    /** The overlay that wins by declared precedence, or null when it is a tie. */
+    winner: string | null;
+    /** True iff a single overlay holds the highest precedence. */
+    resolved: boolean;
+}
+export interface ConflictDeps {
+    /** List the skill (directory) names an overlay ships. Injectable for tests. */
+    readSkillNames?: (entry: OverlayEntry) => string[];
+}
+/**
+ * Skill (directory) names an overlay ships, read from
+ * `<overlay>/.canary/skills/<name>/SKILL.md`. A missing or unreadable skills
+ * dir yields no names (never throws) — mirrors the Python loader's tolerance.
+ */
+export declare function readOverlaySkillNames(entry: OverlayEntry): string[];
+/**
+ * Detect skill-name collisions across the registered overlays. Returns one
+ * {@link SkillConflict} per skill name shipped by two or more overlays, sorted
+ * by skill name.
+ */
+export declare function detectSkillConflicts(reg: OverlayRegistry, deps?: ConflictDeps): SkillConflict[];

package/dist/overlay-lint.d.ts

@@ -0,0 +1,19 @@
+/** Migration target shapes a `deploy_to` entry may name, plus the `all` sentinel. */
+export declare const VALID_DEPLOY_TARGETS: ReadonlySet<string>;
+export interface LintFinding {
+    /** Skill name, or `(overlay)` for an overlay-level finding. */
+    skill: string;
+    level: 'error' | 'warning';
+    message: string;
+}
+export interface LintResult {
+    overlay: string;
+    skillsChecked: number;
+    findings: LintFinding[];
+}
+/**
+ * Lint an overlay clone at `overlayPath`. Returns every finding; the caller
+ * decides how to render/exit. Never throws on a malformed overlay — a missing
+ * skills dir is itself an error finding.
+ */
+export declare function lintOverlay(overlayPath: string): LintResult;

package/dist/overlays-registry.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Reader/writer for `~/.canary/overlays.json` — the tracked-overlay registry.
+ *
+ * Cross-runtime contract (spec Assumption): this file is written ONLY by the
+ * TS side. The Python skill loader never parses it; it directory-scans the
+ * `~/.canary/overlays/<name>/` clone dirs instead. So the schema here serves
+ * the `overlay` commands alone.
+ */
+export declare const SCHEMA_VERSION = 1;
+export interface OverlayEntry {
+    /** Registry key and clone dir name: `<owner>-<repo>`. */
+    name: string;
+    /** The source spec as the user gave it (`github:...`, URL, or path). */
+    source: string;
+    /** Pinned tag/branch, or null to track the remote default branch. */
+    ref: string | null;
+    /** Absolute clone path under `~/.canary/overlays/`. */
+    path: string;
+    /** ISO date the overlay was added. */
+    addedDate: string;
+    /**
+     * Consent for this overlay's `command-succeeds` doctor checks: true when the
+     * user allowed them at `overlay add`, false when declined, null when the
+     * overlay ships no such checks (nothing to gate).
+     */
+    consent: boolean | null;
+    /**
+     * Fingerprint of the `command-succeeds` set that `consent` was granted for,
+     * or null when there are no such checks. Consent is re-requested when the
+     * live manifest's fingerprint no longer matches this.
+     */
+    consentCommandsHash: string | null;
+    /**
+     * Declared arbitration priority when two overlays ship the same skill name
+     * (#333). Higher wins; null/absent is treated as 0. A collision is only
+     * *resolved* when exactly one contending overlay holds the highest
+     * precedence — otherwise which definition wins is accidental and `doctor`
+     * flags it. Both runtimes read this: TS diagnostics and the Python skill
+     * loader must agree on the winner.
+     */
+    precedence: number | null;
+}
+export interface OverlayRegistry {
+    schemaVersion: number;
+    overlays: OverlayEntry[];
+}
+/** Raised when the registry file exists but cannot be read or parsed. */
+export declare class RegistryError extends Error {
+    constructor(message: string);
+}
+export declare function canaryHome(homeDir?: string): string;
+export declare function registryPath(homeDir?: string): string;
+export declare function overlaysDir(homeDir?: string): string;
+export declare function clonePath(name: string, homeDir?: string): string;
+export declare function emptyRegistry(): OverlayRegistry;
+export declare function read(homeDir?: string): OverlayRegistry;
+/**
+ * Whether `command-succeeds` doctor checks may run for an overlay: consent must
+ * have been granted (`true`) AND still cover the live manifest's command set
+ * (`liveHash` matching the recorded fingerprint). A changed manifest (hash
+ * mismatch) revokes consent until it is re-confirmed at `overlay add`.
+ */
+export declare function consentGranted(entry: OverlayEntry, liveHash: string | null): boolean;
+/** Write the registry atomically (temp file + rename), creating `~/.canary`. */
+export declare function write(registry: OverlayRegistry, homeDir?: string): void;
+export declare function get(registry: OverlayRegistry, name: string): OverlayEntry | null;
+export declare function list(registry: OverlayRegistry): OverlayEntry[];
+/** Add an entry. Throws if one with the same name is already registered. */
+export declare function add(registry: OverlayRegistry, entry: OverlayEntry): OverlayRegistry;
+/** Remove an entry by name. Returns the new registry and whether it existed. */
+export declare function remove(registry: OverlayRegistry, name: string): {
+    registry: OverlayRegistry;
+    removed: boolean;
+};

package/dist/reporters/testtracker.d.ts

@@ -0,0 +1,89 @@
+import type { FullResult, Reporter, TestCase, TestResult } from "@playwright/test/reporter";
+export interface TestTrackerReporterOptions {
+    suite?: string;
+    /** Prefix stripped from an absolute test file path. Default: `<cwd>/`. */
+    testFilePrefix?: string;
+    /** Explicit environment label; else derived from env at push time. */
+    environment?: string;
+    url?: string;
+    token?: string;
+    workflow?: string;
+}
+export interface ResolvedConfig {
+    suite: string;
+    testFilePrefix: string;
+    environment?: string;
+    url: string;
+    token: string;
+    workflow: string;
+}
+export interface ResultEntry {
+    full_title: string;
+    test_file: string;
+    status: string;
+    duration_ms?: number;
+    error_message?: string;
+    error_stack?: string;
+    retries: number;
+    tags: string[];
+}
+/**
+ * The wire contract POSTed to `/api/ingest/runs`. Declared explicitly (rather
+ * than inferred from the object literal) so `declaration: true` ships a stable,
+ * reviewable public type and an accidental field change is a compile error.
+ */
+export interface IngestPayload {
+    canary_run_id: string;
+    suite: string;
+    branch: string;
+    commit_sha?: string;
+    workflow: string;
+    environment?: string;
+    status: "passed" | "failed" | "flaky" | "cancelled";
+    started_at: string;
+    finished_at: string;
+    totals: {
+        passed: number;
+        failed: number;
+        flaky: number;
+        skipped: number;
+        total: number;
+    };
+    results: ResultEntry[];
+}
+export declare function mapStatus(pw: string): string;
+/**
+ * Final per-test status, flaky-aware. Playwright's per-attempt
+ * `result.status` is NEVER "flaky" — flakiness is derived at the test level
+ * (a test that failed then passed on retry). `test.outcome()` is the canonical
+ * signal, so a recovered flake is reported as "flaky" (SDET-visible) rather
+ * than the last attempt's "passed". A recovered flake does NOT count as a
+ * failure at the run level (see buildPayload) — management sees a good run.
+ */
+export declare function resolveTestStatus(outcome: string, lastAttemptStatus: string): string;
+export declare function resolveConfig(opts?: TestTrackerReporterOptions, env?: NodeJS.ProcessEnv): ResolvedConfig;
+export declare function shouldPush(cfg: Pick<ResolvedConfig, "url" | "token">, env?: NodeJS.ProcessEnv): boolean;
+/**
+ * Overall run status. When Playwright reports the run as interrupted or timed
+ * out, the run did NOT complete — do not derive a green status from whatever
+ * per-test buckets happened to fill before the abort. `fullResultStatus` is
+ * Playwright's `FullResult.status` (`passed` | `failed` | `timedout` | `interrupted`).
+ */
+export declare function runStatus(totals: {
+    failed: number;
+    flaky: number;
+}, fullResultStatus?: string): "passed" | "failed" | "flaky" | "cancelled";
+export declare function buildPayload(results: ResultEntry[], cfg: ResolvedConfig, timing: {
+    startedAt: string;
+    finishedAt: string;
+}, env?: NodeJS.ProcessEnv, fullResultStatus?: string): IngestPayload;
+export default class TestTrackerReporter implements Reporter {
+    private options;
+    private results;
+    private startTime;
+    private cfg;
+    constructor(options?: TestTrackerReporterOptions);
+    onBegin(): void;
+    onTestEnd(test: TestCase, result: TestResult): void;
+    onEnd(result: FullResult): Promise<void>;
+}

package/dist/reporters/testtracker.js

@@ -0,0 +1,195 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapStatus = mapStatus;
+exports.resolveTestStatus = resolveTestStatus;
+exports.resolveConfig = resolveConfig;
+exports.shouldPush = shouldPush;
+exports.runStatus = runStatus;
+exports.buildPayload = buildPayload;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+// Optional .env load — MUST NOT crash the suite if dotenv is absent.
+try {
+    require("dotenv").config();
+}
+catch {
+    /* dotenv not installed — use ambient env */
+}
+function mapStatus(pw) {
+    switch (pw) {
+        case "passed":
+            return "passed";
+        case "failed":
+        case "timedOut":
+            return "failed";
+        case "flaky":
+            return "flaky";
+        case "skipped":
+        case "interrupted":
+            return "skipped";
+        default:
+            return "skipped";
+    }
+}
+/**
+ * Final per-test status, flaky-aware. Playwright's per-attempt
+ * `result.status` is NEVER "flaky" — flakiness is derived at the test level
+ * (a test that failed then passed on retry). `test.outcome()` is the canonical
+ * signal, so a recovered flake is reported as "flaky" (SDET-visible) rather
+ * than the last attempt's "passed". A recovered flake does NOT count as a
+ * failure at the run level (see buildPayload) — management sees a good run.
+ */
+function resolveTestStatus(outcome, lastAttemptStatus) {
+    if (outcome === "flaky")
+        return "flaky";
+    return mapStatus(lastAttemptStatus);
+}
+function resolveConfig(opts = {}, env = process.env) {
+    const suite = opts.suite ?? env.TESTTRACKER_SUITE;
+    if (!suite) {
+        throw new Error("TestTrackerReporter: `suite` is required (option or TESTTRACKER_SUITE).");
+    }
+    return {
+        suite,
+        testFilePrefix: opts.testFilePrefix ?? env.TESTTRACKER_TEST_FILE_PREFIX ?? `${process.cwd()}/`,
+        environment: opts.environment ?? env.TESTTRACKER_ENVIRONMENT,
+        url: opts.url ?? env.TESTTRACKER_URL ?? "",
+        token: opts.token ?? env.TESTTRACKER_API_TOKEN ?? "",
+        workflow: opts.workflow ?? env.TESTTRACKER_WORKFLOW ?? "playwright",
+    };
+}
+function shouldPush(cfg, env = process.env) {
+    if (!cfg.url || !cfg.token)
+        return false;
+    const isCI = env.CI === "true" || env.GITHUB_ACTIONS === "true";
+    const force = env.TESTTRACKER_PUSH === "true";
+    return isCI || force;
+}
+function stableRunId(env) {
+    const runId = env.GITHUB_RUN_ID;
+    if (runId)
+        return env.GITHUB_RUN_ATTEMPT ? `${runId}-${env.GITHUB_RUN_ATTEMPT}` : runId;
+    const sha = env.GITHUB_SHA?.slice(0, 7);
+    return (sha ? `${sha}-` : "local-") + node_crypto_1.default.randomUUID();
+}
+/**
+ * Overall run status. When Playwright reports the run as interrupted or timed
+ * out, the run did NOT complete — do not derive a green status from whatever
+ * per-test buckets happened to fill before the abort. `fullResultStatus` is
+ * Playwright's `FullResult.status` (`passed` | `failed` | `timedout` | `interrupted`).
+ */
+function runStatus(totals, fullResultStatus) {
+    if (fullResultStatus === "interrupted")
+        return "cancelled";
+    if (fullResultStatus === "timedout" || fullResultStatus === "failed")
+        return "failed";
+    // "passed" or unknown → trust the per-test buckets.
+    if (totals.failed > 0)
+        return "failed";
+    if (totals.flaky > 0)
+        return "flaky";
+    return "passed";
+}
+function buildPayload(results, cfg, timing, env = process.env, fullResultStatus) {
+    const count = (s) => results.filter((r) => r.status === s).length;
+    const passed = count("passed");
+    const failed = count("failed");
+    const flaky = count("flaky");
+    const skipped = count("skipped");
+    return {
+        canary_run_id: stableRunId(env),
+        suite: cfg.suite,
+        branch: env.GITHUB_REF_NAME ?? "local",
+        commit_sha: env.GITHUB_SHA,
+        workflow: cfg.workflow,
+        environment: cfg.environment,
+        status: runStatus({ failed, flaky }, fullResultStatus),
+        started_at: timing.startedAt,
+        finished_at: timing.finishedAt,
+        totals: { passed, failed, flaky, skipped, total: results.length },
+        results,
+    };
+}
+class TestTrackerReporter {
+    options;
+    results = new Map();
+    startTime = Date.now();
+    cfg = null;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    onBegin() {
+        // Resolve config once; a config error here is logged, not thrown, so a
+        // misconfigured reporter never aborts the whole run.
+        try {
+            this.cfg = resolveConfig(this.options);
+        }
+        catch (err) {
+            console.log(`\nTestTracker: disabled — ${err instanceof Error ? err.message : String(err)}`);
+            this.cfg = null;
+        }
+    }
+    onTestEnd(test, result) {
+        if (!this.cfg)
+            return;
+        // A reporter hook must never fail the suite — guard the whole body.
+        try {
+            const fullTitle = test.titlePath().filter(Boolean).join(" > ");
+            // `test.tags` requires Playwright >= 1.42; guard for the peer floor.
+            const tags = (test.tags ?? []).map((t) => t.replace(/^@/, ""));
+            // Keyed by the session-unique `test.id` (NOT the title) so `--repeat-each`
+            // and duplicate-title executions don't collapse into one entry. Retries
+            // share one TestCase (same id), so last-write-wins for the final status
+            // and first-failing-attempt error preservation both still hold — a
+            // recovered flake keeps the error that shows the SDET why it flaked.
+            const prior = this.results.get(test.id);
+            this.results.set(test.id, {
+                full_title: fullTitle,
+                test_file: test.location.file.startsWith(this.cfg.testFilePrefix)
+                    ? test.location.file.slice(this.cfg.testFilePrefix.length)
+                    : test.location.file,
+                status: resolveTestStatus(test.outcome(), result.status),
+                duration_ms: result.duration,
+                error_message: prior?.error_message ?? result.errors[0]?.message,
+                error_stack: prior?.error_stack ?? result.errors[0]?.stack,
+                retries: result.retry,
+                tags,
+            });
+        }
+        catch (err) {
+            console.log(`\nTestTracker: skipped a result — ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    async onEnd(result) {
+        if (!this.cfg || !shouldPush(this.cfg))
+            return;
+        try {
+            const payload = buildPayload([...this.results.values()], this.cfg, {
+                startedAt: new Date(this.startTime).toISOString(),
+                finishedAt: new Date().toISOString(),
+            }, process.env, result?.status);
+            const resp = await fetch(`${this.cfg.url}/api/ingest/runs`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.cfg.token}`,
+                },
+                body: JSON.stringify(payload),
+            });
+            if (resp.ok) {
+                const data = (await resp.json().catch(() => ({})));
+                console.log(`\nTestTracker: run ${data.id ?? "?"} ingested${data.duplicate ? " (duplicate)" : ""}.`);
+            }
+            else {
+                const text = await resp.text().catch(() => "");
+                console.log(`\nTestTracker: push failed ${resp.status} — ${text.slice(0, 200)}`);
+            }
+        }
+        catch (err) {
+            console.log(`\nTestTracker: push error — ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+}
+exports.default = TestTrackerReporter;

package/dist/router.d.ts

@@ -0,0 +1,12 @@
+import type { CommandDeps } from './overlay-commands.js';
+/** Subcommands handled in TypeScript rather than forwarded to the binary. */
+export declare const TS_COMMANDS: readonly string[];
+/** True when `argv` (process.argv.slice(2)) targets a TS-handled command. */
+export declare function isTsCommand(argv: readonly string[]): boolean;
+/**
+ * Dispatch a TS-handled command. Returns the process exit code (or a Promise of
+ * one for async commands like `doctor`), or `null` when the command should fall
+ * through to the Python binary. `deps` is threaded to the command handlers for
+ * testing (real dependencies by default).
+ */
+export declare function route(argv: readonly string[], deps?: CommandDeps): number | Promise<number> | null;

package/dist/router.js

@@ -39,12 +39,12 @@ exports.route = route;
 /**
  * Router for TS-handled `canary` subcommands — the strangler seam (Decision 10).
  *
- * The npm shim (`bin/canary.js`) forwards every command to the bundled Python
- * binary except the ones handled here in TypeScript. Phase 1 wires `overlay`;
- * `doctor` slots into the same table in Phase 2.
+ * The npm shim (`bin/canary.js`) forwards every command to the bundled
+ * TypeScript engine (`dist/engine/cli.js`) except the ones handled here
+ * natively: `overlay` and `doctor`.
  *
  * `route()` returns a process exit code, or `null` when the command is not
- * TS-handled and the shim should fall through to the Python binary.
+ * TS-handled and the shim should fall through to the bundled engine.
  */
 const overlay = __importStar(require("./overlay-commands.js"));
 const doctor_js_1 = require("./doctor.js");

package/dist/skill-requirements.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Skill runtime-requirement verification (#336).
+ *
+ * Skills declare the runtime tools they need in SKILL.md frontmatter:
+ *
+ *     requires: [python3>=3.11, node>=20]
+ *
+ * `canary doctor` reads those declarations from the skills installed in a
+ * consuming repo (overlay / global / local) and reports named, remediable
+ * failures — so a skill fails at `doctor` time with a clear message instead of
+ * cryptically at first run.
+ *
+ * Scope / naming: `requires` is deliberately distinct from harness's *planned*
+ * `capabilities:` field (Intense-Visions/harness-engineering#558). The two are
+ * orthogonal: harness `capabilities` bounds the Claude Code tools/network/file
+ * access a skill may use (a permission ceiling); canary `requires` verifies the
+ * runtime environment (is the interpreter installed, at a new-enough version).
+ * Keeping the field names separate avoids a future collision.
+ *
+ * Token grammar (v1): `<command>` or `<command><op><version>`, op in
+ * `>=` | `>` | `==`, version a dotted numeric (`3`, `3.11`, `1.22.0`). Non-command
+ * capability tokens (e.g. browser bundles) are intentionally NOT invented here;
+ * an unparseable token verifies as `unverifiable`, never a hard failure.
+ */
+export type RequirementOp = '>=' | '>' | '==';
+export interface Requirement {
+    /** The original token, verbatim. */
+    raw: string;
+    /** The command to look for on PATH, or null when the token is unparseable. */
+    command: string | null;
+    op: RequirementOp | null;
+    /** Dotted numeric version constraint, or null for a presence-only token. */
+    version: string | null;
+}
+export type RequirementStatus = 'ok' | 'missing' | 'too-old' | 'unverifiable';
+export interface RequirementResult {
+    requirement: Requirement;
+    status: RequirementStatus;
+    detail: string;
+}
+/** What a command probe reports: is it on PATH, and its detected version. */
+export interface ProbeResult {
+    present: boolean;
+    version: string | null;
+}
+export type CommandProbe = (command: string) => ProbeResult;
+/** Parse one requirement token into its command + optional version constraint. */
+export declare function parseRequirement(token: string): Requirement;
+/** Verify one requirement against a command probe. Never throws. */
+export declare function checkRequirement(req: Requirement, probe: CommandProbe): RequirementResult;
+/**
+ * Extract the `requires:` flow list from SKILL.md frontmatter — the same
+ * tiny-YAML subset the Python loader parses (`requires: [a, b]`). Returns []
+ * when absent or when there is no frontmatter. Block-sequence form is not
+ * supported (kept in lockstep with the Python parser).
+ */
+export declare function parseRequiresField(md: string): string[];

package/dist/source-spec.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Source-spec grammar for `canary overlay add <source>` (mirrors the
+ * `github:owner/repo` / git-URL / local-path forms used elsewhere in the
+ * ecosystem). Parsing is net-new — there is no existing engine parser.
+ */
+export interface ParsedSource {
+    /** Argument handed to `git clone` (URL or local path). */
+    cloneUrl: string;
+    /** Registry key + clone directory name: `<owner>-<repo>` (or path basename). */
+    name: string;
+}
+/** Raised when a source spec does not match any accepted form. */
+export declare class SourceSpecError extends Error {
+    constructor(message: string);
+}
+/**
+ * Parse a source spec into a clone URL and overlay name. Throws
+ * {@link SourceSpecError} on anything that does not match an accepted form.
+ */
+export declare function parseSource(raw: string): ParsedSource;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "canary-test-cli",
-  "version": "5.15.0",
+  "version": "6.0.0",
   "description": "Canary — AI-powered test automation agent",
   "license": "MIT",
   "repository": {
@@ -10,21 +10,45 @@
   "bin": {
     "canary": "./bin/canary.js"
   },
+  "exports": {
+    "./reporter": {
+      "types": "./dist/reporters/testtracker.d.ts",
+      "require": "./dist/reporters/testtracker.js",
+      "default": "./dist/reporters/testtracker.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "peerDependencies": {
+    "@playwright/test": ">=1.42.0"
+  },
+  "peerDependenciesMeta": {
+    "@playwright/test": {
+      "optional": true
+    }
+  },
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && node scripts/build-engine.mjs",
     "prepare": "npm run build",
-    "postinstall": "node scripts/install.js",
-    "test": "npm run build && node --test \"scripts/__tests__/*.test.js\""
+    "test": "tsc && node --test \"scripts/__tests__/*.test.js\""
   },
   "engines": {
     "node": ">=18"
   },
   "files": [
-    "bin/",
+    "bin/canary.js",
     "dist/",
-    "scripts/install.js"
+    "agent/frameworks/registry.json"
   ],
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "@supabase/supabase-js": "^2.45.0",
+    "commander": "^15.0.0",
+    "js-yaml": "^5.2.2",
+    "picocolors": "^1.1.1",
+    "zod": "^4.4.3"
+  },
   "devDependencies": {
+    "@playwright/test": "^1.61.1",
     "@types/node": "^22.0.0",
     "typescript": "^5.6.0"
   }