canary-test-cli 5.15.0 → 6.0.0

package/dist/engine/history/detector.js

@@ -0,0 +1,68 @@
+/**
+ * Flakiness and regression detection on top of history query results.
+ *
+ * Faithful TS port of `agent/history/detector.py`. Pure functions — no I/O.
+ * Input is the timeline output from the history store.
+ */
+export var FlakeTrend;
+(function (FlakeTrend) {
+    FlakeTrend["Rising"] = "rising";
+    FlakeTrend["Falling"] = "falling";
+    FlakeTrend["Stable"] = "stable";
+})(FlakeTrend || (FlakeTrend = {}));
+const TREND_THRESHOLD = 0.1;
+/** Classify a time-ordered list of per-run flake rates (0.0–1.0). */
+export function classifyFlakeTrend(rates) {
+    if (rates.length < 2)
+        return FlakeTrend.Stable;
+    const mid = Math.floor(rates.length / 2);
+    const firstHalf = rates.slice(0, mid);
+    const secondHalf = rates.slice(mid);
+    const mean = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+    const delta = mean(secondHalf) - mean(firstHalf);
+    if (delta >= TREND_THRESHOLD)
+        return FlakeTrend.Rising;
+    if (delta <= -TREND_THRESHOLD)
+        return FlakeTrend.Falling;
+    return FlakeTrend.Stable;
+}
+const BAD_STATUSES = new Set(['failed', 'flaky']);
+/**
+ * Detect whether a test regressed: green for `minGreen` runs, then failing for
+ * the last `recentFailures` runs.
+ */
+export function detectRegressions(timeline, minGreen = 5, recentFailures = 3) {
+    const none = {
+        is_regression: false,
+        green_streak: 0,
+        first_failure_commit: null,
+    };
+    if (timeline.length === 0)
+        return none;
+    const tail = timeline.length >= recentFailures ? timeline.slice(-recentFailures) : [];
+    if (tail.length < recentFailures)
+        return none;
+    if (!tail.every((r) => BAD_STATUSES.has(r.status)))
+        return none;
+    const firstFailIdx = timeline.length - recentFailures;
+    let streak = 0;
+    for (let i = firstFailIdx - 1; i >= 0; i--) {
+        if (timeline[i].status === 'passed')
+            streak++;
+        else
+            break;
+    }
+    if (streak < minGreen) {
+        return {
+            is_regression: false,
+            green_streak: streak,
+            first_failure_commit: null,
+        };
+    }
+    return {
+        is_regression: true,
+        green_streak: streak,
+        first_failure_commit: timeline[firstFailIdx].commit_sha ?? null,
+    };
+}
+//# sourceMappingURL=detector.js.map

package/dist/engine/history/ndjson-store.js

@@ -0,0 +1,177 @@
+/**
+ * Local NDJSON-backed history store — the TS side of the TS↔Python seam.
+ *
+ * Reads `history-v2.jsonl` (one JSON RunRecord per line) written by the Python
+ * `LocalHistoryStore`. Faithful port of the query_* semantics from
+ * `agent/history/local_store.py`.
+ *
+ * Deviation from the Python reader (deliberate): where Python's `_read_all`
+ * swallows any parse error and returns `[]`, this reader throws on malformed
+ * JSON and on an explicit unrecognized `schema_version`, so a corrupt or
+ * future-version history fails loudly rather than silently analysing nothing.
+ */
+import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { def } from '../util/coalesce.js';
+import { round1 } from '../util/round.js';
+import { SCHEMA_VERSION } from './record.js';
+import { serializeLocalRecord } from './schema.js';
+export class NdjsonHistoryStore {
+    path;
+    constructor(path) {
+        this.path = path;
+    }
+    readAll() {
+        let text;
+        try {
+            text = readFileSync(this.path, 'utf-8');
+        }
+        catch (err) {
+            // Missing file → empty history (matches Python's exists() guard).
+            if (err.code === 'ENOENT')
+                return [];
+            throw err;
+        }
+        const records = [];
+        for (const raw of text.split('\n')) {
+            const line = raw.trim();
+            if (!line)
+                continue;
+            const record = JSON.parse(line);
+            const version = record.schema_version;
+            if (version !== undefined && version !== SCHEMA_VERSION) {
+                throw new Error(`Unsupported history schema_version ${version} (expected ${SCHEMA_VERSION})`);
+            }
+            records.push(record);
+        }
+        return records;
+    }
+    /**
+     * Append a run + its results as one NDJSON line. Idempotent: a run whose
+     * `run_id` is already present is silently skipped (matches Python
+     * `LocalHistoryStore.push_run`).
+     */
+    pushRun(run, results) {
+        const existingIds = new Set(this.readAll().map((r) => r.run_id));
+        if (existingIds.has(run.run_id))
+            return;
+        const record = serializeLocalRecord(run, results);
+        mkdirSync(dirname(this.path), { recursive: true });
+        appendFileSync(this.path, JSON.stringify(record) + '\n', 'utf-8');
+    }
+    queryFlaky(window, suite, minRate) {
+        let records = this.readAll();
+        if (suite)
+            records = records.filter((r) => r.suite === suite);
+        records = records.slice(-window);
+        const counts = new Map();
+        for (const record of records) {
+            for (const t of def(record.tests, [])) {
+                let c = counts.get(t.test_name);
+                if (!c) {
+                    c = newFlakyCounter(record, t);
+                    counts.set(t.test_name, c);
+                }
+                c.total_runs += 1;
+                applyStatus(c, def(t.status, ''));
+                c.last_seen_run = record.run_id;
+            }
+        }
+        const results = [];
+        for (const c of counts.values()) {
+            if (c.total_runs === 0)
+                continue;
+            const rate = round1((c.flake_count / c.total_runs) * 100);
+            if (rate >= minRate)
+                results.push({ ...c, flake_rate_pct: rate });
+        }
+        return results.sort((a, b) => b.flake_rate_pct - a.flake_rate_pct);
+    }
+    queryTimeline(testName) {
+        const timeline = [];
+        for (const record of this.readAll()) {
+            for (const t of def(record.tests, [])) {
+                if (t.test_name === testName)
+                    timeline.push(toTimelineEntry(record, t));
+            }
+        }
+        return timeline.sort((a, b) => cmp(a.timestamp, b.timestamp));
+    }
+    querySummary(suite, runs) {
+        const suiteRecords = this.readAll().filter((r) => r.suite === suite);
+        const recent = runs > 0 ? suiteRecords.slice(-runs) : suiteRecords;
+        if (recent.length === 0) {
+            return { suite, total_runs: 0, avg_pass_rate: 0.0 };
+        }
+        return {
+            suite,
+            total_runs: recent.length,
+            avg_pass_rate: avgPassRate(recent),
+            runs: recent.map(toSummaryRun),
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+// Row-mapping helpers — extracted so the query methods stay under the arch
+// complexity threshold. Each isolates one dense `??`-fallback cluster.
+// ---------------------------------------------------------------------------
+function newFlakyCounter(record, t) {
+    return {
+        test_name: t.test_name,
+        test_file: def(t.test_file, ''),
+        suite: def(t.suite, def(record.suite, '')),
+        area: def(t.area, null),
+        flake_count: 0,
+        pass_count: 0,
+        fail_count: 0,
+        total_runs: 0,
+        last_seen_run: null,
+    };
+}
+function applyStatus(c, status) {
+    if (status === 'flaky')
+        c.flake_count += 1;
+    else if (status === 'passed')
+        c.pass_count += 1;
+    else if (status === 'failed')
+        c.fail_count += 1;
+}
+function toTimelineEntry(record, t) {
+    return {
+        run_id: record.run_id,
+        suite: def(record.suite, ''),
+        branch: def(record.branch, ''),
+        commit_sha: def(record.commit_sha, ''),
+        timestamp: def(record.timestamp, ''),
+        status: def(t.status, ''),
+        failure_category: def(t.failure_category, null),
+        error_text: def(t.error_text, null),
+        retry_count: def(t.retry_count, 0),
+    };
+}
+function toSummaryRun(r) {
+    return {
+        run_id: r.run_id,
+        branch: def(r.branch, ''),
+        timestamp: def(r.timestamp, ''),
+        passed: def(r.passed, 0),
+        failed: def(r.failed, 0),
+        flaky: def(r.flaky, 0),
+        total: def(r.total, 0),
+    };
+}
+function avgPassRate(recent) {
+    const rates = [];
+    for (const r of recent) {
+        const total = def(r.total, 0);
+        if (total > 0)
+            rates.push((def(r.passed, 0) / total) * 100);
+    }
+    if (rates.length === 0)
+        return 0.0;
+    return round1(rates.reduce((a, b) => a + b, 0) / rates.length);
+}
+function cmp(a, b) {
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+//# sourceMappingURL=ndjson-store.js.map

package/dist/engine/history/record.js

@@ -0,0 +1,14 @@
+/**
+ * TypeScript types for the v2 run-history record schema.
+ *
+ * Mirrors `agent/history/schema.py` (RunRecord + TestResult) as persisted, one
+ * JSON object per line, in `test-results/reports/history-v2.jsonl`.
+ *
+ * The on-disk records written by the Python engine do NOT carry a per-record
+ * version field (the "v2" lives in the filename). The reader therefore treats a
+ * missing `schema_version` as the current version, but throws on an explicit
+ * unrecognized one — a forward-compat guard, exercised by the store tests.
+ */
+/** The schema version this reader understands. */
+export const SCHEMA_VERSION = 2;
+//# sourceMappingURL=record.js.map

package/dist/engine/history/schema.js

@@ -0,0 +1,59 @@
+/**
+ * Write-side schema for run-history records.
+ *
+ * Faithful TS port of `agent/history/schema.py`. `record.ts` holds the *read*
+ * shapes (loose, as parsed from disk); this module holds the *write* inputs
+ * (the dataclass field sets) and the serializers that turn them into the exact
+ * dict shapes Python's `asdict()` produces — so a record written by TS is read
+ * identically by the Python `LocalHistoryStore` (the write-seam proof).
+ */
+import { def } from '../util/coalesce.js';
+/** `{suite}-{commit[:8]}-{epoch}` — identical to Python `make_run_id`. */
+export function makeRunId(suite, commitSha, timestampEpoch) {
+    return `${suite}-${commitSha.slice(0, 8)}-${timestampEpoch}`;
+}
+/** `asdict(run)` — every RunRecord field, optionals defaulted to null. */
+export function serializeRun(run) {
+    return {
+        run_id: run.run_id,
+        suite: run.suite,
+        repo: run.repo,
+        branch: run.branch,
+        commit_sha: run.commit_sha,
+        timestamp: run.timestamp,
+        total: run.total,
+        passed: run.passed,
+        failed: run.failed,
+        flaky: run.flaky,
+        skipped: run.skipped,
+        commit_message: def(run.commit_message, null),
+        env: def(run.env, null),
+        base_url: def(run.base_url, null),
+        duration_ms: def(run.duration_ms, null),
+    };
+}
+/** `asdict(t)` — every TestResult field, optionals/defaults applied. */
+export function serializeTestResult(t) {
+    return {
+        run_id: t.run_id,
+        suite: t.suite,
+        repo: t.repo,
+        test_name: t.test_name,
+        test_file: t.test_file,
+        status: t.status,
+        area: def(t.area, null),
+        failure_category: def(t.failure_category, null),
+        error_text: def(t.error_text, null),
+        retry_count: def(t.retry_count, 0),
+        duration_ms: def(t.duration_ms, null),
+        tags: def(t.tags, []),
+    };
+}
+/**
+ * The nested NDJSON line shape written by the local store: a serialized run
+ * with its `tests` embedded (matches Python `LocalHistoryStore.push_run`).
+ */
+export function serializeLocalRecord(run, results) {
+    return { ...serializeRun(run), tests: results.map(serializeTestResult) };
+}
+//# sourceMappingURL=schema.js.map

package/dist/engine/history/store.js

@@ -0,0 +1,47 @@
+/**
+ * History store factory + the async store contract.
+ *
+ * Port of `agent/history/store.py` (`HistoryStore` ABC + `make_store`).
+ *
+ * DISCOVERED BOUNDARY CONSTRAINT (differs from Python): the Python stores are
+ * synchronous because `supabase-py`'s `.execute()` is blocking. The JS SDK
+ * (`@supabase/supabase-js`) is Promise-based, so a remote store cannot expose
+ * synchronous queries. This module therefore defines an ASYNC store contract
+ * (`AsyncHistoryStore`) that `makeStore` returns. The analysis pilot's
+ * synchronous `NdjsonHistoryStore` is left untouched (the analysis engine reads
+ * it directly); here it is wrapped in a thin async adapter so both backends
+ * present one uniform async surface.
+ */
+import { NdjsonHistoryStore, } from './ndjson-store.js';
+import { SupabaseHistoryStore } from './supabase-store.js';
+const DEFAULT_NDJSON_PATH = 'test-results/reports/history-v2.jsonl';
+/** Async adapter over the synchronous local NDJSON store. */
+export class LocalAsyncAdapter {
+    inner;
+    constructor(inner) {
+        this.inner = inner;
+    }
+    async pushRun(run, results) {
+        this.inner.pushRun(run, results);
+    }
+    async queryFlaky(window, suite, minRate) {
+        return this.inner.queryFlaky(window, suite, minRate);
+    }
+    async queryTimeline(testName) {
+        return this.inner.queryTimeline(testName);
+    }
+    async querySummary(suite, runs) {
+        return this.inner.querySummary(suite, runs);
+    }
+}
+/**
+ * Return a Supabase-backed store when a db-url is configured (arg then
+ * `CANARY_HISTORY_DB_URL`), else a local NDJSON store. Mirrors `make_store`.
+ */
+export function makeStore(dbUrl, ndjsonPath) {
+    const resolvedUrl = dbUrl ?? process.env.CANARY_HISTORY_DB_URL;
+    if (resolvedUrl)
+        return new SupabaseHistoryStore(resolvedUrl);
+    return new LocalAsyncAdapter(new NdjsonHistoryStore(ndjsonPath ?? DEFAULT_NDJSON_PATH));
+}
+//# sourceMappingURL=store.js.map

package/dist/engine/history/supabase-store.js

@@ -0,0 +1,113 @@
+/**
+ * Supabase-backed history store.
+ *
+ * Port of `agent/history/supabase_store.py`, using `@supabase/supabase-js`.
+ * The JS SDK is Promise-based, so every method is async (see the boundary note
+ * in `store.ts`). The client is injectable so tests can mock it — no live
+ * network is ever required.
+ */
+import { createClient } from '@supabase/supabase-js';
+import { def } from '../util/coalesce.js';
+import { round1 } from '../util/round.js';
+import { serializeRun, serializeTestResult, } from './schema.js';
+const ERROR_TEXT_MAX = 2000;
+/**
+ * Resolve the Supabase project URL. A plain `https://…` url passes through; a
+ * `postgresql+asyncpg://user:pass@host/db` url yields `https://<host>`. Never
+ * returns the raw connection string (it embeds credentials). Pure + exported
+ * for direct parity testing against Python `_parse_project_url`.
+ */
+export function parseProjectUrl(dbUrl) {
+    if (dbUrl.startsWith('https://'))
+        return dbUrl;
+    try {
+        const host = new URL(dbUrl).hostname;
+        return `https://${host}`;
+    }
+    catch {
+        return '<redacted-unparseable-url>';
+    }
+}
+function toResultRow(t) {
+    const row = serializeTestResult(t);
+    const err = row.error_text;
+    if (typeof err === 'string' && err.length > ERROR_TEXT_MAX) {
+        row.error_text = err.slice(0, ERROR_TEXT_MAX);
+    }
+    return row;
+}
+function flattenTimelineRow(row) {
+    const runInfo = def(row.canary_runs, {});
+    return {
+        run_id: String(def(row.run_id, '')),
+        suite: String(def(row.suite, '')),
+        branch: String(def(runInfo.branch, '')),
+        commit_sha: String(def(runInfo.commit_sha, '')),
+        timestamp: String(def(runInfo.timestamp, '')),
+        status: String(def(row.status, '')),
+        failure_category: def(row.failure_category, null) ?? null,
+        error_text: def(row.error_text, null) ?? null,
+        retry_count: Number(def(row.retry_count, 0)),
+    };
+}
+export class SupabaseHistoryStore {
+    client;
+    constructor(dbUrl, client) {
+        this.client =
+            client ??
+                createClient(parseProjectUrl(dbUrl), def(process.env.SUPABASE_ANON_KEY, ''));
+    }
+    async pushRun(run, results) {
+        await this.client.from('canary_runs').upsert(serializeRun(run));
+        if (results.length > 0) {
+            await this.client
+                .from('canary_test_results')
+                .upsert(results.map(toResultRow));
+        }
+    }
+    async queryFlaky(_window, suite, minRate) {
+        let query = this.client
+            .from('canary_flake_summary')
+            .select('*')
+            .gte('flake_rate_pct', minRate)
+            .order('flake_rate_pct', { ascending: false });
+        if (suite)
+            query = query.eq('suite', suite);
+        const { data } = await query;
+        return def(data, []);
+    }
+    async queryTimeline(testName) {
+        const { data } = await this.client
+            .from('canary_test_results')
+            .select('run_id, suite, status, failure_category, error_text, retry_count, ' +
+            'canary_runs!inner(branch, commit_sha, timestamp)')
+            .eq('test_name', testName)
+            .order('canary_runs(timestamp)');
+        return def(data, []).map(flattenTimelineRow);
+    }
+    async querySummary(suite, runs) {
+        const { data } = await this.client
+            .from('canary_runs')
+            .select('run_id, branch, timestamp, passed, failed, flaky, total')
+            .eq('suite', suite)
+            .order('timestamp', { ascending: false })
+            .limit(runs);
+        const recent = [...def(data, [])].reverse();
+        if (recent.length === 0) {
+            return { suite, total_runs: 0, avg_pass_rate: 0.0 };
+        }
+        const rates = recent
+            .filter((r) => Number(def(r.total, 0)) > 0)
+            .map((r) => (Number(r.passed) / Number(r.total)) * 100);
+        const avg = rates.length > 0
+            ? round1(rates.reduce((a, b) => a + b, 0) / rates.length)
+            : 0.0;
+        return {
+            suite,
+            total_runs: recent.length,
+            avg_pass_rate: avg,
+            runs: recent,
+        };
+    }
+}
+//# sourceMappingURL=supabase-store.js.map

package/dist/engine/main-deps.js

@@ -0,0 +1,105 @@
+/**
+ * Shared injectable dependencies + glyph constants for the main `canary` CLI
+ * and its inline sub-apps (skills / workflow / company-knowledge).
+ *
+ * Every command reaches the outside world through {@link MainDeps} so tests run
+ * with capturing sinks, a fixed env/cwd/home, and fake class factories -- the
+ * commander analog of the Python CLI tests' `mock.patch(...)` seams. The
+ * production {@link defaultMainDeps} wires process-backed real classes.
+ *
+ * Output glyphs are declared as `\u{...}` escapes (ASCII-source rule) and
+ * emitted verbatim; picocolors strips its color on a non-TTY sink so the plain
+ * text matches rich's markup stripping byte-for-byte (see `guardian/cli.ts`).
+ */
+import { spawnSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { TestClassifier } from './core/classifier.js';
+import { CompanyKnowledge } from './core/company-knowledge.js';
+import { CanaryTestExecutor } from './core/executor.js';
+import { FrameworkRegistry } from './core/framework-registry.js';
+import { HarnessMigrator } from './core/migrator.js';
+import { PatternHealer } from './core/pattern-healer.js';
+import { FrameworkRecommender } from './core/recommender.js';
+import { Scaffolder } from './core/scaffolder.js';
+import { SkillRegistry } from './core/skill-registry.js';
+import { StaticLinter } from './core/static-linter.js';
+import { TicketUpdater } from './core/ticket-updater.js';
+import { WorkflowDiscovery } from './core/workflow-discovery.js';
+// --- output glyphs (emitted verbatim; see module docstring) -------------------
+export const CHECK_MARK = '\u{2705}'; // white heavy check mark
+export const WARN = '\u{26a0}'; // warning sign
+export const CROSS = '\u{2717}'; // ballot X
+export const ROCKET = '\u{1f680}';
+export const HAMMER = '\u{1f6e0}';
+export const NEXT = '\u{23ed}\u{fe0f}'; // next-track button + VS16
+export const REDX = '\u{274c}'; // cross mark
+export const WRENCH = '\u{1f527}';
+export const MAGNIFIER = '\u{1f50d}';
+export const CHECK = '\u{2713}'; // light check mark
+export const ARROW = '\u{2192}'; // rightwards arrow
+export const EM_DASH = '\u{2014}';
+export const ELLIPSIS = '\u{2026}';
+/** A stdin-backed prompt: reads piped lines once, returns `def` when exhausted. */
+function makeStdinPrompt() {
+    let lines = null;
+    let idx = 0;
+    return (_text, def) => {
+        if (lines === null) {
+            try {
+                lines = readFileSync(0, 'utf-8').split('\n');
+            }
+            catch {
+                lines = [];
+            }
+        }
+        const raw = idx < lines.length ? lines[idx++] : '';
+        return raw === '' ? def : raw;
+    };
+}
+/** Process-backed defaults for production. */
+export function defaultMainDeps() {
+    return {
+        out: (s) => process.stdout.write(`${s}\n`),
+        err: (s) => process.stderr.write(`${s}\n`),
+        env: process.env,
+        cwd: () => process.cwd(),
+        home: () => homedir(),
+        // The npm bin injects the real package version; the bare factory default
+        // reports 'unknown' (Python's PackageNotFoundError fallback shape).
+        pkgVersion: () => 'unknown',
+        openBrowser: () => {
+            // Best-effort no-op default; a desktop launcher is wired by the bin.
+        },
+        runSubprocess: (cmd, args, opts = {}) => {
+            const res = spawnSync(cmd, args, {
+                encoding: 'utf-8',
+                maxBuffer: Infinity,
+                ...(opts.cwd ? { cwd: opts.cwd } : {}),
+                ...(opts.inherit ? { stdio: 'inherit' } : {}),
+            });
+            if (res.error)
+                return { status: null, stdout: '', stderr: '' };
+            return {
+                status: res.status,
+                stdout: res.stdout ?? '',
+                stderr: res.stderr ?? '',
+            };
+        },
+        prompt: makeStdinPrompt(),
+        pythonExe: () => 'python3',
+        makeClassifier: () => new TestClassifier(),
+        makeRecommender: () => new FrameworkRecommender(),
+        makeRegistry: () => new FrameworkRegistry(),
+        makeExecutor: () => new CanaryTestExecutor(),
+        makeScaffolder: () => new Scaffolder(),
+        makeMigrator: () => new HarnessMigrator(),
+        makeLinter: () => new StaticLinter(),
+        makeHealer: () => new PatternHealer(),
+        makeSkillRegistry: () => new SkillRegistry(),
+        makeWorkflowDiscovery: () => new WorkflowDiscovery(),
+        makeTicketUpdater: () => new TicketUpdater(),
+        loadCompanyKnowledge: (env) => CompanyKnowledge.load(undefined, env),
+    };
+}
+//# sourceMappingURL=main-deps.js.map