canary-test-cli 5.15.0 → 6.0.0

package/dist/engine/core/quality-scorer.js

@@ -0,0 +1,213 @@
+/**
+ * Test quality static analyser.
+ *
+ * Faithful TypeScript port of `agent/core/quality_scorer.py`. Scores test code
+ * on coverage breadth, assertion density, and flakiness risk, plus a
+ * magic-number maintainability nudge. Purely lexical — no execution.
+ */
+import { num1, roundHalfEvenInt } from '../util/round.js';
+const TEST_FN = {
+    pytest: /^[ \t]*def test_/gm,
+    playwright: /\btest\s*\(/gm,
+    vitest: /\b(?:it|test)\s*\(/gm,
+    k6: /\bcheck\s*\(/gm,
+};
+const ASSERTIONS = {
+    // `assert x`, pytest.raises, unittest self.assert*, AND a call to any
+    // assert*-named helper (e.g. `assert_valid(...)`) — the last so a test that
+    // delegates its check to a custom assert helper isn't misread as asserting
+    // nothing. (`\bassert\b` alone does NOT match `assert_valid`: `_` is a word
+    // char, so the `\b` after `assert` fails there.)
+    pytest: /\bassert\b|\bpytest\.raises\b|\bself\.assert\w+\b|\bassert\w*\s*\(/g,
+    playwright: /\bexpect\s*\(|\btoBeVisible\b|\btoHaveText\b|\btoHaveTitle\b|\btoHaveURL\b|\btoBeEnabled\b|\btoBeDisabled\b|\btoBeChecked\b|\btoHaveValue\b|\btoHaveCount\b/g,
+    // Plus non-`expect` assertion styles common in JS/TS: node:assert / vitest
+    // `assert(...)` / `assert.equal(...)`, and chai BDD `x.should.equal`.
+    vitest: /\bexpect\s*\(|\btoBe\s*\(|\btoEqual\s*\(|\btoThrow\b|\btoContain\s*\(|\btoBeNull\b|\btoBeUndefined\b|\btoMatchObject\b|\bassert\s*\(|\bassert\.\w+|\.should\b/g,
+    k6: /\bcheck\s*\(|'[^']+'\s*:\s*\([^)]*\)\s*=>/g,
+};
+const NEGATIVE_KW = /\b(error|invalid|empty|null|undefined|throws|raises|exception|fail|missing|negative|reject|4\d{2}|5\d{2}|boundary|edge)\b/i;
+const PARAMETRIZE = /@pytest\.mark\.parametrize|test\.each\s*\(|describe\.each\s*\(|it\.each\s*\(/;
+const SLEEP = /time\.sleep\s*\(|page\.waitForTimeout\s*\(|await\s+new\s+Promise[^)]*setTimeout|setTimeout\s*\(/g;
+const RANDOM = /Math\.random\s*\(|random\.random\s*\(|random\.choice\s*\(|random\.randint\s*\(/;
+const TIMESTAMP = /Date\.now\s*\(|datetime\.now\s*\(|datetime\.utcnow\s*\(/;
+// Blank out string contents before magic-number scanning.
+const STRING_LITERAL = /(['"])(?:\\.|(?!\1).)*?\1/g;
+const NUMERIC_LITERAL = /(?<![\w.])-?\d+(?:\.\d+)?(?![\w.])/g;
+const ALLOWED_NUMBERS = new Set(['0', '1', '2', '-1', '10', '100']);
+const HTTP_STATUS = new Set([
+    '200',
+    '201',
+    '202',
+    '204',
+    '301',
+    '302',
+    '304',
+    '400',
+    '401',
+    '403',
+    '404',
+    '405',
+    '409',
+    '410',
+    '422',
+    '429',
+    '500',
+    '501',
+    '502',
+    '503',
+    '504',
+]);
+const MAX_MAGIC_FINDINGS = 10;
+function countMatches(re, code) {
+    return (code.match(re) ?? []).length;
+}
+function isAllowedNumber(token) {
+    if (ALLOWED_NUMBERS.has(token) || HTTP_STATUS.has(token))
+        return true;
+    const bare = token.replace(/^-/, '');
+    return /^\d$/.test(bare);
+}
+function detectMagicNumbers(code) {
+    const findings = [];
+    const lines = code.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+        const stripped = lines[i].trim();
+        if (stripped.startsWith('#') || stripped.startsWith('//'))
+            continue;
+        const scrubbed = lines[i].replace(STRING_LITERAL, '""');
+        for (const m of scrubbed.matchAll(NUMERIC_LITERAL)) {
+            const token = m[0];
+            if (isAllowedNumber(token))
+                continue;
+            findings.push(`line ${i + 1}: magic number ${token} — name it or derive it`);
+            if (findings.length >= MAX_MAGIC_FINDINGS)
+                return findings;
+        }
+    }
+    return findings;
+}
+const COVERAGE_BASE = [0, 25, 45, 65, 80, 90];
+function scoreCoverage(code, framework) {
+    const details = [];
+    const pattern = TEST_FN[framework] ?? TEST_FN['pytest'];
+    const count = countMatches(pattern, code);
+    const label = framework === 'k6' ? 'check' : 'test function';
+    details.push(`${count} ${label}${count !== 1 ? 's' : ''} found`);
+    const base = Math.min(90, COVERAGE_BASE[Math.min(count, 5)]);
+    let bonus = 0;
+    if (NEGATIVE_KW.test(code)) {
+        bonus += 10;
+        details.push('Covers error/invalid paths');
+    }
+    if (PARAMETRIZE.test(code)) {
+        bonus += 10;
+        details.push('Parametrized test cases detected');
+    }
+    return [Math.min(100, base + bonus), details];
+}
+function densityScore(density) {
+    if (density === 0)
+        return 0;
+    if (density < 1)
+        return 25;
+    if (density < 2)
+        return 55;
+    if (density < 3)
+        return 75;
+    if (density < 4)
+        return 88;
+    return 97;
+}
+function scoreAssertions(code, framework) {
+    const fnPat = TEST_FN[framework] ?? TEST_FN['pytest'];
+    const assertPat = ASSERTIONS[framework] ?? ASSERTIONS['pytest'];
+    const testCount = Math.max(1, countMatches(fnPat, code));
+    const assertCount = countMatches(assertPat, code);
+    const density = assertCount / testCount;
+    const details = [
+        `${assertCount} assertion${assertCount !== 1 ? 's' : ''}, ${num1(density)} per test`,
+    ];
+    return [densityScore(density), details];
+}
+function scoreFlakiness(code) {
+    const details = [];
+    let score = 100;
+    const sleepN = countMatches(SLEEP, code);
+    if (sleepN) {
+        score -= Math.min(40, sleepN * 20);
+        details.push(`${sleepN} hardcoded wait${sleepN !== 1 ? 's' : ''} detected`);
+    }
+    if (RANDOM.test(code)) {
+        score -= 15;
+        details.push('Non-deterministic random values detected');
+    }
+    if (TIMESTAMP.test(code)) {
+        score -= 10;
+        details.push('Timestamp-dependent assertions detected');
+    }
+    if (details.length === 0)
+        details.push('No flakiness signals detected');
+    return [Math.max(0, score), details];
+}
+function grade(score) {
+    if (score >= 85)
+        return 'A';
+    if (score >= 70)
+        return 'B';
+    if (score >= 55)
+        return 'C';
+    if (score >= 40)
+        return 'D';
+    return 'F';
+}
+function magicDetails(findings) {
+    if (findings.length === 0)
+        return ['No magic numbers detected'];
+    return [`${findings.length} magic number(s) detected`, ...findings];
+}
+/**
+ * True iff `code` defines at least one test function but contains no assertions
+ * — a test that asserts nothing.
+ *
+ * Faithful port of `is_assertion_free_test`. The high-precision "weak test"
+ * signal the PR guardian consumes: it requires BOTH a test function (so a
+ * non-test helper added to a test file isn't flagged) AND zero assertions. A
+ * snapshot or table-driven test still matches an assertion pattern (`expect(` /
+ * `assert`), so it is not flagged — keeping false positives, and the trust cost
+ * of them, low. Reuses the module's {@link TEST_FN}/{@link ASSERTIONS} maps.
+ */
+export function isAssertionFreeTest(code, framework) {
+    const fw = framework.toLowerCase();
+    const fnPat = TEST_FN[fw] ?? TEST_FN['pytest'];
+    const assertPat = ASSERTIONS[fw] ?? ASSERTIONS['pytest'];
+    const hasTest = countMatches(fnPat, code) > 0;
+    const hasAssertion = countMatches(assertPat, code) > 0;
+    return hasTest && !hasAssertion;
+}
+export class QualityScorer {
+    /** Score `code` (a source string) for the given framework. */
+    score(code, framework) {
+        const fw = framework.toLowerCase();
+        const [coverage, covDetails] = scoreCoverage(code, fw);
+        const [assertion, asrDetails] = scoreAssertions(code, fw);
+        const [flakiness, flkDetails] = scoreFlakiness(code);
+        const magic = detectMagicNumbers(code);
+        const raw = roundHalfEvenInt(0.4 * coverage + 0.4 * assertion + 0.2 * flakiness);
+        const composite = Math.max(0, raw - Math.min(15, magic.length * 3));
+        return {
+            score: composite,
+            grade: grade(composite),
+            coverage_breadth: coverage,
+            assertion_density: assertion,
+            flakiness_risk: flakiness,
+            magic_numbers: magic.length,
+            details: [
+                ...covDetails,
+                ...asrDetails,
+                ...flkDetails,
+                ...magicDetails(magic),
+            ],
+        };
+    }
+}
+//# sourceMappingURL=quality-scorer.js.map

package/dist/engine/core/recommender.js

@@ -0,0 +1,152 @@
+/**
+ * Framework Recommender — picks testing tools for a classification, ranked.
+ *
+ * Faithful TypeScript port of `agent/core/recommender.py`. Wires the classifier
+ * result + the framework registry into a ranked candidate list.
+ */
+import { def } from '../util/coalesce.js';
+import { FrameworkRegistry } from './framework-registry.js';
+const MAX_CANDIDATES = 3;
+const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']);
+/** Whether a framework may surface given the current env license signals. */
+export function licenseAllowed(framework) {
+    const gate = framework.license_gate;
+    if (!gate)
+        return true;
+    const value = def(process.env[gate], '').trim();
+    if (!value || FALSE_VALUES.has(value.toLowerCase()))
+        return false;
+    const scopes = framework.license_scopes;
+    if (scopes)
+        return scopes.includes(value);
+    return true;
+}
+const LANG_EXT = {
+    python: 'py',
+    javascript: 'js',
+    typescript: 'ts',
+};
+function fileExtensionFor(f) {
+    const exts = def(f.file_extensions, []);
+    if (exts.length > 0)
+        return exts[0];
+    const langs = def(f.languages, []);
+    if (langs.length > 0)
+        return def(LANG_EXT[langs[0].toLowerCase()], 'ts');
+    return 'ts';
+}
+function buildReason(f) {
+    const reasons = [];
+    reasons.push(...def(f.recommended_for, []));
+    reasons.push(...def(f.strengths, []).slice(0, 2));
+    if (f.maturity)
+        reasons.push(`Maturity level: ${f.maturity}`);
+    return reasons;
+}
+function formatCandidate(f, confidence) {
+    const candidate = {
+        framework: f.name,
+        category: def(f.category, ''),
+        file_extension: fileExtensionFor(f),
+        reason: buildReason(f),
+        confidence,
+    };
+    if (f.license_note) {
+        candidate.license = def(f.license, null);
+        candidate.warning = f.license_note;
+        const label = def(f.license, 'non-OSI license');
+        candidate.reason.unshift(`⚠ ${label}: review against your license policy`);
+    }
+    return candidate;
+}
+export class FrameworkRecommender {
+    registry;
+    constructor(registry = new FrameworkRegistry()) {
+        this.registry = registry;
+    }
+    recommend(classification, metadata = null, frameworkHint = null) {
+        if (classification.test_type === 'observability') {
+            return this.recommendObservability(classification);
+        }
+        let frameworks = this.registry.getByCategory(classification.test_type);
+        frameworks = frameworks.filter(licenseAllowed);
+        if (frameworks.length === 0)
+            return [];
+        const candidates = this.applyLanguageFilter(frameworks, metadata);
+        const ranked = this.rankPool(candidates, frameworkHint);
+        return this.dedupeAndFormat(ranked, frameworkHint, classification.confidence);
+    }
+    applyLanguageFilter(frameworks, metadata) {
+        if (metadata === null)
+            return frameworks;
+        const detected = new Set(def(metadata.detected_languages, []));
+        if (detected.size === 0)
+            return frameworks;
+        const filtered = frameworks.filter((f) => def(f.languages, []).some((l) => detected.has(l)));
+        return filtered.length > 0 ? filtered : frameworks;
+    }
+    rankPool(candidates, frameworkHint) {
+        const hinted = frameworkHint
+            ? candidates.find((f) => f.name === frameworkHint.toLowerCase())
+            : undefined;
+        const preferred = candidates.filter((f) => f.status === 'preferred');
+        const rest = candidates.filter((f) => f.status !== 'preferred');
+        return [...(hinted ? [hinted] : []), ...preferred, ...rest];
+    }
+    dedupeAndFormat(pool, frameworkHint, confidence) {
+        const hintName = frameworkHint ? frameworkHint.toLowerCase() : null;
+        const ranked = [];
+        const seen = new Set();
+        for (const fw of pool) {
+            if (seen.has(fw.name))
+                continue;
+            seen.add(fw.name);
+            const entry = formatCandidate(fw, confidence);
+            if (hintName !== null && fw.name === hintName) {
+                entry.reason.unshift(`prompt-named framework (${fw.name})`);
+            }
+            ranked.push(entry);
+            if (ranked.length >= MAX_CANDIDATES)
+                break;
+        }
+        return ranked;
+    }
+    recommendObservability(classification) {
+        const confidence = classification.confidence;
+        const candidates = [];
+        const scope = def(process.env['CANARY_SCOPE'], '').trim();
+        if (scope) {
+            candidates.push({
+                framework: `${scope}-dashboard`,
+                category: 'observability',
+                file_extension: '',
+                reason: [
+                    `configured aggregation dashboard (CANARY_SCOPE=${scope})`,
+                    'overlay reporting sink — receives results in addition to ReportPortal',
+                ],
+                confidence,
+                kind: 'reporting-sink',
+            });
+        }
+        candidates.push({
+            framework: 'reportportal',
+            category: 'observability',
+            file_extension: '',
+            reason: [
+                'self-hosted OSS reporting sink — default for observability output',
+            ],
+            confidence,
+            kind: 'reporting-sink',
+        });
+        const otel = this.registry
+            .getByCategory('observability')
+            .find((f) => f.name === 'opentelemetry');
+        if (otel) {
+            const c = formatCandidate(otel, confidence);
+            c.kind = 'instrumentation';
+            candidates.push(c);
+        }
+        return candidates.slice(0, MAX_CANDIDATES);
+    }
+}
+//# sourceMappingURL=recommender.js.map

package/dist/engine/core/reporter.js

@@ -0,0 +1,211 @@
+/**
+ * Standardized reporting for Canary execution results.
+ *
+ * Faithful TypeScript port of `agent/core/reporter.py`. Exports generation and
+ * execution results to JSON or SARIF format for consumption by Datadog,
+ * SonarQube, GitHub Code Scanning, and similar dashboards.
+ *
+ * SARIF 2.1.0 spec: https://docs.oasis-open.org/sarif/sarif/v2.1.0/
+ *
+ * Python→TS nuances:
+ *   - **JSON shape is a contract.** Both serializers mirror Python's
+ *     `json.dumps(..., indent=2, default=str)`: `JSON.stringify(x, replacer, 2)`
+ *     with the library-default `ensure_ascii=True` reproduced via
+ *     {@link ensureAscii} (so an em-dash in a message emits `—`, exactly
+ *     as the oracle does). Object key insertion order preserves the field order
+ *     Python emits.
+ *   - **`default=str`.** Python coerces any value its encoder cannot natively
+ *     serialize (a `Path`, a `datetime`, ...) via `str()`. JS inputs are already
+ *     plain JSON objects; the only common value `JSON.stringify` would *throw*
+ *     on is `BigInt`, which the replacer stringifies. There is no JS analog of a
+ *     `Path` object, so that specific coercion is not reproducible — see the
+ *     ported test, which exercises the BigInt path instead.
+ *   - Python truthiness (`""`/`{}`/`None` falsy) via {@link pyTruthy}; missing
+ *     dict keys via {@link pyGet}.
+ */
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+const SARIF_SCHEMA = 'https://json.schemastore.org/sarif-2.1.0.json';
+const TOOL_NAME = 'Canary';
+const TOOL_VERSION = '0.1.0';
+const TOOL_URI = 'https://github.com/bop-clocktower/canary';
+const RULES = [
+    {
+        id: 'canary/test-generation',
+        name: 'TestGeneration',
+        shortDescription: { text: 'AI-generated test file' },
+        helpUri: TOOL_URI,
+    },
+    {
+        id: 'canary/test-execution',
+        name: 'TestExecution',
+        shortDescription: { text: 'Automated test execution result' },
+        helpUri: TOOL_URI,
+    },
+];
+export const SUPPORTED_FORMATS = ['json', 'sarif'];
+// ---------------------------------------------------------------------------
+// Python-compatibility helpers (mirrors `guardian/diff-extractor.ts`)
+// ---------------------------------------------------------------------------
+/**
+ * Python-truthiness for JSON-shaped values: `None`/`undefined`, `false`, `0`,
+ * `""`, empty array, and empty object are all falsy (mirrors `if x:`).
+ */
+function pyTruthy(value) {
+    if (value === null || value === undefined || value === false)
+        return false;
+    if (value === 0 || value === '')
+        return false;
+    if (Array.isArray(value))
+        return value.length > 0;
+    if (typeof value === 'object')
+        return Object.keys(value).length > 0;
+    return Boolean(value);
+}
+/** Python `a or b`: the fallback wins only when `a` is falsy. */
+function pyOr(value, fallback) {
+    return pyTruthy(value) ? value : fallback;
+}
+/** Python `dict.get(key, default)`: default only on a missing key. */
+function pyGet(obj, key, fallback) {
+    return Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : fallback;
+}
+/**
+ * Reproduce Python's `json.dumps(..., ensure_ascii=True)` (the library default)
+ * on `JSON.stringify` output: escape every code point >= 0x80 as `\uXXXX`. Only
+ * touches the >= 0x80 range, so the ASCII escapes `JSON.stringify` already
+ * produced are left intact. (Same helper as `guardian/pr-check.ts`.)
+ */
+function ensureAscii(json) {
+    return json.replace(/[€-￿]/g, (ch) => '\\u' + ch.charCodeAt(0).toString(16).padStart(4, '0'));
+}
+/**
+ * `json.dumps(default=str)` replacer. Values the encoder can't natively handle
+ * are coerced via `str()`; in JS the only common such value that would
+ * otherwise *throw* is `BigInt`, which we stringify.
+ */
+function pyDefaultStr(_key, value) {
+    return typeof value === 'bigint' ? value.toString() : value;
+}
+/** Converts Canary pipeline results into standardized report formats. */
+export class Reporter {
+    /**
+     * Serialize `result` to `fmt` ('json' or 'sarif') and write to disk; throws
+     * `Error` (Python `ValueError`) for unsupported formats. Returns the written
+     * path (Python returns a `Path`; here a string).
+     */
+    write(result, fmt, outputPath) {
+        if (!SUPPORTED_FORMATS.includes(fmt)) {
+            throw new Error(`Unsupported report format '${fmt}'. ` +
+                `Choose from: ${SUPPORTED_FORMATS.join(', ')}`);
+        }
+        const path = pyTruthy(outputPath)
+            ? outputPath
+            : `canary-report.${fmt}`;
+        const content = fmt === 'json' ? this.toJson(result) : this.toSarif(result);
+        mkdirSync(dirname(path), { recursive: true });
+        writeFileSync(path, content, 'utf-8');
+        return path;
+    }
+    /** Serialize `result` as pretty-printed JSON. */
+    toJson(result) {
+        return ensureAscii(JSON.stringify(result, pyDefaultStr, 2));
+    }
+    /** Serialize `result` as SARIF 2.1.0 JSON. */
+    toSarif(result) {
+        const sarif = {
+            version: '2.1.0',
+            $schema: SARIF_SCHEMA,
+            runs: [this.buildRun(result)],
+        };
+        return ensureAscii(JSON.stringify(sarif, pyDefaultStr, 2));
+    }
+    // --------------------------------------------------------------------
+    // SARIF construction
+    // --------------------------------------------------------------------
+    buildRun(result) {
+        return {
+            tool: {
+                driver: {
+                    name: TOOL_NAME,
+                    version: TOOL_VERSION,
+                    informationUri: TOOL_URI,
+                    rules: RULES,
+                },
+            },
+            results: this.buildResults(result),
+        };
+    }
+    buildResults(result) {
+        const sarifResults = [];
+        const outputFile = pyGet(result, 'output_file', '');
+        const testType = pyGet(result, 'test_type', 'unknown');
+        const framework = pyGet(result, 'framework', 'unknown');
+        // Generation result — always present.
+        const genProps = {
+            framework,
+            test_type: testType,
+            reasoning: pyGet(result, 'reasoning', []),
+        };
+        if (pyTruthy(pyGet(result, 'quality', null))) {
+            genProps['quality'] = result['quality'];
+        }
+        sarifResults.push({
+            ruleId: 'canary/test-generation',
+            message: {
+                text: `Generated ${String(framework)} test (${String(testType)})` +
+                    (pyTruthy(outputFile) ? ` — ${String(outputFile)}` : ''),
+            },
+            level: 'none',
+            locations: pyTruthy(outputFile) ? [location(String(outputFile))] : [],
+            properties: genProps,
+        });
+        // Execution result — only if the test was run.
+        const execution = pyGet(result, 'execution', null);
+        if (pyTruthy(execution)) {
+            const exec = execution;
+            const exitCode = pyGet(exec, 'exit_code', -1);
+            const passed = exitCode === 0;
+            const fixed = pyGet(exec, 'fixed', false);
+            const messageParts = [
+                `Test execution ${passed ? 'passed' : 'failed'} ` +
+                    `(exit code ${String(exitCode)})`,
+            ];
+            if (pyTruthy(fixed)) {
+                messageParts.push('Self-healed after initial failure.');
+            }
+            if (!passed) {
+                const stderr = pyOr(pyGet(exec, 'stderr', null), pyGet(exec, 'stdout', ''));
+                if (pyTruthy(stderr)) {
+                    const s = String(stderr);
+                    // Truncate long error output for readability in dashboards. Python
+                    // `stderr[:300]` and `len(stderr) > 300` count code points; JS slice
+                    // and .length count UTF-16 units, so astral chars would truncate
+                    // early and flip the >300 guard. Count/slice by code point instead.
+                    const chars = Array.from(s);
+                    const preview = chars.slice(0, 300).join('') + (chars.length > 300 ? '…' : '');
+                    messageParts.push(`Error: ${preview}`);
+                }
+            }
+            sarifResults.push({
+                ruleId: 'canary/test-execution',
+                message: { text: messageParts.join(' ') },
+                level: passed ? 'none' : 'error',
+                locations: pyTruthy(outputFile) ? [location(String(outputFile))] : [],
+                properties: {
+                    exit_code: exitCode,
+                    fixed,
+                },
+            });
+        }
+        return sarifResults;
+    }
+}
+function location(uri) {
+    return {
+        physicalLocation: {
+            artifactLocation: { uri, uriBaseId: '%SRCROOT%' },
+        },
+    };
+}
+//# sourceMappingURL=reporter.js.map