ci-triage 0.1.0

package/dist/flake-detector.js

@@ -0,0 +1,107 @@
+import { fetchRunHistory } from './history.js';
+function testKey(className, testName) {
+    return `${className}::${testName}`;
+}
+function normalizeClassName(className) {
+    return className ?? '';
+}
+function toPattern(statuses, maxLength) {
+    const pattern = statuses
+        .slice(0, maxLength)
+        .map((status) => (status === 'pass' ? 'pass' : 'fail'));
+    while (pattern.length < maxLength) {
+        pattern.push('unknown');
+    }
+    return pattern;
+}
+function computePassRate7d(statuses, runs) {
+    const cutoff = Date.now() - 7 * 24 * 60 * 60 * 1000;
+    const inWindow = runs
+        .map((run, index) => ({ run, status: statuses[index] }))
+        .filter(({ run, status }) => Boolean(status) && Date.parse(run.created_at) >= cutoff)
+        .map(({ status }) => status);
+    if (inWindow.length === 0) {
+        return 0;
+    }
+    const passCount = inWindow.filter((status) => status === 'pass').length;
+    return passCount / inWindow.length;
+}
+function hasAlternatingPattern(statuses) {
+    if (statuses.length < 4) {
+        return false;
+    }
+    let transitions = 0;
+    for (let i = 1; i < statuses.length; i += 1) {
+        if (statuses[i] !== statuses[i - 1]) {
+            transitions += 1;
+        }
+    }
+    const transitionRate = transitions / (statuses.length - 1);
+    const hasPass = statuses.includes('pass');
+    const hasFailLike = statuses.some((s) => s === 'fail' || s === 'error');
+    return hasPass && hasFailLike && transitionRate >= 0.6;
+}
+function isConsistentFailure(statuses) {
+    return statuses.length >= 3 && statuses.every((status) => status === 'fail' || status === 'error');
+}
+function clampConfidence(value) {
+    return Math.max(0, Math.min(1, Number(value.toFixed(2))));
+}
+function evaluateFlake(statuses, runs) {
+    const passRate = computePassRate7d(statuses, runs);
+    const last5 = toPattern(statuses, 5);
+    const alternating = hasAlternatingPattern(statuses);
+    const consistentFail = isConsistentFailure(statuses);
+    const recentPass = statuses.slice(0, 3).includes('pass');
+    if (statuses.length === 0) {
+        return {
+            is_flaky: false,
+            confidence: 0.2,
+            pass_rate_7d: 0,
+            last_5_runs: last5,
+        };
+    }
+    if (consistentFail) {
+        return {
+            is_flaky: false,
+            confidence: 0.9,
+            pass_rate_7d: passRate,
+            last_5_runs: last5,
+        };
+    }
+    let score = 0.2;
+    if (recentPass) {
+        score += 0.25;
+    }
+    if (alternating) {
+        score += 0.35;
+    }
+    if (passRate > 0.2 && passRate < 0.9) {
+        score += 0.2;
+    }
+    if (statuses.length >= 5) {
+        score += 0.05;
+    }
+    return {
+        is_flaky: score >= 0.6,
+        confidence: clampConfidence(score),
+        pass_rate_7d: Number(passRate.toFixed(2)),
+        last_5_runs: last5,
+    };
+}
+export async function detectFlakes(currentFailures, repoSlug, historyDepth) {
+    const history = await fetchRunHistory(repoSlug, historyDepth);
+    return currentFailures.map((failure) => {
+        const className = normalizeClassName(failure.class_name);
+        const key = testKey(className, failure.test_name);
+        const statuses = history
+            .map((run) => run.tests[key])
+            .filter((status) => Boolean(status));
+        const flaky = evaluateFlake(statuses, history);
+        return {
+            test_name: failure.test_name,
+            class_name: className,
+            flaky,
+        };
+    });
+}

package/dist/github.js

@@ -0,0 +1,52 @@
+import { execFileSync } from 'node:child_process';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { CI_TRIAGE_MARKER } from './comment.js';
+function ghJson(args, token) {
+    const out = execFileSync('gh', args, {
+        encoding: 'utf8',
+        env: {
+            ...process.env,
+            GH_TOKEN: token,
+            GITHUB_TOKEN: token,
+        },
+    });
+    return JSON.parse(out);
+}
+function sanitizeFileName(name) {
+    return name.replace(/[^a-zA-Z0-9._-]/g, '-');
+}
+export function uploadArtifact(name, content) {
+    const fileName = `${sanitizeFileName(name)}.json`;
+    const filePath = join(tmpdir(), fileName);
+    writeFileSync(filePath, content, 'utf8');
+    return filePath;
+}
+export function getPRForRun(repo, runId, token) {
+    const prs = ghJson(['api', `repos/${repo}/actions/runs/${runId}/pull_requests`], token);
+    return prs[0]?.number ?? null;
+}
+export function postOrUpdateComment(repo, prNumber, body, token) {
+    const comments = ghJson(['api', `repos/${repo}/issues/${prNumber}/comments`, '-f', 'per_page=100'], token);
+    const existing = comments.find((comment) => comment.body?.includes(CI_TRIAGE_MARKER));
+    if (existing) {
+        execFileSync('gh', ['api', `repos/${repo}/issues/comments/${existing.id}`, '-X', 'PATCH', '-f', `body=${body}`], {
+            encoding: 'utf8',
+            env: {
+                ...process.env,
+                GH_TOKEN: token,
+                GITHUB_TOKEN: token,
+            },
+        });
+        return;
+    }
+    execFileSync('gh', ['api', `repos/${repo}/issues/${prNumber}/comments`, '-f', `body=${body}`], {
+        encoding: 'utf8',
+        env: {
+            ...process.env,
+            GH_TOKEN: token,
+            GITHUB_TOKEN: token,
+        },
+    });
+}

package/dist/history.js

@@ -0,0 +1,133 @@
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, readdirSync, rmSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { parseJUnitXmlFile } from './junit.js';
+const historyCache = new Map();
+function cacheKey(repoSlug, historyDepth) {
+    return `${repoSlug}:${historyDepth}`;
+}
+function runJson(args) {
+    const out = execFileSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+    return JSON.parse(out);
+}
+function runText(args) {
+    return execFileSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+}
+function walkFiles(root) {
+    const out = [];
+    const entries = readdirSync(root);
+    for (const entry of entries) {
+        const full = join(root, entry);
+        const stats = statSync(full);
+        if (stats.isDirectory()) {
+            out.push(...walkFiles(full));
+            continue;
+        }
+        out.push(full);
+    }
+    return out;
+}
+function testKey(className, testName) {
+    return `${className}::${testName}`;
+}
+function parseLogForTests(log) {
+    const tests = {};
+    const lines = log.split('\n');
+    for (const line of lines) {
+        const pytestMatch = line.match(/^FAILED\s+(.+?::.+?)(?:\s+-\s+.+)?$/);
+        if (pytestMatch) {
+            const pathAndName = pytestMatch[1];
+            const parts = pathAndName.split('::');
+            const testName = parts.pop() ?? pathAndName;
+            const className = parts.join('::');
+            tests[testKey(className, testName)] = 'fail';
+            continue;
+        }
+        const jestMatch = line.match(/^\s*●\s+(.+?)\s+›\s+(.+?)\s*$/);
+        if (jestMatch) {
+            tests[testKey(jestMatch[1], jestMatch[2])] = 'fail';
+            continue;
+        }
+        const genericMatch = line.match(/^(FAIL|FAILED|ERROR)\s+([A-Za-z0-9_.:/-]+)\s+(.+)$/);
+        if (genericMatch) {
+            const status = genericMatch[1] === 'ERROR' ? 'error' : 'fail';
+            tests[testKey(genericMatch[2], genericMatch[3])] = status;
+        }
+    }
+    return tests;
+}
+function tryReadJUnitResults(repoSlug, runId) {
+    const tempDir = mkdtempSync(join(tmpdir(), 'ci-triage-run-'));
+    try {
+        try {
+            runText(['run', 'download', String(runId), '--repo', repoSlug, '--dir', tempDir]);
+        }
+        catch {
+            return {};
+        }
+        const xmlFiles = walkFiles(tempDir).filter((file) => file.toLowerCase().endsWith('.xml'));
+        if (xmlFiles.length === 0) {
+            return {};
+        }
+        const tests = {};
+        for (const xmlFile of xmlFiles) {
+            const parsed = parseJUnitXmlFile(xmlFile);
+            for (const result of parsed) {
+                tests[testKey(result.class_name, result.test_name)] = result.status;
+            }
+        }
+        return tests;
+    }
+    finally {
+        rmSync(tempDir, { recursive: true, force: true });
+    }
+}
+function tryReadLogResults(repoSlug, runId) {
+    try {
+        const logs = runText(['run', 'view', String(runId), '--repo', repoSlug, '--log-failed']);
+        return parseLogForTests(logs);
+    }
+    catch {
+        return {};
+    }
+}
+export function clearHistoryCache() {
+    historyCache.clear();
+}
+export async function fetchRunHistory(repoSlug, historyDepth) {
+    const key = cacheKey(repoSlug, historyDepth);
+    const cached = historyCache.get(key);
+    if (cached) {
+        return cached;
+    }
+    let runs = [];
+    try {
+        runs = runJson([
+            'run',
+            'list',
+            '--repo',
+            repoSlug,
+            '--limit',
+            String(historyDepth),
+            '--json',
+            'databaseId,conclusion,createdAt',
+        ]);
+    }
+    catch {
+        historyCache.set(key, []);
+        return [];
+    }
+    const history = runs.map((run) => {
+        const junitTests = tryReadJUnitResults(repoSlug, run.databaseId);
+        const tests = Object.keys(junitTests).length > 0 ? junitTests : tryReadLogResults(repoSlug, run.databaseId);
+        return {
+            run_id: run.databaseId,
+            created_at: run.createdAt,
+            conclusion: run.conclusion,
+            tests,
+        };
+    });
+    historyCache.set(key, history);
+    return history;
+}

package/dist/index.js

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+import { writeFileSync } from 'node:fs';
+import { classify } from './classifier.js';
+import { fetchFailedLog, fetchRunById, fetchRunMetadata, fetchRuns } from './fetcher.js';
+import { parseFailures } from './parser.js';
+import { buildJsonReport, toConsoleText, toJson, toMarkdown } from './reporter.js';
+function usage() {
+    console.error('Usage: ci-failure-triager owner/repo [limit] [--run <id>] [--md report.md] [--json]');
+    process.exit(1);
+}
+function parseArgs(argv) {
+    const repo = argv[2];
+    const args = argv.slice(3);
+    if (!repo || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
+        usage();
+    }
+    const firstPositional = args.find((arg) => !arg.startsWith('--'));
+    const parsedLimit = firstPositional ? Number(firstPositional) : 10;
+    const limit = Math.min(Math.max(Number.isFinite(parsedLimit) ? parsedLimit : 10, 1), 100);
+    const runIdx = args.indexOf('--run');
+    const runIdRaw = runIdx >= 0 ? args[runIdx + 1] : undefined;
+    const runId = runIdRaw ? Number(runIdRaw) : undefined;
+    const mdIdx = args.indexOf('--md');
+    const markdownPath = mdIdx >= 0 ? args[mdIdx + 1] ?? '' : undefined;
+    return {
+        repo,
+        runId: Number.isFinite(runId) ? runId : undefined,
+        limit,
+        markdownPath,
+        outputJson: args.includes('--json'),
+    };
+}
+function pickRun(options) {
+    if (options.runId) {
+        return fetchRunById(options.repo, options.runId);
+    }
+    const runs = fetchRuns(options.repo, options.limit);
+    return runs.find((r) => r.conclusion === 'failure') ?? null;
+}
+function main() {
+    const options = parseArgs(process.argv);
+    const run = pickRun(options);
+    if (!run) {
+        console.error('No failed runs found.');
+        process.exit(2);
+    }
+    const rawLog = fetchFailedLog(options.repo, run.databaseId);
+    const failures = parseFailures(rawLog);
+    const classified = failures.map((failure) => ({
+        ...failure,
+        classification: classify(failure),
+    }));
+    const report = buildJsonReport({
+        repo: options.repo,
+        run,
+        failures: classified,
+        metadata: fetchRunMetadata(options.repo, run.databaseId),
+    });
+    if (options.outputJson) {
+        process.stdout.write(toJson(report));
+    }
+    else {
+        process.stdout.write(toConsoleText(report));
+    }
+    if (options.markdownPath) {
+        writeFileSync(options.markdownPath, toMarkdown(report), 'utf8');
+        if (!options.outputJson) {
+            process.stdout.write(`Wrote triage report: ${options.markdownPath}\n`);
+        }
+    }
+}
+main();

package/dist/index.test.js

@@ -0,0 +1,6 @@
+import { describe, it, expect } from 'vitest';
+describe('smoke', () => {
+    it('works', () => {
+        expect(true).toBe(true);
+    });
+});

package/dist/junit.js

@@ -0,0 +1,94 @@
+import { readFileSync } from 'node:fs';
+import { XMLParser, XMLValidator } from 'fast-xml-parser';
+const parser = new XMLParser({
+    ignoreAttributes: false,
+    parseAttributeValue: true,
+    trimValues: true,
+});
+function asArray(value) {
+    if (value === undefined) {
+        return [];
+    }
+    return Array.isArray(value) ? value : [value];
+}
+function extractFailure(node) {
+    if (!node) {
+        return {};
+    }
+    const first = Array.isArray(node) ? node[0] : node;
+    return {
+        message: typeof first?.['@_message'] === 'string' ? first['@_message'] : undefined,
+        stack: typeof first?.['#text'] === 'string' ? first['#text'] : undefined,
+    };
+}
+function parseTestCase(testCase) {
+    let status = 'pass';
+    let failure_message;
+    let stack_trace;
+    if (testCase.error) {
+        status = 'error';
+        const extracted = extractFailure(testCase.error);
+        failure_message = extracted.message;
+        stack_trace = extracted.stack;
+    }
+    else if (testCase.failure) {
+        status = 'fail';
+        const extracted = extractFailure(testCase.failure);
+        failure_message = extracted.message;
+        stack_trace = extracted.stack;
+    }
+    else if (testCase.skipped !== undefined) {
+        status = 'skipped';
+    }
+    return {
+        test_name: String(testCase['@_name'] ?? ''),
+        class_name: String(testCase['@_classname'] ?? ''),
+        duration: Number(testCase['@_time'] ?? 0),
+        status,
+        failure_message,
+        stack_trace,
+    };
+}
+function collectSuites(root) {
+    const suites = [];
+    if (root.testsuite && typeof root.testsuite === 'object') {
+        const directSuites = asArray(root.testsuite);
+        suites.push(...directSuites);
+    }
+    if (root.testsuites && typeof root.testsuites === 'object') {
+        const testsuitesNode = root.testsuites;
+        if (testsuitesNode.testsuite && typeof testsuitesNode.testsuite === 'object') {
+            const nestedSuites = asArray(testsuitesNode.testsuite);
+            suites.push(...nestedSuites);
+        }
+    }
+    return suites;
+}
+export function parseJUnitXml(xmlContent) {
+    if (!xmlContent.trim()) {
+        return [];
+    }
+    if (XMLValidator.validate(xmlContent) !== true) {
+        return [];
+    }
+    let parsed;
+    try {
+        parsed = parser.parse(xmlContent);
+    }
+    catch {
+        return [];
+    }
+    const suites = collectSuites(parsed);
+    const results = [];
+    for (const suite of suites) {
+        const testcases = asArray(suite.testcase);
+        for (const testCase of testcases) {
+            results.push(parseTestCase(testCase));
+        }
+    }
+    return results;
+}
+export function parseJUnitXmlFile(filePath) {
+    const content = readFileSync(filePath, 'utf8');
+    return parseJUnitXml(content);
+}

package/dist/mcp-server.js

@@ -0,0 +1,175 @@
+#!/usr/bin/env node
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { z } from 'zod';
+import { pathToFileURL } from 'node:url';
+import { classify } from './classifier.js';
+import { fetchFailedLog, fetchRunById, fetchRunMetadata, fetchRuns } from './fetcher.js';
+import { detectFlakes } from './flake-detector.js';
+import { parseFailures } from './parser.js';
+import { buildJsonReport } from './reporter.js';
+export const MCP_TOOL_NAMES = ['triage_run', 'list_failures', 'is_flaky', 'suggest_fix'];
+const defaultDeps = {
+    fetchRuns,
+    fetchRunById,
+    fetchFailedLog,
+    fetchRunMetadata,
+    parseFailures,
+    classify,
+    buildJsonReport,
+    detectFlakes,
+};
+function findRun(repo, runId, deps) {
+    if (runId !== undefined) {
+        return deps.fetchRunById(repo, runId);
+    }
+    const runs = deps.fetchRuns(repo, 50);
+    const failed = runs.find((run) => run.conclusion === 'failure');
+    if (!failed) {
+        throw new Error('No failed runs found.');
+    }
+    return failed;
+}
+function flattenFailures(report) {
+    const failures = [];
+    for (const job of report.jobs) {
+        for (const step of job.steps ?? []) {
+            for (const failure of step.failures ?? []) {
+                failures.push({
+                    step: step.name,
+                    category: failure.category,
+                    severity: failure.severity,
+                    error: failure.error,
+                    suggested_fix: failure.suggested_fix,
+                });
+            }
+        }
+    }
+    return failures;
+}
+function formatToolResult(data) {
+    return {
+        content: [{ type: 'text', text: JSON.stringify(data, null, 2) }],
+    };
+}
+export async function triageRun(args, deps = defaultDeps) {
+    const run = findRun(args.repo, args.run_id, deps);
+    const rawLog = deps.fetchFailedLog(args.repo, run.databaseId);
+    const failures = deps.parseFailures(rawLog);
+    const classified = failures.map((failure) => ({
+        ...failure,
+        classification: deps.classify(failure),
+    }));
+    return deps.buildJsonReport({
+        repo: args.repo,
+        run,
+        failures: classified,
+        metadata: deps.fetchRunMetadata(args.repo, run.databaseId),
+    });
+}
+export async function listFailures(args, deps = defaultDeps) {
+    const limit = args.limit ?? 10;
+    const runs = deps.fetchRuns(args.repo, Math.max(limit, 10));
+    return runs
+        .filter((run) => run.conclusion === 'failure')
+        .slice(0, limit)
+        .map((run) => ({
+        run_id: run.databaseId,
+        workflow: run.workflowName,
+        title: run.displayTitle,
+        url: run.url,
+        conclusion: run.conclusion,
+    }));
+}
+function splitTestName(rawName) {
+    if (rawName.includes('::')) {
+        const parts = rawName.split('::');
+        const testName = parts.pop() ?? rawName;
+        const className = parts.join('::');
+        return className ? { class_name: className, test_name: testName } : { test_name: testName };
+    }
+    if (rawName.includes(' > ')) {
+        const parts = rawName.split(' > ');
+        const testName = parts.pop() ?? rawName;
+        const className = parts.join(' > ');
+        return className ? { class_name: className, test_name: testName } : { test_name: testName };
+    }
+    return { test_name: rawName };
+}
+export async function isFlaky(args, deps = defaultDeps) {
+    const historyDepth = args.history_depth ?? 20;
+    const parsed = splitTestName(args.test_name);
+    const result = await deps.detectFlakes([parsed], args.repo, historyDepth);
+    const first = result[0];
+    if (!first) {
+        return {
+            test_name: parsed.test_name,
+            class_name: parsed.class_name ?? '',
+            is_flaky: false,
+            confidence: 0,
+            pass_rate: 0,
+            last_5_runs: ['unknown', 'unknown', 'unknown', 'unknown', 'unknown'],
+        };
+    }
+    return {
+        test_name: first.test_name,
+        class_name: first.class_name,
+        is_flaky: first.flaky.is_flaky,
+        confidence: first.flaky.confidence,
+        pass_rate: first.flaky.pass_rate_7d,
+        last_5_runs: first.flaky.last_5_runs,
+    };
+}
+export async function suggestFix(args, deps = defaultDeps) {
+    const report = await triageRun({ repo: args.repo, run_id: args.run_id }, deps);
+    return {
+        run_id: report.run_id,
+        failures: flattenFailures(report),
+    };
+}
+export function createMcpServer(deps = defaultDeps) {
+    const server = new McpServer({ name: 'ci-triage-mcp', version: '0.1.0' });
+    server.registerTool('triage_run', {
+        description: 'Triage a CI run and return the full TriageReport JSON.',
+        inputSchema: z.object({
+            repo: z.string().min(1),
+            run_id: z.number().int().positive().optional(),
+        }),
+    }, async (args) => formatToolResult(await triageRun(args, deps)));
+    server.registerTool('list_failures', {
+        description: 'List recent failed CI runs for a repository.',
+        inputSchema: z.object({
+            repo: z.string().min(1),
+            limit: z.number().int().min(1).max(100).default(10).optional(),
+        }),
+    }, async (args) => formatToolResult(await listFailures(args, deps)));
+    server.registerTool('is_flaky', {
+        description: 'Check if a test appears flaky based on recent run history.',
+        inputSchema: z.object({
+            repo: z.string().min(1),
+            test_name: z.string().min(1),
+            history_depth: z.number().int().min(1).max(200).default(20).optional(),
+        }),
+    }, async (args) => formatToolResult(await isFlaky(args, deps)));
+    server.registerTool('suggest_fix', {
+        description: 'Return simplified failure + suggested fix entries for a CI run.',
+        inputSchema: z.object({
+            repo: z.string().min(1),
+            run_id: z.number().int().positive(),
+        }),
+    }, async (args) => formatToolResult(await suggestFix(args, deps)));
+    return server;
+}
+export async function startMcpServer() {
+    const server = createMcpServer();
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+}
+const isMain = process.argv[1] !== undefined && pathToFileURL(process.argv[1]).href === import.meta.url;
+if (isMain) {
+    startMcpServer().catch((error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`Failed to start MCP server: ${message}\n`);
+        process.exit(1);
+    });
+}