ci-triage 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +98 -5
- package/action.yml +21 -1
- package/dist/action.js +78 -63
- package/dist/classifier.js +70 -0
- package/dist/flake-store.js +211 -0
- package/dist/index.js +485 -21
- package/dist/llm-analyzer.js +106 -0
- package/dist/multi.js +116 -0
- package/dist/parser.js +116 -0
- package/dist/providers/circleci.js +106 -0
- package/dist/providers/github.js +103 -0
- package/dist/providers/gitlab.js +96 -0
- package/dist/providers/index.js +44 -0
- package/dist/providers/types.js +2 -0
- package/dist/repo-context.js +97 -0
- package/dist/reporter.js +1 -0
- package/package.json +6 -2
package/dist/index.js
CHANGED
|
@@ -1,67 +1,526 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { writeFileSync } from 'node:fs';
|
|
3
|
+
import { createRequire } from 'node:module';
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
3
5
|
import { classify } from './classifier.js';
|
|
4
|
-
import {
|
|
5
|
-
import { parseFailures } from './parser.js';
|
|
6
|
+
import { parseAllFailures } from './parser.js';
|
|
6
7
|
import { buildJsonReport, toConsoleText, toJson, toMarkdown } from './reporter.js';
|
|
8
|
+
import { getProvider, detectProvider } from './providers/index.js';
|
|
9
|
+
import { getDbStats, getFlakes, getRemediations, markResolved, markVerified, persistRun, pruneRuns } from './flake-store.js';
|
|
10
|
+
import { analyzeLlm } from './llm-analyzer.js';
|
|
11
|
+
import { fetchRepoContext } from './repo-context.js';
|
|
12
|
+
import { buildMultiJson, detectSharedPatterns, formatMultiSummary } from './multi.js';
|
|
13
|
+
// --version support
|
|
14
|
+
const _require = createRequire(import.meta.url);
|
|
15
|
+
let pkgVersion = 'unknown';
|
|
16
|
+
try {
|
|
17
|
+
const pkg = _require('../package.json');
|
|
18
|
+
pkgVersion = pkg.version;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
// ignore
|
|
22
|
+
}
|
|
7
23
|
function usage() {
|
|
8
|
-
console.error(
|
|
24
|
+
console.error([
|
|
25
|
+
`ci-triage v${pkgVersion}`,
|
|
26
|
+
'',
|
|
27
|
+
'Usage:',
|
|
28
|
+
' ci-triage owner/repo [limit] [--run <id>] [--md report.md] [--json]',
|
|
29
|
+
' ci-triage multi owner/repo1 owner/repo2 owner/repo3 [--json]',
|
|
30
|
+
' ci-triage flakes owner/repo',
|
|
31
|
+
' ci-triage db prune [--days 90]',
|
|
32
|
+
' ci-triage db stats [owner/repo] [--json]',
|
|
33
|
+
' ci-triage resolve <owner/repo> <run-id> "<description>" [--commit <sha>]',
|
|
34
|
+
' ci-triage verify <owner/repo> <run-id> --clean-run <clean-run-id>',
|
|
35
|
+
' ci-triage history <owner/repo> [--json]',
|
|
36
|
+
'',
|
|
37
|
+
'Options:',
|
|
38
|
+
' --run <id> Triage a specific run ID',
|
|
39
|
+
' --md <path> Write markdown report to file',
|
|
40
|
+
' --json Output JSON instead of human text',
|
|
41
|
+
' --provider <p> Force CI provider: github | gitlab | circleci',
|
|
42
|
+
' --llm Enable LLM root-cause analysis (requires OPENAI_API_KEY)',
|
|
43
|
+
' --llm-model <model> Override LLM model (default: gpt-4.1-mini)',
|
|
44
|
+
' --version Print version and exit',
|
|
45
|
+
].join('\n'));
|
|
9
46
|
process.exit(1);
|
|
10
47
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
48
|
+
export const EMPTY_OUTPUT_WARNING = 'Warning: No structured failures detected. The log may contain infra/script failures\n' +
|
|
49
|
+
"that the heuristic parser doesn't recognize. Re-run with OPENAI_API_KEY set or\n" +
|
|
50
|
+
'pass --llm for root-cause analysis.\n';
|
|
51
|
+
export function shouldWarnOnEmptyOutput(totalFailures, llmRan) {
|
|
52
|
+
return totalFailures === 0 && !llmRan;
|
|
53
|
+
}
|
|
54
|
+
export function emitEmptyOutputWarning(totalFailures, llmRan) {
|
|
55
|
+
if (shouldWarnOnEmptyOutput(totalFailures, llmRan)) {
|
|
56
|
+
process.stderr.write(EMPTY_OUTPUT_WARNING);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function buildRepoContextHint(repoContext, failures) {
|
|
60
|
+
if (!repoContext)
|
|
61
|
+
return null;
|
|
62
|
+
const pagesFailure = failures.some((f) => /pages|deploy-pages|github pages/i.test(`${f.stepName} ${f.error}`));
|
|
63
|
+
if (pagesFailure && repoContext.private && !repoContext.pagesEnabled) {
|
|
64
|
+
return '(private repo, Pages not enabled)';
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
export function parseArgs(argv) {
|
|
69
|
+
const args = argv.slice(2);
|
|
70
|
+
// --version
|
|
71
|
+
if (args.includes('--version') || args.includes('-v')) {
|
|
72
|
+
process.stdout.write(`ci-triage v${pkgVersion}\n`);
|
|
73
|
+
process.exit(0);
|
|
74
|
+
}
|
|
75
|
+
// subcommand: ci-triage flakes owner/repo
|
|
76
|
+
if (args[0] === 'flakes') {
|
|
77
|
+
const repo = args[1];
|
|
78
|
+
if (!repo || !/^[A-Za-z0-9_./%-]+\/[A-Za-z0-9_./%-]+$/.test(repo)) {
|
|
79
|
+
console.error('Usage: ci-triage flakes owner/repo');
|
|
80
|
+
process.exit(1);
|
|
81
|
+
}
|
|
82
|
+
return { repo, limit: 0, outputJson: false, subcommand: 'flakes' };
|
|
83
|
+
}
|
|
84
|
+
// subcommand: ci-triage multi owner/repo1 owner/repo2 [--json]
|
|
85
|
+
if (args[0] === 'multi') {
|
|
86
|
+
const repos = args.slice(1).filter((arg) => !arg.startsWith('--'));
|
|
87
|
+
if (repos.length === 0 ||
|
|
88
|
+
repos.some((repo) => !/^[A-Za-z0-9_./%-]+\/[A-Za-z0-9_./%-]+$/.test(repo))) {
|
|
89
|
+
console.error('Usage: ci-triage multi owner/repo1 owner/repo2 owner/repo3 [--json]');
|
|
90
|
+
process.exit(1);
|
|
91
|
+
}
|
|
92
|
+
return { repo: repos[0], repos, limit: 0, outputJson: args.includes('--json'), subcommand: 'multi' };
|
|
93
|
+
}
|
|
94
|
+
// subcommand: ci-triage db prune [--days 90]
|
|
95
|
+
if (args[0] === 'db' && args[1] === 'prune') {
|
|
96
|
+
const daysIdx = args.indexOf('--days');
|
|
97
|
+
const days = daysIdx >= 0 ? Number(args[daysIdx + 1]) : 90;
|
|
98
|
+
return {
|
|
99
|
+
repo: '',
|
|
100
|
+
limit: 0,
|
|
101
|
+
outputJson: args.includes('--json'),
|
|
102
|
+
subcommand: 'db-prune',
|
|
103
|
+
dbPruneDays: isFinite(days) ? days : 90,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
// subcommand: ci-triage db stats [owner/repo]
|
|
107
|
+
if (args[0] === 'db' && args[1] === 'stats') {
|
|
108
|
+
const repo = args[2];
|
|
109
|
+
if (repo && !/^[A-Za-z0-9_./%-]+\/[A-Za-z0-9_./%-]+$/.test(repo)) {
|
|
110
|
+
console.error('Usage: ci-triage db stats [owner/repo]');
|
|
111
|
+
process.exit(1);
|
|
112
|
+
}
|
|
113
|
+
return {
|
|
114
|
+
repo: '',
|
|
115
|
+
limit: 0,
|
|
116
|
+
outputJson: args.includes('--json'),
|
|
117
|
+
subcommand: 'db-stats',
|
|
118
|
+
dbStatRepo: repo,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
// subcommand: ci-triage resolve <owner/repo> <run-id> "<description>" [--commit <sha>]
|
|
122
|
+
if (args[0] === 'resolve') {
|
|
123
|
+
const repo = args[1];
|
|
124
|
+
const resolveRunId = args[2];
|
|
125
|
+
const resolveDescription = args[3];
|
|
126
|
+
if (!repo || !resolveRunId || !resolveDescription) {
|
|
127
|
+
console.error('Usage: ci-triage resolve owner/repo run-id "description" [--commit <sha>]');
|
|
128
|
+
process.exit(1);
|
|
129
|
+
}
|
|
130
|
+
const commitIdx = args.indexOf('--commit');
|
|
131
|
+
const resolveCommit = commitIdx >= 0 ? args[commitIdx + 1] : undefined;
|
|
132
|
+
return { repo, limit: 0, outputJson: false, subcommand: 'resolve', resolveRunId, resolveDescription, resolveCommit };
|
|
133
|
+
}
|
|
134
|
+
// subcommand: ci-triage verify <owner/repo> <run-id> --clean-run <clean-run-id>
|
|
135
|
+
if (args[0] === 'verify') {
|
|
136
|
+
const repo = args[1];
|
|
137
|
+
const verifyRunId = args[2];
|
|
138
|
+
const cleanRunIdx = args.indexOf('--clean-run');
|
|
139
|
+
const cleanRunId = cleanRunIdx >= 0 ? args[cleanRunIdx + 1] : undefined;
|
|
140
|
+
if (!repo || !verifyRunId || !cleanRunId) {
|
|
141
|
+
console.error('Usage: ci-triage verify owner/repo run-id --clean-run clean-run-id');
|
|
142
|
+
process.exit(1);
|
|
143
|
+
}
|
|
144
|
+
return { repo, limit: 0, outputJson: false, subcommand: 'verify', verifyRunId, cleanRunId };
|
|
145
|
+
}
|
|
146
|
+
// subcommand: ci-triage history <owner/repo> [--json]
|
|
147
|
+
if (args[0] === 'history') {
|
|
148
|
+
const historyRepo = args[1];
|
|
149
|
+
if (!historyRepo || !/^[A-Za-z0-9_./%-]+\/[A-Za-z0-9_./%-]+$/.test(historyRepo)) {
|
|
150
|
+
console.error('Usage: ci-triage history owner/repo [--json]');
|
|
151
|
+
process.exit(1);
|
|
152
|
+
}
|
|
153
|
+
return { repo: '', limit: 0, outputJson: args.includes('--json'), subcommand: 'history', historyRepo };
|
|
154
|
+
}
|
|
155
|
+
const repo = args[0];
|
|
156
|
+
if (!repo || !/^[A-Za-z0-9_./%-]+\/[A-Za-z0-9_./%-]+$/.test(repo)) {
|
|
15
157
|
usage();
|
|
16
158
|
}
|
|
17
|
-
const
|
|
18
|
-
const parsedLimit =
|
|
159
|
+
const positionalArgs = args.slice(1).filter((arg) => !arg.startsWith('--'));
|
|
160
|
+
const parsedLimit = positionalArgs[0] ? Number(positionalArgs[0]) : 10;
|
|
19
161
|
const limit = Math.min(Math.max(Number.isFinite(parsedLimit) ? parsedLimit : 10, 1), 100);
|
|
20
162
|
const runIdx = args.indexOf('--run');
|
|
21
163
|
const runIdRaw = runIdx >= 0 ? args[runIdx + 1] : undefined;
|
|
22
164
|
const runId = runIdRaw ? Number(runIdRaw) : undefined;
|
|
23
165
|
const mdIdx = args.indexOf('--md');
|
|
24
166
|
const markdownPath = mdIdx >= 0 ? args[mdIdx + 1] ?? '' : undefined;
|
|
167
|
+
const providerIdx = args.indexOf('--provider');
|
|
168
|
+
const provider = providerIdx >= 0 ? args[providerIdx + 1] : undefined;
|
|
169
|
+
const llmModelIdx = args.indexOf('--llm-model');
|
|
170
|
+
const llmModel = llmModelIdx >= 0 ? args[llmModelIdx + 1] : undefined;
|
|
25
171
|
return {
|
|
26
172
|
repo,
|
|
27
173
|
runId: Number.isFinite(runId) ? runId : undefined,
|
|
28
174
|
limit,
|
|
29
175
|
markdownPath,
|
|
30
176
|
outputJson: args.includes('--json'),
|
|
177
|
+
provider,
|
|
178
|
+
llm: args.includes('--llm'),
|
|
179
|
+
llmModel,
|
|
31
180
|
};
|
|
32
181
|
}
|
|
33
|
-
function
|
|
34
|
-
|
|
35
|
-
|
|
182
|
+
async function runFlakesCommand(repo) {
|
|
183
|
+
const flakes = getFlakes(repo);
|
|
184
|
+
if (flakes.length === 0) {
|
|
185
|
+
console.log(`No flaky tests found for ${repo} in local history.`);
|
|
186
|
+
console.log('Run ci-triage a few times to build up history.');
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
console.log(`\nKnown flaky tests in ${repo}:\n`);
|
|
190
|
+
console.log(`${'Test Name'.padEnd(60)} ${'Fails'.padStart(6)} ${'Passes'.padStart(7)} ${'Ratio'.padStart(7)} Last Seen`);
|
|
191
|
+
console.log('-'.repeat(110));
|
|
192
|
+
for (const f of flakes) {
|
|
193
|
+
const ratio = `${(f.flake_ratio * 100).toFixed(1)}%`;
|
|
194
|
+
const lastSeen = f.last_seen.slice(0, 10);
|
|
195
|
+
console.log(`${f.test_name.slice(0, 59).padEnd(60)} ${String(f.fail_count).padStart(6)} ${String(f.pass_count).padStart(7)} ${ratio.padStart(7)} ${lastSeen}`);
|
|
196
|
+
}
|
|
197
|
+
console.log(`\nTotal flaky tests: ${flakes.length}`);
|
|
198
|
+
}
|
|
199
|
+
async function runDbStatsCommand(repo, outputJson) {
|
|
200
|
+
const stats = getDbStats(repo);
|
|
201
|
+
if (outputJson) {
|
|
202
|
+
process.stdout.write(`${JSON.stringify(stats, null, 2)}\n`);
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
const rows = [
|
|
206
|
+
['scope', repo ?? 'all repos'],
|
|
207
|
+
['run_count', String(stats.run_count)],
|
|
208
|
+
['test_result_count', String(stats.test_result_count)],
|
|
209
|
+
['flaky_test_count', String(stats.flaky_test_count)],
|
|
210
|
+
['db_path', stats.db_path],
|
|
211
|
+
['db_size_bytes', String(stats.db_size_bytes)],
|
|
212
|
+
];
|
|
213
|
+
const keyWidth = Math.max(...rows.map(([k]) => k.length));
|
|
214
|
+
const valueWidth = Math.max(...rows.map(([, v]) => v.length));
|
|
215
|
+
const border = `+${'-'.repeat(keyWidth + 2)}+${'-'.repeat(valueWidth + 2)}+`;
|
|
216
|
+
console.log(border);
|
|
217
|
+
console.log(`| ${'metric'.padEnd(keyWidth)} | ${'value'.padEnd(valueWidth)} |`);
|
|
218
|
+
console.log(border);
|
|
219
|
+
for (const [key, value] of rows) {
|
|
220
|
+
console.log(`| ${key.padEnd(keyWidth)} | ${value.padEnd(valueWidth)} |`);
|
|
221
|
+
}
|
|
222
|
+
console.log(border);
|
|
223
|
+
}
|
|
224
|
+
function runResolveCommand(repo, runId, description, commit) {
|
|
225
|
+
markResolved(repo, runId, 'manual', description, commit);
|
|
226
|
+
console.log(`✓ Marked run ${runId} as resolved: ${description}`);
|
|
227
|
+
}
|
|
228
|
+
function runVerifyCommand(repo, runId, cleanRunId) {
|
|
229
|
+
markVerified(repo, runId, cleanRunId);
|
|
230
|
+
console.log(`✓ Verified fix: run ${runId} resolved by clean run ${cleanRunId}`);
|
|
231
|
+
}
|
|
232
|
+
function runHistoryCommand(repo, outputJson) {
|
|
233
|
+
const records = getRemediations(repo);
|
|
234
|
+
if (outputJson) {
|
|
235
|
+
process.stdout.write(`${JSON.stringify(records, null, 2)}\n`);
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
if (records.length === 0) {
|
|
239
|
+
console.log(`No remediation history found for ${repo}.`);
|
|
240
|
+
return;
|
|
36
241
|
}
|
|
37
|
-
const
|
|
38
|
-
|
|
242
|
+
const cols = {
|
|
243
|
+
run_id: Math.max(6, ...records.map((r) => r.run_id.length)),
|
|
244
|
+
fix_action: Math.max(10, ...records.map((r) => r.fix_action.length)),
|
|
245
|
+
description: Math.max(11, ...records.map((r) => r.description.length)),
|
|
246
|
+
resolved_at: 10,
|
|
247
|
+
verified: 8,
|
|
248
|
+
};
|
|
249
|
+
const sep = `+-${'-'.repeat(cols.run_id)}-+-${'-'.repeat(cols.fix_action)}-+-${'-'.repeat(cols.description)}-+-${'-'.repeat(cols.resolved_at)}-+-${'-'.repeat(cols.verified)}-+`;
|
|
250
|
+
const header = `| ${'run_id'.padEnd(cols.run_id)} | ${'fix_action'.padEnd(cols.fix_action)} | ${'description'.padEnd(cols.description)} | ${'resolved_at'.padEnd(cols.resolved_at)} | ${'verified?'.padEnd(cols.verified)} |`;
|
|
251
|
+
console.log(sep);
|
|
252
|
+
console.log(header);
|
|
253
|
+
console.log(sep);
|
|
254
|
+
for (const r of records) {
|
|
255
|
+
const desc = r.description.length > cols.description ? r.description.slice(0, cols.description - 1) + '…' : r.description;
|
|
256
|
+
const line = `| ${r.run_id.padEnd(cols.run_id)} | ${r.fix_action.padEnd(cols.fix_action)} | ${desc.padEnd(cols.description)} | ${r.resolved_at.slice(0, 10).padEnd(cols.resolved_at)} | ${(r.is_verified ? 'yes' : 'no').padEnd(cols.verified)} |`;
|
|
257
|
+
console.log(line);
|
|
258
|
+
}
|
|
259
|
+
console.log(sep);
|
|
260
|
+
console.log(`\nTotal: ${records.length} remediation(s)`);
|
|
39
261
|
}
|
|
40
|
-
function
|
|
262
|
+
export async function triageRepo(options) {
|
|
263
|
+
const providerName = options.provider ?? detectProvider();
|
|
264
|
+
const provider = getProvider(providerName);
|
|
265
|
+
const repoContextPromise = fetchRepoContext(options.repo).catch(() => null);
|
|
266
|
+
const canHandle = await provider.canHandle(options.repo);
|
|
267
|
+
if (!canHandle) {
|
|
268
|
+
throw new Error(`Provider ${providerName} cannot handle repo ${options.repo}`);
|
|
269
|
+
}
|
|
270
|
+
const run = await provider.resolveRun(options.repo, options.runId);
|
|
271
|
+
if (!run) {
|
|
272
|
+
throw new Error('No failed runs found.');
|
|
273
|
+
}
|
|
274
|
+
const runRef = { provider: providerName, repo: options.repo, runId: run.id };
|
|
275
|
+
const [logBundle, metadata] = await Promise.all([
|
|
276
|
+
provider.fetchLogs(runRef),
|
|
277
|
+
provider.fetchMetadata(runRef),
|
|
278
|
+
]);
|
|
279
|
+
const rawLog = logBundle.combined;
|
|
280
|
+
const failures = parseAllFailures(rawLog);
|
|
281
|
+
const classified = failures.map((failure) => ({
|
|
282
|
+
...failure,
|
|
283
|
+
classification: classify(failure),
|
|
284
|
+
}));
|
|
285
|
+
const runInfo = {
|
|
286
|
+
databaseId: Number(run.id) || 0,
|
|
287
|
+
displayTitle: run.displayTitle,
|
|
288
|
+
workflowName: run.workflowName,
|
|
289
|
+
conclusion: run.conclusion,
|
|
290
|
+
url: run.url,
|
|
291
|
+
};
|
|
292
|
+
const report = buildJsonReport({
|
|
293
|
+
repo: options.repo,
|
|
294
|
+
run: runInfo,
|
|
295
|
+
failures: classified,
|
|
296
|
+
metadata: {
|
|
297
|
+
headSha: metadata.headSha,
|
|
298
|
+
headBranch: metadata.headBranch,
|
|
299
|
+
event: metadata.event,
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
const llmEnabled = options.llm || !!process.env['OPENAI_API_KEY'];
|
|
303
|
+
if (llmEnabled) {
|
|
304
|
+
const repoContext = await repoContextPromise;
|
|
305
|
+
const failureEntries = classified.map((f) => ({
|
|
306
|
+
type: f.classification?.type ?? 'unknown',
|
|
307
|
+
error: f.error,
|
|
308
|
+
stack: f.stack?.join('\n'),
|
|
309
|
+
category: f.classification?.category ?? 'unknown',
|
|
310
|
+
severity: f.classification?.severity ?? 'low',
|
|
311
|
+
suggested_fix: f.classification?.suggestedFix ?? '',
|
|
312
|
+
fix_action: f.classification?.fixAction ?? 'unknown',
|
|
313
|
+
flaky: { is_flaky: false, confidence: 0, pass_rate_7d: 1, last_5_runs: [] },
|
|
314
|
+
}));
|
|
315
|
+
const analysis = await analyzeLlm(failureEntries, rawLog, {
|
|
316
|
+
model: options.llmModel,
|
|
317
|
+
enabled: llmEnabled,
|
|
318
|
+
repoContext,
|
|
319
|
+
});
|
|
320
|
+
report.analysis = analysis;
|
|
321
|
+
}
|
|
322
|
+
try {
|
|
323
|
+
const testMap = {};
|
|
324
|
+
for (const f of classified) {
|
|
325
|
+
if (f.stepName)
|
|
326
|
+
testMap[f.stepName] = 'fail';
|
|
327
|
+
}
|
|
328
|
+
persistRun(options.repo, String(run.id), new Date().toISOString(), testMap);
|
|
329
|
+
}
|
|
330
|
+
catch {
|
|
331
|
+
// non-fatal
|
|
332
|
+
}
|
|
333
|
+
return report;
|
|
334
|
+
}
|
|
335
|
+
async function runMultiCommand(options) {
|
|
336
|
+
const reports = [];
|
|
337
|
+
const repos = options.repos ?? [];
|
|
338
|
+
for (const repo of repos) {
|
|
339
|
+
try {
|
|
340
|
+
const report = await triageRepo({
|
|
341
|
+
repo,
|
|
342
|
+
provider: options.provider,
|
|
343
|
+
llm: options.llm,
|
|
344
|
+
llmModel: options.llmModel,
|
|
345
|
+
});
|
|
346
|
+
reports.push(report);
|
|
347
|
+
}
|
|
348
|
+
catch (err) {
|
|
349
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
350
|
+
console.error(`Failed to triage ${repo}: ${message}`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
const patterns = detectSharedPatterns(reports);
|
|
354
|
+
if (options.outputJson) {
|
|
355
|
+
process.stdout.write(buildMultiJson(reports, patterns));
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
process.stdout.write(formatMultiSummary(reports, patterns));
|
|
359
|
+
}
|
|
360
|
+
async function main() {
|
|
41
361
|
const options = parseArgs(process.argv);
|
|
42
|
-
|
|
362
|
+
// Handle subcommands
|
|
363
|
+
if (options.subcommand === 'flakes') {
|
|
364
|
+
await runFlakesCommand(options.repo);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
if (options.subcommand === 'db-prune') {
|
|
368
|
+
const days = options.dbPruneDays ?? 90;
|
|
369
|
+
const deleted = pruneRuns(days);
|
|
370
|
+
console.log(`Pruned ${deleted} run(s) older than ${days} days from ~/.ci-triage/flake.db`);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
if (options.subcommand === 'db-stats') {
|
|
374
|
+
await runDbStatsCommand(options.dbStatRepo, options.outputJson);
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
if (options.subcommand === 'resolve') {
|
|
378
|
+
runResolveCommand(options.repo, options.resolveRunId, options.resolveDescription, options.resolveCommit);
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
if (options.subcommand === 'verify') {
|
|
382
|
+
runVerifyCommand(options.repo, options.verifyRunId, options.cleanRunId);
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
if (options.subcommand === 'history') {
|
|
386
|
+
runHistoryCommand(options.historyRepo, options.outputJson);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (options.subcommand === 'multi') {
|
|
390
|
+
await runMultiCommand(options);
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
// Resolve provider
|
|
394
|
+
const providerName = options.provider ?? detectProvider();
|
|
395
|
+
const provider = getProvider(providerName);
|
|
396
|
+
const repoContextPromise = fetchRepoContext(options.repo).catch(() => null);
|
|
397
|
+
const canHandle = await provider.canHandle(options.repo);
|
|
398
|
+
if (!canHandle) {
|
|
399
|
+
process.exit(1);
|
|
400
|
+
}
|
|
401
|
+
// Resolve run
|
|
402
|
+
let run = null;
|
|
403
|
+
try {
|
|
404
|
+
run = await provider.resolveRun(options.repo, options.runId);
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
408
|
+
process.exit(2);
|
|
409
|
+
}
|
|
43
410
|
if (!run) {
|
|
44
411
|
console.error('No failed runs found.');
|
|
45
412
|
process.exit(2);
|
|
46
413
|
}
|
|
47
|
-
|
|
48
|
-
const
|
|
414
|
+
// Fetch logs and metadata in parallel
|
|
415
|
+
const runRef = { provider: providerName, repo: options.repo, runId: run.id };
|
|
416
|
+
const [logBundle, metadata] = await Promise.all([
|
|
417
|
+
provider.fetchLogs(runRef),
|
|
418
|
+
provider.fetchMetadata(runRef),
|
|
419
|
+
]);
|
|
420
|
+
const rawLog = logBundle.combined;
|
|
421
|
+
const failures = parseAllFailures(rawLog);
|
|
49
422
|
const classified = failures.map((failure) => ({
|
|
50
423
|
...failure,
|
|
51
424
|
classification: classify(failure),
|
|
52
425
|
}));
|
|
426
|
+
// Build compat RunInfo for reporter
|
|
427
|
+
const runInfo = {
|
|
428
|
+
databaseId: Number(run.id) || 0,
|
|
429
|
+
displayTitle: run.displayTitle,
|
|
430
|
+
workflowName: run.workflowName,
|
|
431
|
+
conclusion: run.conclusion,
|
|
432
|
+
url: run.url,
|
|
433
|
+
};
|
|
53
434
|
const report = buildJsonReport({
|
|
54
435
|
repo: options.repo,
|
|
55
|
-
run,
|
|
436
|
+
run: runInfo,
|
|
56
437
|
failures: classified,
|
|
57
|
-
metadata:
|
|
438
|
+
metadata: {
|
|
439
|
+
headSha: metadata.headSha,
|
|
440
|
+
headBranch: metadata.headBranch,
|
|
441
|
+
event: metadata.event,
|
|
442
|
+
},
|
|
58
443
|
});
|
|
444
|
+
// LLM analysis (gated)
|
|
445
|
+
const llmEnabled = options.llm || !!process.env['OPENAI_API_KEY'];
|
|
446
|
+
if (llmEnabled) {
|
|
447
|
+
const repoContext = await repoContextPromise;
|
|
448
|
+
const failureEntries = classified.map((f) => ({
|
|
449
|
+
type: f.classification?.type ?? 'unknown',
|
|
450
|
+
error: f.error,
|
|
451
|
+
stack: f.stack?.join('\n'),
|
|
452
|
+
category: f.classification?.category ?? 'unknown',
|
|
453
|
+
severity: f.classification?.severity ?? 'low',
|
|
454
|
+
suggested_fix: f.classification?.suggestedFix ?? '',
|
|
455
|
+
flaky: { is_flaky: false, confidence: 0, pass_rate_7d: 1, last_5_runs: [] },
|
|
456
|
+
}));
|
|
457
|
+
const analysis = await analyzeLlm(failureEntries, rawLog, {
|
|
458
|
+
model: options.llmModel,
|
|
459
|
+
enabled: llmEnabled,
|
|
460
|
+
repoContext,
|
|
461
|
+
});
|
|
462
|
+
report.analysis = analysis;
|
|
463
|
+
}
|
|
464
|
+
// Persist to SQLite flake store (best-effort)
|
|
465
|
+
try {
|
|
466
|
+
const testMap = {};
|
|
467
|
+
for (const f of classified) {
|
|
468
|
+
if (f.stepName)
|
|
469
|
+
testMap[f.stepName] = 'fail';
|
|
470
|
+
}
|
|
471
|
+
persistRun(options.repo, String(run.id), new Date().toISOString(), testMap);
|
|
472
|
+
}
|
|
473
|
+
catch {
|
|
474
|
+
// non-fatal
|
|
475
|
+
}
|
|
476
|
+
// Surface remediation status (best-effort)
|
|
477
|
+
try {
|
|
478
|
+
const remediations = getRemediations(options.repo);
|
|
479
|
+
const currentRunStr = String(run.id);
|
|
480
|
+
const currentResolved = remediations.find((r) => r.run_id === currentRunStr);
|
|
481
|
+
if (currentResolved) {
|
|
482
|
+
console.log(`✓ This run was previously marked as resolved`);
|
|
483
|
+
}
|
|
484
|
+
const unresolved = remediations.filter((r) => r.run_id !== currentRunStr && !r.is_verified);
|
|
485
|
+
if (unresolved.length > 0) {
|
|
486
|
+
console.log(`⚠ ${unresolved.length} previously unresolved run(s) for this repo`);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
catch {
|
|
490
|
+
// non-fatal
|
|
491
|
+
}
|
|
59
492
|
if (options.outputJson) {
|
|
60
493
|
process.stdout.write(toJson(report));
|
|
61
494
|
}
|
|
62
495
|
else {
|
|
63
496
|
process.stdout.write(toConsoleText(report));
|
|
497
|
+
const repoContext = await repoContextPromise;
|
|
498
|
+
const repoHint = buildRepoContextHint(repoContext, classified.map((f) => ({ stepName: f.stepName, error: f.error })));
|
|
499
|
+
if (repoHint) {
|
|
500
|
+
console.log(repoHint);
|
|
501
|
+
}
|
|
502
|
+
// Print LLM analysis summary if available
|
|
503
|
+
if (report.analysis?.mode === 'llm') {
|
|
504
|
+
console.log('\n── LLM Root-Cause Analysis ─────────────────────────────');
|
|
505
|
+
console.log(`Model: ${report.analysis.model} (${report.analysis.provider})`);
|
|
506
|
+
if (report.analysis.root_cause) {
|
|
507
|
+
console.log(`Root Cause: ${report.analysis.root_cause}`);
|
|
508
|
+
}
|
|
509
|
+
if (report.analysis.fix_suggestions?.length) {
|
|
510
|
+
console.log('Fix Suggestions:');
|
|
511
|
+
for (const s of report.analysis.fix_suggestions) {
|
|
512
|
+
console.log(` • ${s}`);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
if (report.analysis.llm?.usage) {
|
|
516
|
+
const u = report.analysis.llm.usage;
|
|
517
|
+
console.log(`Tokens: ${u.input_tokens} in / ${u.output_tokens} out — est. $${u.estimated_cost_usd.toFixed(4)}`);
|
|
518
|
+
}
|
|
519
|
+
console.log('─────────────────────────────────────────────────────────\n');
|
|
520
|
+
}
|
|
64
521
|
}
|
|
522
|
+
const llmRan = report.analysis?.mode === 'llm';
|
|
523
|
+
emitEmptyOutputWarning(report.summary.total_failures, llmRan);
|
|
65
524
|
if (options.markdownPath) {
|
|
66
525
|
writeFileSync(options.markdownPath, toMarkdown(report), 'utf8');
|
|
67
526
|
if (!options.outputJson) {
|
|
@@ -69,4 +528,9 @@ function main() {
|
|
|
69
528
|
}
|
|
70
529
|
}
|
|
71
530
|
}
|
|
72
|
-
|
|
531
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
532
|
+
main().catch((err) => {
|
|
533
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
534
|
+
process.exit(1);
|
|
535
|
+
});
|
|
536
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
/** Pricing per 1M tokens (update as models change) */
|
|
3
|
+
const PRICING_PER_1M = {
|
|
4
|
+
'gpt-4.1-mini': { input: 0.40, output: 1.60 },
|
|
5
|
+
'gpt-4.1': { input: 2.00, output: 8.00 },
|
|
6
|
+
'gpt-4o-mini': { input: 0.15, output: 0.60 },
|
|
7
|
+
'gpt-4o': { input: 2.50, output: 10.00 },
|
|
8
|
+
};
|
|
9
|
+
function estimateCost(model, inputTokens, outputTokens) {
|
|
10
|
+
const pricing = PRICING_PER_1M[model] ?? { input: 0.40, output: 1.60 };
|
|
11
|
+
return (inputTokens / 1_000_000) * pricing.input + (outputTokens / 1_000_000) * pricing.output;
|
|
12
|
+
}
|
|
13
|
+
function formatRepoContext(context) {
|
|
14
|
+
const visibilityLine = `- Visibility: ${context.visibility}${context.plan ? ` (${context.plan} plan)` : ''}`;
|
|
15
|
+
const pagesLine = `- GitHub Pages: ${context.pagesEnabled ? 'enabled' : 'not enabled'}`;
|
|
16
|
+
const scanningLine = `- Code scanning: ${context.hasCodeScanning ? 'enabled' : 'not enabled'}`;
|
|
17
|
+
const workflowsLine = `- Workflow files: ${context.workflowFiles.length ? context.workflowFiles.join(', ') : 'none found'}`;
|
|
18
|
+
const scopesLine = `- Token scopes: ${context.tokenScopes.length ? context.tokenScopes.join(', ') : 'none detected'}`;
|
|
19
|
+
return [
|
|
20
|
+
'Repository context:',
|
|
21
|
+
visibilityLine,
|
|
22
|
+
pagesLine,
|
|
23
|
+
scanningLine,
|
|
24
|
+
workflowsLine,
|
|
25
|
+
scopesLine,
|
|
26
|
+
'',
|
|
27
|
+
'Given this context, consider whether workflow files should be removed rather than fixed.',
|
|
28
|
+
'',
|
|
29
|
+
].join('\n');
|
|
30
|
+
}
|
|
31
|
+
export function buildPrompt(failures, logExcerpt, repoContext = null) {
|
|
32
|
+
const failureSummary = failures
|
|
33
|
+
.slice(0, 5)
|
|
34
|
+
.map((f) => `- [${f.category}] ${f.error}${f.stack ? `\n Stack: ${f.stack.slice(0, 300)}` : ''}`)
|
|
35
|
+
.join('\n');
|
|
36
|
+
const logSnippet = logExcerpt.slice(-3000); // last 3k chars of log
|
|
37
|
+
const contextPrefix = repoContext ? `${formatRepoContext(repoContext)}\n` : '';
|
|
38
|
+
return `${contextPrefix}You are a CI failure analyst. Given the following CI failures and log excerpt, provide:
|
|
39
|
+
1. A concise root cause (1-2 sentences)
|
|
40
|
+
2. Up to 3 specific fix suggestions
|
|
41
|
+
|
|
42
|
+
Failures:
|
|
43
|
+
${failureSummary}
|
|
44
|
+
|
|
45
|
+
Log excerpt (last section):
|
|
46
|
+
${logSnippet}
|
|
47
|
+
|
|
48
|
+
Respond in JSON format:
|
|
49
|
+
{
|
|
50
|
+
"root_cause": "...",
|
|
51
|
+
"fix_suggestions": ["...", "..."]
|
|
52
|
+
}`;
|
|
53
|
+
}
|
|
54
|
+
export async function analyzeLlm(failures, logExcerpt, options = {}) {
|
|
55
|
+
const model = options.model ?? 'gpt-4.1-mini';
|
|
56
|
+
const enabled = options.enabled ?? (!!process.env['OPENAI_API_KEY']);
|
|
57
|
+
if (!enabled || !process.env['OPENAI_API_KEY']) {
|
|
58
|
+
return { mode: 'heuristic', fallback_reason: 'OPENAI_API_KEY not set or --llm not passed' };
|
|
59
|
+
}
|
|
60
|
+
const client = new OpenAI({ apiKey: process.env['OPENAI_API_KEY'] });
|
|
61
|
+
try {
|
|
62
|
+
const prompt = buildPrompt(failures, logExcerpt, options.repoContext ?? null);
|
|
63
|
+
const response = await client.responses.create({
|
|
64
|
+
model,
|
|
65
|
+
input: prompt,
|
|
66
|
+
});
|
|
67
|
+
const inputTokens = response.usage?.input_tokens ?? 0;
|
|
68
|
+
const outputTokens = response.usage?.output_tokens ?? 0;
|
|
69
|
+
const estimatedCost = estimateCost(model, inputTokens, outputTokens);
|
|
70
|
+
const text = response.output_text ?? '';
|
|
71
|
+
let root_cause = '';
|
|
72
|
+
let fix_suggestions = [];
|
|
73
|
+
try {
|
|
74
|
+
// Strip markdown code fences if present
|
|
75
|
+
const cleaned = text.replace(/```(?:json)?\s*/g, '').replace(/```\s*$/g, '').trim();
|
|
76
|
+
const parsed = JSON.parse(cleaned);
|
|
77
|
+
root_cause = parsed.root_cause ?? '';
|
|
78
|
+
fix_suggestions = parsed.fix_suggestions ?? [];
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
root_cause = text.slice(0, 500);
|
|
82
|
+
}
|
|
83
|
+
return {
|
|
84
|
+
mode: 'llm',
|
|
85
|
+
provider: 'openai',
|
|
86
|
+
model,
|
|
87
|
+
root_cause,
|
|
88
|
+
fix_suggestions,
|
|
89
|
+
llm: {
|
|
90
|
+
usage: {
|
|
91
|
+
input_tokens: inputTokens,
|
|
92
|
+
output_tokens: outputTokens,
|
|
93
|
+
estimated_cost_usd: estimatedCost,
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
100
|
+
console.error(`[ci-triage] LLM analysis failed, falling back to heuristic: ${reason}`);
|
|
101
|
+
return {
|
|
102
|
+
mode: 'heuristic',
|
|
103
|
+
fallback_reason: reason,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
}
|