@allurereport/plugin-agent 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/plugin.js ADDED
@@ -0,0 +1,2368 @@
1
+ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
2
+ if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
3
+ if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
4
+ return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
5
+ };
6
+ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
7
+ if (kind === "m") throw new TypeError("Private method is not writable");
8
+ if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
9
+ if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
10
+ return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
11
+ };
12
+ var _AgentPlugin_runtime;
13
+ import { appendFile, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
14
+ import { basename, dirname, extname, join, relative, resolve } from "node:path";
15
+ import process, { env } from "node:process";
16
+ import { formatDuration, isAttachment, isStep, } from "@allurereport/core-api";
17
+ import { parse } from "yaml";
18
+ import { renderAgentsGuide } from "./guidance.js";
19
+ const AGENT_OUTPUT_ENV = "ALLURE_AGENT_OUTPUT";
20
+ const AGENT_EXPECTATIONS_ENV = "ALLURE_AGENT_EXPECTATIONS";
21
+ const AGENT_COMMAND_ENV = "ALLURE_AGENT_COMMAND";
22
+ const AGENT_PROJECT_ROOT_ENV = "ALLURE_AGENT_PROJECT_ROOT";
23
+ const AGENT_NAME_ENV = "ALLURE_AGENT_NAME";
24
+ const AGENT_LOOP_ID_ENV = "ALLURE_AGENT_LOOP_ID";
25
+ const AGENT_TASK_ID_ENV = "ALLURE_AGENT_TASK_ID";
26
+ const AGENT_CONVERSATION_ID_ENV = "ALLURE_AGENT_CONVERSATION_ID";
27
+ const AGENT_SCHEMA_VERSION = "allure-agent-output/v1";
28
+ const MANAGED_ENTRIES = ["index.md", "AGENTS.md", "tests", "artifacts", "manifest", "project"];
29
+ const STATUS_ORDER = {
30
+ failed: 0,
31
+ broken: 1,
32
+ unknown: 2,
33
+ skipped: 3,
34
+ passed: 4,
35
+ };
36
+ const FINDING_SEVERITY_ORDER = {
37
+ high: 0,
38
+ warning: 1,
39
+ info: 2,
40
+ };
41
+ const NONTRIVIAL_DURATION_MS = 100;
42
+ const STEP_SPAM_THRESHOLD = 25;
43
+ const NOOP_RATIO_THRESHOLD = 0.8;
44
+ const ASSERTION_STEP_PATTERN = /\b(assert|expect|check|verify|validate|should)\b/i;
45
+ const LOW_VALUE_STDERR_WARNING_PATTERNS = [
46
+ /\bNO_COLOR\b/i,
47
+ /\bExperimentalWarning\b/i,
48
+ /\bDeprecationWarning\b/i,
49
+ /\bAllure TestOps\b/i,
50
+ /\bCJS build of Vite's Node API is deprecated\b/i,
51
+ ];
52
+ const VITEST_SETUP_FRAME_PATTERN = /^\d+\|\s*(beforeAll|beforeEach|afterAll|afterEach)\b/i;
53
+ const ACTIONABLE_STDERR_PATTERNS = [
54
+ {
55
+ kind: "setup",
56
+ pattern: /\b(xcresulttool|xcrun: error: unable to find utility|not a developer tool or in PATH)\b/i,
57
+ },
58
+ {
59
+ kind: "import",
60
+ pattern: /\b(ERR_MODULE_NOT_FOUND|Cannot find module|Cannot find package|Failed to resolve import|Failed to load url|Module not found)\b/i,
61
+ },
62
+ {
63
+ kind: "suite-load",
64
+ pattern: /\b(Unhandled Error|Error while loading|Failed to load test file|Failed to collect tests)\b/i,
65
+ },
66
+ {
67
+ kind: "setup",
68
+ pattern: /\b(beforeAll|beforeEach|afterAll|afterEach|global setup|setup failed|setup error)\b/i,
69
+ },
70
+ ];
71
+ const STACK_TRACE_LINE_PATTERN = /^\s*(at\s+|file:|node:internal|Caused by:\s*$|\^+$)/;
72
+ const normalizeMarkdownPath = (value) => value.replace(/\\/g, "/");
73
+ const escapeInlineMarkdown = (value) => value.replace(/\\/g, "\\\\").replace(/([`*_[\]<>])/g, "\\$1");
74
+ const sanitizePathSegment = (value, fallback) => {
75
+ const sanitized = (value ?? fallback).replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "");
76
+ return sanitized.length > 0 ? sanitized : fallback;
77
+ };
78
+ const uniqueValues = (values) => Array.from(new Set(values));
79
+ const attachmentName = (link) => ("name" in link ? link.name : undefined);
80
+ const attachmentDisplayName = (link) => attachmentName(link) ?? link.originalFileName ?? link.id;
81
+ const statusLabel = (status) => status.toUpperCase();
82
+ const compareTestResultsByStatusThenName = (left, right) => {
83
+ const byStatus = STATUS_ORDER[left.status] - STATUS_ORDER[right.status];
84
+ if (byStatus !== 0) {
85
+ return byStatus;
86
+ }
87
+ const leftName = left.fullName ?? left.name;
88
+ const rightName = right.fullName ?? right.name;
89
+ const byName = leftName.localeCompare(rightName);
90
+ if (byName !== 0) {
91
+ return byName;
92
+ }
93
+ return left.id.localeCompare(right.id);
94
+ };
95
+ const sortByNewestAttempt = (items) => [...items].sort((left, right) => {
96
+ const byStart = (right.start ?? 0) - (left.start ?? 0);
97
+ if (byStart !== 0) {
98
+ return byStart;
99
+ }
100
+ const byStop = (right.stop ?? 0) - (left.stop ?? 0);
101
+ if (byStop !== 0) {
102
+ return byStop;
103
+ }
104
+ return right.id.localeCompare(left.id);
105
+ });
106
+ const formatTimestamp = (value) => (value === undefined ? "unknown" : new Date(value).toISOString());
107
+ const formatDurationValue = (value) => formatDuration(value);
108
+ const escapeJsonPointerSegment = (value) => value.replace(/~/g, "~0").replace(/\//g, "~1");
109
+ const isFailedLikeStatus = (status) => status === "failed" || status === "broken";
110
+ const normalizeStringArray = (value) => {
111
+ if (!Array.isArray(value)) {
112
+ return [];
113
+ }
114
+ return value.filter((item) => typeof item === "string" && item.length > 0);
115
+ };
116
+ const normalizeLabelValues = (value) => {
117
+ if (!value || typeof value !== "object") {
118
+ return {};
119
+ }
120
+ return Object.fromEntries(Object.entries(value).flatMap(([name, rawValue]) => {
121
+ const values = typeof rawValue === "string"
122
+ ? [rawValue]
123
+ : Array.isArray(rawValue)
124
+ ? rawValue.filter((item) => typeof item === "string")
125
+ : [];
126
+ return values.length ? [[name, values]] : [];
127
+ }));
128
+ };
129
+ const normalizeSelectors = (input) => ({
130
+ environments: normalizeStringArray(input?.environments),
131
+ fullNames: normalizeStringArray(input?.full_names),
132
+ fullNamePrefixes: normalizeStringArray(input?.full_name_prefixes),
133
+ labelValues: normalizeLabelValues(input?.label_values),
134
+ });
135
+ const hasSelector = (selectors) => selectors.environments.length > 0 ||
136
+ selectors.fullNames.length > 0 ||
137
+ selectors.fullNamePrefixes.length > 0 ||
138
+ Object.keys(selectors.labelValues).length > 0;
139
+ const normalizeNotes = (value) => {
140
+ if (typeof value === "string") {
141
+ return value.length > 0 ? [value] : [];
142
+ }
143
+ return normalizeStringArray(value);
144
+ };
145
+ const formatLabelRequirement = (name, values) => `${name} in [${values.join(", ")}]`;
146
+ const buildLabelRecord = (labels) => {
147
+ const record = new Map();
148
+ for (const label of labels) {
149
+ if (label.value === undefined) {
150
+ continue;
151
+ }
152
+ const current = record.get(label.name) ?? [];
153
+ current.push(label.value);
154
+ record.set(label.name, current);
155
+ }
156
+ return record;
157
+ };
158
+ const matchesLabelSelectors = (labels, selector) => {
159
+ const labelRecord = buildLabelRecord(labels);
160
+ return Object.entries(selector).every(([name, values]) => {
161
+ const actual = labelRecord.get(name) ?? [];
162
+ return actual.some((value) => values.includes(value));
163
+ });
164
+ };
165
+ const collectMissingLabelSelectors = (labels, selector) => {
166
+ const labelRecord = buildLabelRecord(labels);
167
+ const mismatches = [];
168
+ for (const [name, values] of Object.entries(selector)) {
169
+ const actual = labelRecord.get(name) ?? [];
170
+ if (!actual.some((value) => values.includes(value))) {
171
+ mismatches.push(`missing ${formatLabelRequirement(name, values)}`);
172
+ }
173
+ }
174
+ return mismatches;
175
+ };
176
+ const matchSelectors = (params) => {
177
+ const { tr, environmentId, selectors, selectorRoot } = params;
178
+ const fullName = tr.fullName ?? tr.name;
179
+ const reasons = [];
180
+ const references = [];
181
+ selectors.fullNames.forEach((candidate, index) => {
182
+ if (candidate === fullName) {
183
+ reasons.push("full name");
184
+ references.push(`${selectorRoot}.full_names[${index}]`);
185
+ }
186
+ });
187
+ selectors.fullNamePrefixes.forEach((candidate, index) => {
188
+ if (fullName.startsWith(candidate)) {
189
+ reasons.push("full name prefix");
190
+ references.push(`${selectorRoot}.full_name_prefixes[${index}]`);
191
+ }
192
+ });
193
+ selectors.environments.forEach((candidate, index) => {
194
+ if (candidate === environmentId ||
195
+ sanitizePathSegment(candidate, candidate) === sanitizePathSegment(environmentId, environmentId)) {
196
+ reasons.push("environment");
197
+ references.push(`${selectorRoot}.environments[${index}]`);
198
+ }
199
+ });
200
+ const labelSelectorsPresent = Object.keys(selectors.labelValues).length > 0;
201
+ const labelMatch = labelSelectorsPresent ? matchesLabelSelectors(tr.labels, selectors.labelValues) : false;
202
+ if (labelMatch) {
203
+ reasons.push("label values");
204
+ references.push(...Object.keys(selectors.labelValues).map((name) => `${selectorRoot}.label_values/${escapeJsonPointerSegment(name)}`));
205
+ }
206
+ return {
207
+ matched: reasons.length > 0,
208
+ matchedByNonLabel: reasons.includes("full name") || reasons.includes("full name prefix") || reasons.includes("environment"),
209
+ labelMatch,
210
+ reasons: uniqueValues(reasons),
211
+ references: uniqueValues(references),
212
+ };
213
+ };
214
+ const analyzeStepTree = (steps) => {
215
+ const summary = {
216
+ totalSteps: 0,
217
+ noopSteps: 0,
218
+ meaningfulSteps: 0,
219
+ nestedSteps: 0,
220
+ attachmentRefs: 0,
221
+ assertionLikeSteps: 0,
222
+ };
223
+ const visit = (nodes) => {
224
+ for (const node of nodes) {
225
+ if (isAttachment(node)) {
226
+ summary.attachmentRefs += 1;
227
+ continue;
228
+ }
229
+ if (!isStep(node)) {
230
+ continue;
231
+ }
232
+ summary.totalSteps += 1;
233
+ const hasNestedSteps = node.steps.some(isStep);
234
+ const hasAttachmentChildren = node.steps.some(isAttachment);
235
+ const hasEvidence = node.parameters.length > 0 ||
236
+ !!node.error?.message ||
237
+ !!node.message ||
238
+ !!node.trace ||
239
+ hasNestedSteps ||
240
+ hasAttachmentChildren;
241
+ if (hasNestedSteps) {
242
+ summary.nestedSteps += 1;
243
+ }
244
+ if (ASSERTION_STEP_PATTERN.test(node.name)) {
245
+ summary.assertionLikeSteps += 1;
246
+ }
247
+ if (hasEvidence) {
248
+ summary.meaningfulSteps += 1;
249
+ }
250
+ else {
251
+ summary.noopSteps += 1;
252
+ }
253
+ if (node.steps.length) {
254
+ visit(node.steps);
255
+ }
256
+ }
257
+ };
258
+ visit(steps);
259
+ return summary;
260
+ };
261
+ const mergeStepSummaries = (items) => items.reduce((acc, item) => ({
262
+ totalSteps: acc.totalSteps + item.totalSteps,
263
+ noopSteps: acc.noopSteps + item.noopSteps,
264
+ meaningfulSteps: acc.meaningfulSteps + item.meaningfulSteps,
265
+ nestedSteps: acc.nestedSteps + item.nestedSteps,
266
+ attachmentRefs: acc.attachmentRefs + item.attachmentRefs,
267
+ assertionLikeSteps: acc.assertionLikeSteps + item.assertionLikeSteps,
268
+ }), {
269
+ totalSteps: 0,
270
+ noopSteps: 0,
271
+ meaningfulSteps: 0,
272
+ nestedSteps: 0,
273
+ attachmentRefs: 0,
274
+ assertionLikeSteps: 0,
275
+ });
276
+ const buildAttemptSignature = (attempt) => JSON.stringify({
277
+ status: attempt.tr.status,
278
+ errorMessage: attempt.tr.error?.message,
279
+ errorTrace: attempt.tr.error?.trace,
280
+ stepSummary: attempt.stepSummary,
281
+ fixtureSummary: attempt.fixtureStepSummary,
282
+ artifactNames: attempt.artifacts.map((artifact) => ({
283
+ name: artifact.displayName,
284
+ missing: artifact.missing,
285
+ })),
286
+ });
287
+ const getPackageName = (tr) => tr.labels.find(({ name }) => name === "package")?.value;
288
+ const toLabelEntries = (labels) => labels.map((label) => ({
289
+ name: label.name,
290
+ value: label.value,
291
+ }));
292
+ const toFindingCounts = (findings) => {
293
+ const counts = {
294
+ total: findings.length,
295
+ high: 0,
296
+ warning: 0,
297
+ info: 0,
298
+ };
299
+ for (const finding of findings) {
300
+ counts[finding.severity] += 1;
301
+ }
302
+ return counts;
303
+ };
304
+ const buildEnvironmentSummary = (entries) => {
305
+ const byEnvironment = new Map();
306
+ for (const entry of entries) {
307
+ const bucket = byEnvironment.get(entry.environmentId) ?? {
308
+ total: 0,
309
+ failed: 0,
310
+ broken: 0,
311
+ skipped: 0,
312
+ unknown: 0,
313
+ passed: 0,
314
+ };
315
+ bucket.total += 1;
316
+ bucket[entry.tr.status] += 1;
317
+ byEnvironment.set(entry.environmentId, bucket);
318
+ }
319
+ return Array.from(byEnvironment.entries())
320
+ .sort(([left], [right]) => left.localeCompare(right))
321
+ .map(([environmentId, stats]) => ({
322
+ environmentId,
323
+ ...stats,
324
+ }));
325
+ };
326
+ const emptyStatusCounts = () => ({
327
+ total: 0,
328
+ failed: 0,
329
+ broken: 0,
330
+ unknown: 0,
331
+ skipped: 0,
332
+ passed: 0,
333
+ });
334
+ const toStatusCountsFromStatistic = (stats) => ({
335
+ total: stats.total ?? 0,
336
+ failed: stats.failed ?? 0,
337
+ broken: stats.broken ?? 0,
338
+ unknown: stats.unknown ?? 0,
339
+ skipped: stats.skipped ?? 0,
340
+ passed: stats.passed ?? 0,
341
+ });
342
+ const toStatusCountsFromEntries = (entries) => {
343
+ const counts = emptyStatusCounts();
344
+ for (const entry of entries) {
345
+ counts.total += 1;
346
+ counts[entry.tr.status] += 1;
347
+ }
348
+ return counts;
349
+ };
350
+ const subtractStatusCounts = (left, right) => ({
351
+ total: Math.max(left.total - right.total, 0),
352
+ failed: Math.max(left.failed - right.failed, 0),
353
+ broken: Math.max(left.broken - right.broken, 0),
354
+ unknown: Math.max(left.unknown - right.unknown, 0),
355
+ skipped: Math.max(left.skipped - right.skipped, 0),
356
+ passed: Math.max(left.passed - right.passed, 0),
357
+ });
358
+ const summarizeStatusCounts = (counts) => `${counts.total} total (${counts.failed} failed, ${counts.broken} broken, ${counts.unknown} unknown, ${counts.skipped} skipped, ${counts.passed} passed)`;
359
+ const normalizeLogLine = (value) => value.replace(/\s+/g, " ").trim();
360
+ const normalizeWarningLine = (value) => normalizeLogLine(value).replace(/^\(node:\d+\)\s+Warning:\s*/i, "Warning: ");
361
+ const buildCountedValues = (values) => {
362
+ const counts = new Map();
363
+ for (const value of values) {
364
+ counts.set(value, (counts.get(value) ?? 0) + 1);
365
+ }
366
+ return Array.from(counts.entries())
367
+ .map(([message, count]) => ({ message, count }))
368
+ .sort((left, right) => right.count - left.count || left.message.localeCompare(right.message));
369
+ };
370
+ const classifyRunnerIssueKind = (value) => {
371
+ const normalized = normalizeLogLine(value);
372
+ for (const { kind, pattern } of ACTIONABLE_STDERR_PATTERNS) {
373
+ if (pattern.test(normalized)) {
374
+ return kind;
375
+ }
376
+ }
377
+ return undefined;
378
+ };
379
+ const classifyStderr = (content) => {
380
+ if (!content?.trim()) {
381
+ return {
382
+ actionable: [],
383
+ noisyWarnings: [],
384
+ };
385
+ }
386
+ const actionableLines = [];
387
+ const warningLines = [];
388
+ for (const rawLine of content.split(/\r?\n/)) {
389
+ const line = normalizeLogLine(rawLine);
390
+ if (!line || STACK_TRACE_LINE_PATTERN.test(rawLine) || VITEST_SETUP_FRAME_PATTERN.test(line)) {
391
+ continue;
392
+ }
393
+ if (LOW_VALUE_STDERR_WARNING_PATTERNS.some((pattern) => pattern.test(line))) {
394
+ warningLines.push(normalizeWarningLine(line));
395
+ continue;
396
+ }
397
+ const kind = classifyRunnerIssueKind(line);
398
+ if (kind) {
399
+ actionableLines.push({
400
+ source: "stderr",
401
+ kind,
402
+ message: line,
403
+ count: 1,
404
+ });
405
+ }
406
+ }
407
+ const actionable = Array.from(actionableLines.reduce((acc, issue) => {
408
+ const key = `${issue.source}:${issue.kind}:${issue.message}`;
409
+ const current = acc.get(key);
410
+ if (current) {
411
+ current.count += 1;
412
+ }
413
+ else {
414
+ acc.set(key, { ...issue });
415
+ }
416
+ return acc;
417
+ }, new Map()))
418
+ .map(([, issue]) => issue)
419
+ .sort((left, right) => right.count - left.count || left.message.localeCompare(right.message));
420
+ return {
421
+ actionable,
422
+ noisyWarnings: buildCountedValues(warningLines),
423
+ };
424
+ };
425
+ const summarizeGlobalErrors = (errors) => errors
426
+ .map((error) => normalizeLogLine(error.message ?? error.trace?.split(/\r?\n/)[0] ?? ""))
427
+ .filter(Boolean)
428
+ .map((message) => ({
429
+ source: "global_error",
430
+ kind: classifyRunnerIssueKind(message) ?? "global-error",
431
+ message,
432
+ count: 1,
433
+ }))
434
+ .reduce((acc, issue) => {
435
+ const key = `${issue.source}:${issue.kind}:${issue.message}`;
436
+ const current = acc.get(key);
437
+ if (current) {
438
+ current.count += 1;
439
+ }
440
+ else {
441
+ acc.set(key, issue);
442
+ }
443
+ return acc;
444
+ }, new Map());
445
+ const toRunnerIssueArray = (issues) => Array.from(issues.values()).sort((left, right) => right.count - left.count || left.message.localeCompare(right.message));
446
+ const buildModelingSummary = (params) => {
447
+ const { entries, stats, globalErrors, stderrContent } = params;
448
+ const visibleResults = toStatusCountsFromStatistic(stats);
449
+ const modeledStats = toStatusCountsFromEntries(entries);
450
+ const unmodeledFromStats = subtractStatusCounts(visibleResults, modeledStats);
451
+ const stderr = classifyStderr(stderrContent);
452
+ const globalErrorIssues = toRunnerIssueArray(summarizeGlobalErrors(globalErrors));
453
+ const stderrActionableCount = stderr.actionable.reduce((acc, issue) => acc + issue.count, 0);
454
+ const stderrWarningCount = stderr.noisyWarnings.reduce((acc, issue) => acc + issue.count, 0);
455
+ const runnerFailureSamples = [...globalErrorIssues, ...stderr.actionable].sort((left, right) => right.count - left.count || left.message.localeCompare(right.message));
456
+ const reasons = [];
457
+ if (unmodeledFromStats.total > 0) {
458
+ reasons.push(`Visible results were not fully rendered as logical tests: ${summarizeStatusCounts(unmodeledFromStats)}`);
459
+ }
460
+ if (globalErrors.length + stderrActionableCount > 0) {
461
+ reasons.push(`${globalErrors.length + stderrActionableCount} runner-level failures were detected outside logical test files.`);
462
+ }
463
+ const completeness = reasons.length > 0 ? "partial" : "complete";
464
+ return {
465
+ completeness,
466
+ reasons,
467
+ modeledStats,
468
+ unmodeledFromStats,
469
+ runnerFailures: {
470
+ total: globalErrors.length + stderrActionableCount,
471
+ globalErrors: globalErrors.length,
472
+ stderrActionable: stderrActionableCount,
473
+ samples: runnerFailureSamples.slice(0, 5),
474
+ },
475
+ stderr: {
476
+ actionableCount: stderrActionableCount,
477
+ actionableSamples: stderr.actionable.slice(0, 5).map((issue) => issue.message),
478
+ noisyWarningCount: stderrWarningCount,
479
+ noisyWarningSamples: stderr.noisyWarnings.slice(0, 5).map((warning) => warning.message),
480
+ },
481
+ compact: {
482
+ visible_results: visibleResults.total,
483
+ logical_tests: modeledStats.total,
484
+ unmodeled_visible_results: unmodeledFromStats.total,
485
+ runner_failures_outside_logical_tests: runnerFailureSamples.length,
486
+ completeness,
487
+ },
488
+ };
489
+ };
490
+ const renderCodeBlock = (content) => {
491
+ if (!content?.trim()) {
492
+ return undefined;
493
+ }
494
+ return `~~~text\n${content.trimEnd()}\n~~~`;
495
+ };
496
+ const renderParameters = (parameters) => {
497
+ if (!parameters.length) {
498
+ return "None";
499
+ }
500
+ return parameters
501
+ .map((parameter) => {
502
+ const markers = [];
503
+ let value = parameter.value;
504
+ if (parameter.hidden) {
505
+ value = "<hidden>";
506
+ }
507
+ else if (parameter.masked) {
508
+ value = "<masked>";
509
+ }
510
+ if (parameter.excluded) {
511
+ markers.push("excluded");
512
+ }
513
+ const suffix = markers.length ? ` (${markers.join(", ")})` : "";
514
+ return `- ${escapeInlineMarkdown(parameter.name)}: ${escapeInlineMarkdown(value)}${suffix}`;
515
+ })
516
+ .join("\n");
517
+ };
518
+ const renderLabels = (labels) => {
519
+ if (!labels.length) {
520
+ return "None";
521
+ }
522
+ return labels
523
+ .map(({ name, value }) => {
524
+ if (value === undefined) {
525
+ return `- ${escapeInlineMarkdown(name)}`;
526
+ }
527
+ return `- ${escapeInlineMarkdown(name)}: ${escapeInlineMarkdown(value)}`;
528
+ })
529
+ .join("\n");
530
+ };
531
+ const renderLinks = (links) => {
532
+ if (!links.length) {
533
+ return "None";
534
+ }
535
+ return links
536
+ .map(({ name, type, url }) => {
537
+ const labelParts = [type, name].filter(Boolean).map((part) => escapeInlineMarkdown(part));
538
+ const label = labelParts.length ? labelParts.join(" / ") : escapeInlineMarkdown(url);
539
+ return `- [${label}](${url})`;
540
+ })
541
+ .join("\n");
542
+ };
543
+ const renderOptionalMarkdownSection = (title, content) => {
544
+ if (!content?.trim()) {
545
+ return undefined;
546
+ }
547
+ return `## ${title}\n\n${content.trim()}`;
548
+ };
549
+ const renderError = (error) => {
550
+ if (!error || (!error.message && !error.trace && !error.actual && !error.expected)) {
551
+ return "None";
552
+ }
553
+ const lines = [];
554
+ if (error.message) {
555
+ lines.push(`- Message: ${escapeInlineMarkdown(error.message)}`);
556
+ }
557
+ if (error.actual !== undefined) {
558
+ lines.push(`- Actual: ${escapeInlineMarkdown(error.actual)}`);
559
+ }
560
+ if (error.expected !== undefined) {
561
+ lines.push(`- Expected: ${escapeInlineMarkdown(error.expected)}`);
562
+ }
563
+ const trace = renderCodeBlock(error.trace);
564
+ if (trace) {
565
+ lines.push("- Trace:");
566
+ lines.push("");
567
+ lines.push(trace);
568
+ }
569
+ return lines.join("\n");
570
+ };
571
+ const renderArtifactLine = (artifact, includeSources) => {
572
+ const details = [];
573
+ if (artifact.contentType) {
574
+ details.push(artifact.contentType);
575
+ }
576
+ if (artifact.contentLength !== undefined) {
577
+ details.push(`${artifact.contentLength} bytes`);
578
+ }
579
+ const detailsSuffix = details.length ? ` (${details.join(", ")})` : "";
580
+ const sourceSuffix = includeSources && artifact.sources.length ? ` [sources: ${artifact.sources.join("; ")}]` : "";
581
+ if (artifact.relativePath) {
582
+ return `- [${escapeInlineMarkdown(artifact.displayName)}](${normalizeMarkdownPath(artifact.relativePath)})${detailsSuffix}${sourceSuffix}`;
583
+ }
584
+ return `- ${escapeInlineMarkdown(artifact.displayName)} (missing attachment)${sourceSuffix}`;
585
+ };
586
+ const formatParameterSummary = (parameter) => {
587
+ if (parameter.hidden) {
588
+ return `${parameter.name}=<hidden>`;
589
+ }
590
+ if (parameter.masked) {
591
+ return `${parameter.name}=<masked>`;
592
+ }
593
+ return `${parameter.name}=${parameter.value}`;
594
+ };
595
+ const renderStepTree = (steps, artifactLookup, indent = 0) => {
596
+ if (!steps.length) {
597
+ return `${" ".repeat(indent)}- No steps`;
598
+ }
599
+ const lines = [];
600
+ const prefix = " ".repeat(indent);
601
+ for (const step of steps) {
602
+ if (isAttachment(step)) {
603
+ const artifact = artifactLookup.get(step.link.id);
604
+ const displayName = attachmentDisplayName(step.link);
605
+ if (artifact?.relativePath) {
606
+ lines.push(`${prefix}- Attachment: [${escapeInlineMarkdown(displayName)}](${normalizeMarkdownPath(artifact.relativePath)})`);
607
+ }
608
+ else {
609
+ lines.push(`${prefix}- Attachment: ${escapeInlineMarkdown(displayName)} (missing attachment)`);
610
+ }
611
+ continue;
612
+ }
613
+ if (!isStep(step)) {
614
+ continue;
615
+ }
616
+ const titleParts = [`[${statusLabel(step.status)}]`, escapeInlineMarkdown(step.name)];
617
+ if (step.duration !== undefined) {
618
+ titleParts.push(`(${formatDurationValue(step.duration)})`);
619
+ }
620
+ lines.push(`${prefix}- ${titleParts.join(" ")}`);
621
+ if (step.parameters.length) {
622
+ lines.push(`${prefix} - Parameters: ${step.parameters.map(formatParameterSummary).join(", ")}`);
623
+ }
624
+ if (step.error?.message) {
625
+ lines.push(`${prefix} - Error: ${escapeInlineMarkdown(step.error.message)}`);
626
+ }
627
+ if (step.steps.length) {
628
+ lines.push(renderStepTree(step.steps, artifactLookup, indent + 1));
629
+ }
630
+ }
631
+ return lines.join("\n");
632
+ };
633
+ const renderFixtureSections = (fixtures, artifactLookup) => {
634
+ if (!fixtures.length) {
635
+ return "None";
636
+ }
637
+ return fixtures
638
+ .map((fixture) => [
639
+ `### ${fixture.type === "before" ? "Before" : "After"} Fixture: ${escapeInlineMarkdown(fixture.name)}`,
640
+ "",
641
+ `- Status: ${statusLabel(fixture.status)}`,
642
+ `- Duration: ${formatDurationValue(fixture.duration)}`,
643
+ `- Started: ${formatTimestamp(fixture.start)}`,
644
+ `- Stopped: ${formatTimestamp(fixture.stop)}`,
645
+ "",
646
+ "#### Error",
647
+ "",
648
+ renderError(fixture.error),
649
+ "",
650
+ "#### Steps",
651
+ "",
652
+ renderStepTree(fixture.steps, artifactLookup),
653
+ ].join("\n"))
654
+ .join("\n\n");
655
+ };
656
+ const renderAttemptSection = (attempt) => {
657
+ const artifactLookup = new Map(attempt.artifacts.map((artifact) => [artifact.id, artifact]));
658
+ return [
659
+ `## ${attempt.heading}`,
660
+ "",
661
+ `- Status: ${statusLabel(attempt.tr.status)}`,
662
+ `- Duration: ${formatDurationValue(attempt.tr.duration)}`,
663
+ `- Started: ${formatTimestamp(attempt.tr.start)}`,
664
+ `- Stopped: ${formatTimestamp(attempt.tr.stop)}`,
665
+ `- Steps Recorded: ${attempt.stepSummary.totalSteps}`,
666
+ `- Attachments Recorded: ${attempt.artifacts.length}`,
667
+ "",
668
+ "### Error",
669
+ "",
670
+ renderError(attempt.tr.error),
671
+ "",
672
+ "### Fixtures",
673
+ "",
674
+ renderFixtureSections(attempt.fixtures, artifactLookup),
675
+ "",
676
+ "### Attachments",
677
+ "",
678
+ attempt.artifacts.length
679
+ ? attempt.artifacts.map((artifact) => renderArtifactLine(artifact, false)).join("\n")
680
+ : "None",
681
+ "",
682
+ "### Steps",
683
+ "",
684
+ renderStepTree(attempt.tr.steps, artifactLookup),
685
+ ].join("\n");
686
+ };
687
+ const renderQualityGateSection = (results) => {
688
+ if (!results.length) {
689
+ return undefined;
690
+ }
691
+ const failed = results.filter(({ success }) => !success);
692
+ const lines = ["## Quality Gate", "", `- Rules evaluated: ${results.length}`, `- Failed rules: ${failed.length}`];
693
+ if (failed.length) {
694
+ lines.push("");
695
+ lines.push("### Failures");
696
+ lines.push("");
697
+ lines.push(failed
698
+ .map((result) => {
699
+ const environmentPrefix = result.environment ? `[${result.environment}] ` : "";
700
+ return `- ${environmentPrefix}${escapeInlineMarkdown(result.rule)}: ${escapeInlineMarkdown(result.message)}`;
701
+ })
702
+ .join("\n"));
703
+ }
704
+ return lines.join("\n");
705
+ };
706
+ const renderGlobalErrors = (errors) => {
707
+ if (!errors.length) {
708
+ return undefined;
709
+ }
710
+ return [
711
+ "## Global Errors",
712
+ "",
713
+ ...errors.flatMap((error, index) => {
714
+ const lines = [`### Error ${index + 1}`, "", renderError(error)];
715
+ return [lines.join("\n"), ""];
716
+ }),
717
+ ]
718
+ .join("\n")
719
+ .trimEnd();
720
+ };
721
+ const renderRunnerIssueSummary = (issue) => {
722
+ const countSuffix = issue.count > 1 ? ` (${issue.count}x)` : "";
723
+ return `- [${issue.source.replace("_", " ")}/${issue.kind}] ${escapeInlineMarkdown(issue.message)}${countSuffix}`;
724
+ };
725
+ const renderModelingSummary = (modeling) => {
726
+ const lines = [
727
+ "## Runtime Modeling Summary",
728
+ "",
729
+ `- completeness: ${modeling.completeness}`,
730
+ `- visible results from stats: ${modeling.compact.visible_results}`,
731
+ `- logical tests rendered: ${modeling.compact.logical_tests}`,
732
+ `- unmodeled visible results: ${summarizeStatusCounts(modeling.unmodeledFromStats)}`,
733
+ `- runner failures outside logical tests: ${modeling.runnerFailures.total}`,
734
+ `- actionable stderr signals: ${modeling.stderr.actionableCount}`,
735
+ `- repeated low-value warnings: ${modeling.stderr.noisyWarningCount}`,
736
+ ];
737
+ if (modeling.reasons.length) {
738
+ lines.push(`- reasons: ${modeling.reasons.map((reason) => escapeInlineMarkdown(reason)).join(" | ")}`);
739
+ }
740
+ lines.push("");
741
+ lines.push("### High-Signal Runner Issues");
742
+ lines.push("");
743
+ lines.push(modeling.runnerFailures.samples.length
744
+ ? modeling.runnerFailures.samples.map(renderRunnerIssueSummary).join("\n")
745
+ : "None");
746
+ lines.push("");
747
+ lines.push("### Repeated Low-Value Warnings");
748
+ lines.push("");
749
+ lines.push(modeling.stderr.noisyWarningSamples.length
750
+ ? modeling.stderr.noisyWarningSamples.map((warning) => `- ${escapeInlineMarkdown(warning)}`).join("\n")
751
+ : "None");
752
+ return lines.join("\n");
753
+ };
754
+ const renderSelectorSummary = (title, selectors) => {
755
+ if (!hasSelector(selectors)) {
756
+ return `- ${title}: None`;
757
+ }
758
+ const parts = [];
759
+ if (selectors.environments.length) {
760
+ parts.push(`environments: ${selectors.environments.join(", ")}`);
761
+ }
762
+ if (selectors.fullNames.length) {
763
+ parts.push(`full names: ${selectors.fullNames.join(", ")}`);
764
+ }
765
+ if (selectors.fullNamePrefixes.length) {
766
+ parts.push(`prefixes: ${selectors.fullNamePrefixes.join(", ")}`);
767
+ }
768
+ const labelParts = Object.entries(selectors.labelValues).map(([name, values]) => formatLabelRequirement(name, values));
769
+ if (labelParts.length) {
770
+ parts.push(`labels: ${labelParts.join("; ")}`);
771
+ }
772
+ return `- ${title}: ${parts.join(" | ")}`;
773
+ };
774
+ const buildCheckSummary = (findings) => {
775
+ const countsBySeverity = {
776
+ high: 0,
777
+ warning: 0,
778
+ info: 0,
779
+ };
780
+ const countsByCategory = {
781
+ bootstrap: 0,
782
+ scope: 0,
783
+ metadata: 0,
784
+ evidence: 0,
785
+ smells: 0,
786
+ };
787
+ for (const finding of findings) {
788
+ countsBySeverity[finding.severity] += 1;
789
+ countsByCategory[finding.category] += 1;
790
+ }
791
+ return {
792
+ total: findings.length,
793
+ countsBySeverity,
794
+ countsByCategory,
795
+ };
796
+ };
797
+ const sortFindings = (findings) => [...findings].sort((left, right) => {
798
+ const bySeverity = FINDING_SEVERITY_ORDER[left.severity] - FINDING_SEVERITY_ORDER[right.severity];
799
+ if (bySeverity !== 0) {
800
+ return bySeverity;
801
+ }
802
+ const byCategory = left.category.localeCompare(right.category);
803
+ if (byCategory !== 0) {
804
+ return byCategory;
805
+ }
806
+ return left.findingId.localeCompare(right.findingId);
807
+ });
808
+ const renderFindingEvidenceLinks = (params) => {
809
+ const { finding, currentFilePath, outputDir } = params;
810
+ if (!finding.evidencePaths.length) {
811
+ return "None";
812
+ }
813
+ return finding.evidencePaths
814
+ .map((evidencePath) => {
815
+ const absolutePath = join(outputDir, evidencePath);
816
+ const markdownPath = normalizeMarkdownPath(relative(dirname(currentFilePath), absolutePath));
817
+ return `- [${escapeInlineMarkdown(evidencePath)}](${markdownPath})`;
818
+ })
819
+ .join("\n");
820
+ };
821
+ const renderFindingsSection = (params) => {
822
+ const { title, findings, currentFilePath, outputDir } = params;
823
+ if (!findings.length) {
824
+ return `## ${title}\n\nNone`;
825
+ }
826
+ const lines = [`## ${title}`, ""];
827
+ for (const finding of sortFindings(findings)) {
828
+ lines.push(`### [${finding.severity.toUpperCase()}] ${escapeInlineMarkdown(finding.category)} / ${escapeInlineMarkdown(finding.checkName)}`);
829
+ lines.push("");
830
+ lines.push(`- Message: ${escapeInlineMarkdown(finding.message)}`);
831
+ lines.push(`- Explanation: ${escapeInlineMarkdown(finding.explanation)}`);
832
+ lines.push(`- Remediation: ${escapeInlineMarkdown(finding.remediationHint)}`);
833
+ if (finding.expectedReference) {
834
+ lines.push(`- Expected Reference: ${escapeInlineMarkdown(finding.expectedReference)}`);
835
+ }
836
+ if (finding.confidence !== undefined) {
837
+ lines.push(`- Confidence: ${finding.confidence}`);
838
+ }
839
+ lines.push("- Evidence:");
840
+ lines.push("");
841
+ lines.push(renderFindingEvidenceLinks({
842
+ finding,
843
+ currentFilePath,
844
+ outputDir,
845
+ }));
846
+ lines.push("");
847
+ }
848
+ return lines.join("\n").trimEnd();
849
+ };
850
+ const renderExpectationSection = (entry) => {
851
+ const lines = [
852
+ "## Expectation Comparison",
853
+ "",
854
+ `- Scope Match: ${entry.scope.scopeMatch}`,
855
+ `- Match Reasons: ${entry.scope.reasons.length ? entry.scope.reasons.join(", ") : "None"}`,
856
+ `- Expected References: ${entry.scope.expectedReferences.length ? entry.scope.expectedReferences.join(", ") : "None"}`,
857
+ `- Metadata Mismatches: ${entry.scope.metadataMismatches.length ? entry.scope.metadataMismatches.join("; ") : "None"}`,
858
+ ];
859
+ return lines.join("\n");
860
+ };
861
+ const renderRerunGuidance = (findings) => {
862
+ const relevant = findings.filter(({ category }) => category === "evidence" || category === "smells" || category === "metadata");
863
+ if (!relevant.length) {
864
+ return undefined;
865
+ }
866
+ const lines = [
867
+ "## Rerun Guidance",
868
+ "",
869
+ "- Add meaningful runtime steps for the important state transitions before rerunning.",
870
+ "- Attach focused logs, payloads, screenshots, or DOM snapshots near the failing point.",
871
+ ];
872
+ if (relevant.some(({ category }) => category === "metadata")) {
873
+ lines.push("- Add or repair the labels and parameters needed to identify the intended scope.");
874
+ }
875
+ if (relevant.some(({ checkName }) => checkName === "noop-dominated-steps")) {
876
+ lines.push("- Replace repetitive event-style steps with a compact text attachment when the signal is mostly logs.");
877
+ }
878
+ lines.push("- Rerun only the relevant tests with the same expectations file so the next review is scoped and comparable.");
879
+ return lines.join("\n");
880
+ };
881
+ const renderTestFile = (params) => {
882
+ const { entry, outputDir } = params;
883
+ const { tr, environmentId, attempts, allArtifacts } = entry;
884
+ const lines = [
885
+ "# Test Result",
886
+ "",
887
+ `- Name: ${escapeInlineMarkdown(tr.name)}`,
888
+ `- Full Name: ${escapeInlineMarkdown(tr.fullName ?? tr.name)}`,
889
+ `- Environment: ${escapeInlineMarkdown(environmentId)}`,
890
+ `- History ID: ${escapeInlineMarkdown(tr.historyId ?? "n/a")}`,
891
+ `- Test Result ID: ${escapeInlineMarkdown(tr.id)}`,
892
+ `- Status: ${statusLabel(tr.status)}`,
893
+ `- Duration: ${formatDurationValue(tr.duration)}`,
894
+ `- Started: ${formatTimestamp(tr.start)}`,
895
+ `- Stopped: ${formatTimestamp(tr.stop)}`,
896
+ `- Flaky: ${String(tr.flaky)}`,
897
+ `- Known: ${String(tr.known)}`,
898
+ `- Muted: ${String(tr.muted)}`,
899
+ `- Retries in This Run: ${Math.max(attempts.length - 1, 0)}`,
900
+ ];
901
+ if (tr.titlePath?.length) {
902
+ lines.push(`- Title Path: ${tr.titlePath.map((part) => escapeInlineMarkdown(part)).join(" / ")}`);
903
+ }
904
+ lines.push("");
905
+ lines.push("## Labels");
906
+ lines.push("");
907
+ lines.push(renderLabels(tr.labels));
908
+ lines.push("");
909
+ lines.push("## Parameters");
910
+ lines.push("");
911
+ lines.push(renderParameters(tr.parameters));
912
+ lines.push("");
913
+ lines.push("## Links");
914
+ lines.push("");
915
+ lines.push(renderLinks(tr.links));
916
+ lines.push("");
917
+ lines.push(renderExpectationSection(entry));
918
+ const descriptionSections = [
919
+ renderOptionalMarkdownSection("Description", tr.description),
920
+ renderOptionalMarkdownSection("Precondition", tr.precondition),
921
+ renderOptionalMarkdownSection("Expected Result", tr.expectedResult),
922
+ ].filter(Boolean);
923
+ if (descriptionSections.length) {
924
+ lines.push("");
925
+ lines.push(descriptionSections.join("\n\n"));
926
+ }
927
+ lines.push("");
928
+ lines.push("## Attachments Manifest");
929
+ lines.push("");
930
+ lines.push(allArtifacts.length ? allArtifacts.map((artifact) => renderArtifactLine(artifact, true)).join("\n") : "None");
931
+ lines.push("");
932
+ lines.push(renderFindingsSection({
933
+ title: "Quality Findings",
934
+ findings: entry.findings,
935
+ currentFilePath: entry.filePath,
936
+ outputDir,
937
+ }));
938
+ lines.push("");
939
+ lines.push(...attempts
940
+ .map((attempt) => renderAttemptSection(attempt))
941
+ .join("\n\n")
942
+ .split("\n"));
943
+ const rerunGuidance = renderRerunGuidance(entry.findings);
944
+ if (rerunGuidance) {
945
+ lines.push("");
946
+ lines.push(rerunGuidance);
947
+ }
948
+ return `${lines.join("\n").trimEnd()}\n`;
949
+ };
950
+ const renderIndex = (params) => {
951
+ const { context, command, generatedAt, phase, stats, durationSummary, environmentSummary, modelingSummary, expectations, tests, globalArtifacts, globalErrors, globalExitCode, qualityGateResults, findings, } = params;
952
+ const stdoutArtifact = globalArtifacts.find((artifact) => artifact.displayName === "stdout.txt");
953
+ const stderrArtifact = globalArtifacts.find((artifact) => artifact.displayName === "stderr.txt");
954
+ const remainingGlobalArtifacts = globalArtifacts.filter((artifact) => artifact.displayName !== "stdout.txt" && artifact.displayName !== "stderr.txt");
955
+ const exitCodeSummary = globalExitCode?.actual !== undefined && globalExitCode.actual !== globalExitCode.original
956
+ ? `${globalExitCode.actual} (original: ${globalExitCode.original})`
957
+ : globalExitCode
958
+ ? `${globalExitCode.actual ?? globalExitCode.original}`
959
+ : "unknown";
960
+ const checkSummary = buildCheckSummary(findings);
961
+ const needsAttention = sortFindings(findings)
962
+ .filter(({ severity }) => severity !== "info")
963
+ .slice(0, 10);
964
+ const groupedTests = [
965
+ {
966
+ title: "Failed / Broken",
967
+ entries: tests.filter(({ tr }) => tr.status === "failed" || tr.status === "broken"),
968
+ },
969
+ {
970
+ title: "Unknown / Skipped",
971
+ entries: tests.filter(({ tr }) => tr.status === "unknown" || tr.status === "skipped"),
972
+ },
973
+ {
974
+ title: "Passed",
975
+ entries: tests.filter(({ tr }) => tr.status === "passed"),
976
+ },
977
+ ];
978
+ const lines = [
979
+ `# ${context.reportName}`,
980
+ "",
981
+ "- Format: Allure Agent Markdown",
982
+ `- Generated: ${generatedAt}`,
983
+ `- Report UUID: ${context.reportUuid}`,
984
+ `- Phase: ${phase}`,
985
+ `- Exit Code: ${exitCodeSummary}`,
986
+ `- Command: ${escapeInlineMarkdown(command ?? "unknown")}`,
987
+ "",
988
+ "## Run Summary",
989
+ "",
990
+ `- total: ${stats.total}`,
991
+ `- failed: ${stats.failed ?? 0}`,
992
+ `- broken: ${stats.broken ?? 0}`,
993
+ `- unknown: ${stats.unknown ?? 0}`,
994
+ `- skipped: ${stats.skipped ?? 0}`,
995
+ `- passed: ${stats.passed ?? 0}`,
996
+ `- retries: ${stats.retries ?? 0}`,
997
+ `- flaky: ${stats.flaky ?? 0}`,
998
+ `- total duration: ${formatDurationValue(durationSummary.total)}`,
999
+ `- average duration: ${formatDurationValue(durationSummary.average)}`,
1000
+ `- max duration: ${formatDurationValue(durationSummary.max)}`,
1001
+ ];
1002
+ lines.push("");
1003
+ lines.push("## Environment Summary");
1004
+ lines.push("");
1005
+ lines.push(environmentSummary.length
1006
+ ? environmentSummary
1007
+ .map((environment) => `- ${escapeInlineMarkdown(environment.environmentId)}: ${environment.total} total (${environment.failed} failed, ${environment.broken} broken, ${environment.unknown} unknown, ${environment.skipped} skipped, ${environment.passed} passed)`)
1008
+ .join("\n")
1009
+ : "None");
1010
+ lines.push("");
1011
+ lines.push(renderModelingSummary(modelingSummary));
1012
+ if (expectations) {
1013
+ lines.push("");
1014
+ lines.push("## Expected Scope");
1015
+ lines.push("");
1016
+ lines.push(`- Goal: ${escapeInlineMarkdown(expectations.goal ?? "unknown")}`);
1017
+ lines.push(`- Feature / Task: ${escapeInlineMarkdown(expectations.taskId ?? "unknown")}`);
1018
+ lines.push(`- Expectations Source: [${escapeInlineMarkdown(expectations.relativePath)}](${normalizeMarkdownPath(expectations.relativePath)})`);
1019
+ lines.push(renderSelectorSummary("Expected selectors", expectations.expected));
1020
+ lines.push(renderSelectorSummary("Forbidden selectors", expectations.forbidden));
1021
+ if (expectations.notes.length) {
1022
+ lines.push(`- Notes: ${expectations.notes.map((note) => escapeInlineMarkdown(note)).join(" | ")}`);
1023
+ }
1024
+ }
1025
+ lines.push("");
1026
+ lines.push("## Advisory Check Summary");
1027
+ lines.push("");
1028
+ lines.push(`- modeling completeness: ${modelingSummary.completeness}`);
1029
+ lines.push(`- total findings: ${checkSummary.total}`);
1030
+ lines.push(`- high: ${checkSummary.countsBySeverity.high}`);
1031
+ lines.push(`- warning: ${checkSummary.countsBySeverity.warning}`);
1032
+ lines.push(`- info: ${checkSummary.countsBySeverity.info}`);
1033
+ lines.push(`- bootstrap: ${checkSummary.countsByCategory.bootstrap}`);
1034
+ lines.push(`- scope: ${checkSummary.countsByCategory.scope}`);
1035
+ lines.push(`- metadata: ${checkSummary.countsByCategory.metadata}`);
1036
+ lines.push(`- evidence: ${checkSummary.countsByCategory.evidence}`);
1037
+ lines.push(`- smells: ${checkSummary.countsByCategory.smells}`);
1038
+ lines.push("");
1039
+ lines.push("## Needs Attention First");
1040
+ lines.push("");
1041
+ if (!needsAttention.length) {
1042
+ lines.push("None");
1043
+ }
1044
+ else {
1045
+ lines.push(needsAttention
1046
+ .map((finding) => {
1047
+ const target = finding.subjectType === "test" ? tests.find((entry) => entry.key === finding.subject) : undefined;
1048
+ const targetLink = target ? ` ([test](${normalizeMarkdownPath(target.relativePath)}))` : "";
1049
+ return `- [${finding.severity.toUpperCase()}] ${escapeInlineMarkdown(finding.message)}${targetLink}`;
1050
+ })
1051
+ .join("\n"));
1052
+ }
1053
+ if (stdoutArtifact?.relativePath || stderrArtifact?.relativePath) {
1054
+ lines.push("");
1055
+ lines.push("## Process Logs");
1056
+ lines.push("");
1057
+ if (stdoutArtifact) {
1058
+ lines.push(renderArtifactLine(stdoutArtifact, false));
1059
+ }
1060
+ if (stderrArtifact) {
1061
+ lines.push(renderArtifactLine(stderrArtifact, false));
1062
+ }
1063
+ }
1064
+ const qualityGateSection = renderQualityGateSection(qualityGateResults);
1065
+ if (qualityGateSection) {
1066
+ lines.push("");
1067
+ lines.push(qualityGateSection);
1068
+ }
1069
+ if (remainingGlobalArtifacts.length > 0) {
1070
+ lines.push("");
1071
+ lines.push("## Global Artifacts");
1072
+ lines.push("");
1073
+ lines.push(remainingGlobalArtifacts.map((artifact) => renderArtifactLine(artifact, false)).join("\n"));
1074
+ }
1075
+ const globalErrorsSection = renderGlobalErrors(globalErrors);
1076
+ if (globalErrorsSection) {
1077
+ lines.push("");
1078
+ lines.push(globalErrorsSection);
1079
+ }
1080
+ for (const group of groupedTests) {
1081
+ lines.push("");
1082
+ lines.push(`## ${group.title}`);
1083
+ lines.push("");
1084
+ if (!group.entries.length) {
1085
+ lines.push("None");
1086
+ continue;
1087
+ }
1088
+ lines.push(group.entries
1089
+ .map((entry) => {
1090
+ const counts = toFindingCounts(entry.findings);
1091
+ return `- [${escapeInlineMarkdown(entry.tr.fullName ?? entry.tr.name)}](${normalizeMarkdownPath(entry.relativePath)}) | status: ${statusLabel(entry.tr.status)} | env: ${escapeInlineMarkdown(entry.environmentId)} | duration: ${formatDurationValue(entry.tr.duration)} | retries: ${Math.max(entry.attempts.length - 1, 0)} | scope: ${entry.scope.scopeMatch} | findings: ${counts.total}`;
1092
+ })
1093
+ .join("\n"));
1094
+ }
1095
+ return `${lines.join("\n").trimEnd()}\n`;
1096
+ };
1097
+ const collectAttachmentReferences = (steps, source) => {
1098
+ const result = [];
1099
+ for (const step of steps) {
1100
+ if (isAttachment(step)) {
1101
+ result.push({
1102
+ link: step.link,
1103
+ source,
1104
+ });
1105
+ continue;
1106
+ }
1107
+ if (isStep(step) && step.steps.length) {
1108
+ result.push(...collectAttachmentReferences(step.steps, source));
1109
+ }
1110
+ }
1111
+ return result;
1112
+ };
1113
+ const buildCandidateFileName = (link, fallbackId, preferName, usedNames) => {
1114
+ const preferredName = preferName
1115
+ ? (attachmentName(link) ?? link.originalFileName)
1116
+ : (link.originalFileName ?? attachmentName(link));
1117
+ const rawName = basename(preferredName ?? `${fallbackId}${link.ext ?? ""}`);
1118
+ const safeName = sanitizePathSegment(rawName, `${fallbackId}${link.ext ?? ""}`);
1119
+ if (!usedNames.has(safeName)) {
1120
+ usedNames.add(safeName);
1121
+ return safeName;
1122
+ }
1123
+ const extension = extname(safeName) || link.ext || "";
1124
+ const baseName = extension ? safeName.slice(0, -extension.length) : safeName;
1125
+ let counter = 0;
1126
+ let candidate = sanitizePathSegment(`${baseName}--${fallbackId}${extension}`, `${fallbackId}${extension}`);
1127
+ while (usedNames.has(candidate)) {
1128
+ counter += 1;
1129
+ candidate = sanitizePathSegment(`${baseName}--${fallbackId}-${counter}${extension}`, `${fallbackId}-${counter}${extension}`);
1130
+ }
1131
+ usedNames.add(candidate);
1132
+ return candidate;
1133
+ };
1134
+ const materializeArtifacts = async (params) => {
1135
+ const { references, artifactDir, filePath, preferName, resolveContent, copiedById, usedNames } = params;
1136
+ const refsById = new Map();
1137
+ for (const reference of references) {
1138
+ const current = refsById.get(reference.link.id);
1139
+ if (current) {
1140
+ current.source = `${current.source}; ${reference.source}`;
1141
+ continue;
1142
+ }
1143
+ refsById.set(reference.link.id, { ...reference });
1144
+ }
1145
+ const artifacts = [];
1146
+ for (const { link, source } of refsById.values()) {
1147
+ const sources = uniqueValues(source.split("; ").filter(Boolean));
1148
+ const existing = copiedById.get(link.id);
1149
+ if (existing) {
1150
+ existing.sources = uniqueValues(existing.sources.concat(sources));
1151
+ artifacts.push(existing);
1152
+ continue;
1153
+ }
1154
+ const artifact = {
1155
+ id: link.id,
1156
+ displayName: attachmentDisplayName(link),
1157
+ sources,
1158
+ contentType: link.contentType,
1159
+ missing: true,
1160
+ };
1161
+ if (!link.missed) {
1162
+ const content = await resolveContent(link.id);
1163
+ if (content) {
1164
+ const fileName = buildCandidateFileName(link, sanitizePathSegment(link.id, "artifact"), preferName, usedNames);
1165
+ const targetPath = join(artifactDir, fileName);
1166
+ await mkdir(dirname(targetPath), { recursive: true });
1167
+ await content.writeTo(targetPath);
1168
+ artifact.relativePath = normalizeMarkdownPath(relative(dirname(filePath), targetPath));
1169
+ artifact.contentType = content.getContentType() ?? link.contentType;
1170
+ artifact.contentLength = content.getContentLength();
1171
+ artifact.missing = false;
1172
+ }
1173
+ }
1174
+ copiedById.set(link.id, artifact);
1175
+ artifacts.push(artifact);
1176
+ }
1177
+ return artifacts;
1178
+ };
1179
+ const buildAttemptArtifacts = async (params) => {
1180
+ const { heading, tr, fixtures, store, artifactDir, filePath, copiedById, usedNames } = params;
1181
+ const references = (await store.attachmentsByTrId(tr.id)).map((link) => ({
1182
+ link,
1183
+ source: `${heading}: test result`,
1184
+ }));
1185
+ references.push(...collectAttachmentReferences(tr.steps, `${heading}: test steps`));
1186
+ for (const fixture of fixtures) {
1187
+ references.push(...collectAttachmentReferences(fixture.steps, `${heading}: ${fixture.type === "before" ? "before" : "after"} fixture ${fixture.name}`));
1188
+ }
1189
+ return materializeArtifacts({
1190
+ references,
1191
+ artifactDir,
1192
+ filePath,
1193
+ preferName: false,
1194
+ resolveContent: (id) => store.attachmentContentById(id),
1195
+ copiedById,
1196
+ usedNames,
1197
+ });
1198
+ };
1199
+ const buildGlobalArtifacts = async (outputDir, store) => {
1200
+ const globalArtifactsDir = join(outputDir, "artifacts", "global");
1201
+ const usedNames = new Set();
1202
+ const copiedById = new Map();
1203
+ const references = (await store.allGlobalAttachments()).map((link) => ({
1204
+ link,
1205
+ source: "global attachment",
1206
+ }));
1207
+ return materializeArtifacts({
1208
+ references,
1209
+ artifactDir: globalArtifactsDir,
1210
+ filePath: join(outputDir, "index.md"),
1211
+ preferName: true,
1212
+ resolveContent: (id) => store.attachmentContentById(id),
1213
+ copiedById,
1214
+ usedNames,
1215
+ });
1216
+ };
1217
+ const readMaterializedArtifactText = async (outputDir, artifact) => {
1218
+ if (!artifact?.relativePath) {
1219
+ return undefined;
1220
+ }
1221
+ try {
1222
+ return await readFile(join(outputDir, artifact.relativePath), "utf-8");
1223
+ }
1224
+ catch {
1225
+ return undefined;
1226
+ }
1227
+ };
1228
+ const resolveOutputDir = (options) => {
1229
+ const outputDir = options.outputDir ?? env[AGENT_OUTPUT_ENV];
1230
+ return outputDir ? resolve(outputDir) : undefined;
1231
+ };
1232
+ const cleanupManagedEntries = async (outputDir) => {
1233
+ await Promise.all(MANAGED_ENTRIES.map(async (entry) => {
1234
+ await rm(join(outputDir, entry), {
1235
+ recursive: true,
1236
+ force: true,
1237
+ });
1238
+ }));
1239
+ };
1240
+ const createUniqueSlug = (slug, trId, usedSlugs) => {
1241
+ if (!usedSlugs.has(slug)) {
1242
+ usedSlugs.add(slug);
1243
+ return {
1244
+ slug,
1245
+ collision: false,
1246
+ };
1247
+ }
1248
+ let candidate = `${slug}--${sanitizePathSegment(trId, "tr")}`;
1249
+ let counter = 0;
1250
+ while (usedSlugs.has(candidate)) {
1251
+ counter += 1;
1252
+ candidate = `${slug}--${sanitizePathSegment(trId, "tr")}-${counter}`;
1253
+ }
1254
+ usedSlugs.add(candidate);
1255
+ return {
1256
+ slug: candidate,
1257
+ collision: true,
1258
+ };
1259
+ };
1260
+ const createFindingFactory = () => {
1261
+ let sequence = 0;
1262
+ return (finding) => {
1263
+ sequence += 1;
1264
+ return {
1265
+ findingId: `F${sequence.toString().padStart(4, "0")}`,
1266
+ ...finding,
1267
+ };
1268
+ };
1269
+ };
1270
+ const parseExpectations = (rawContent) => {
1271
+ const parsed = parse(rawContent);
1272
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
1273
+ throw new Error("Expected a YAML or JSON object");
1274
+ }
1275
+ return parsed;
1276
+ };
1277
+ const loadExpectations = async (outputDir, createFinding) => {
1278
+ const configuredPath = env[AGENT_EXPECTATIONS_ENV];
1279
+ if (!configuredPath) {
1280
+ return {
1281
+ expectations: undefined,
1282
+ findings: [],
1283
+ };
1284
+ }
1285
+ const expectationsPath = resolve(configuredPath);
1286
+ try {
1287
+ const rawContent = await readFile(expectationsPath, "utf-8");
1288
+ const parsed = parseExpectations(rawContent);
1289
+ const relativePath = normalizeMarkdownPath("manifest/expected.json");
1290
+ await mkdir(join(outputDir, "manifest"), { recursive: true });
1291
+ await writeFile(join(outputDir, relativePath), `${JSON.stringify(parsed, null, 2)}\n`, "utf-8");
1292
+ return {
1293
+ expectations: {
1294
+ sourcePath: expectationsPath,
1295
+ relativePath,
1296
+ raw: parsed,
1297
+ goal: parsed.goal,
1298
+ taskId: parsed.task_id,
1299
+ notes: normalizeNotes(parsed.notes),
1300
+ expected: normalizeSelectors(parsed.expected),
1301
+ forbidden: normalizeSelectors(parsed.forbidden),
1302
+ },
1303
+ findings: [],
1304
+ };
1305
+ }
1306
+ catch (error) {
1307
+ return {
1308
+ expectations: undefined,
1309
+ findings: [
1310
+ createFinding({
1311
+ subject: "run",
1312
+ subjectType: "run",
1313
+ severity: "high",
1314
+ category: "bootstrap",
1315
+ checkName: "invalid-expectations-file",
1316
+ message: `Could not load ALLURE_AGENT_EXPECTATIONS from ${expectationsPath}`,
1317
+ explanation: `The expectations file could not be parsed as YAML or JSON: ${error.message}`,
1318
+ evidencePaths: [],
1319
+ remediationHint: "Provide a readable YAML or JSON file in ALLURE_AGENT_EXPECTATIONS before rerunning.",
1320
+ expectedReference: undefined,
1321
+ }),
1322
+ ],
1323
+ };
1324
+ }
1325
+ };
1326
+ const loadProjectGuide = async (outputDir) => {
1327
+ const projectRoot = resolve(env[AGENT_PROJECT_ROOT_ENV] ?? process.cwd());
1328
+ const sourcePath = join(projectRoot, "docs", "allure-agent-mode.md");
1329
+ try {
1330
+ const content = await readFile(sourcePath, "utf-8");
1331
+ const relativePath = normalizeMarkdownPath(join("project", "docs", "allure-agent-mode.md"));
1332
+ await mkdir(join(outputDir, "project", "docs"), { recursive: true });
1333
+ await writeFile(join(outputDir, relativePath), content, "utf-8");
1334
+ return {
1335
+ sourcePath,
1336
+ relativePath,
1337
+ };
1338
+ }
1339
+ catch (error) {
1340
+ if (error.code === "ENOENT") {
1341
+ return undefined;
1342
+ }
1343
+ throw error;
1344
+ }
1345
+ };
1346
+ const computeScopeEvaluation = (params) => {
1347
+ const { tr, environmentId, expectations } = params;
1348
+ if (!expectations) {
1349
+ return {
1350
+ scopeMatch: "unknown",
1351
+ reasons: [],
1352
+ expectedReferences: [],
1353
+ metadataMismatches: [],
1354
+ };
1355
+ }
1356
+ const positivePresent = hasSelector(expectations.expected);
1357
+ const positive = matchSelectors({
1358
+ tr,
1359
+ environmentId,
1360
+ selectors: expectations.expected,
1361
+ selectorRoot: "expected",
1362
+ });
1363
+ const forbidden = matchSelectors({
1364
+ tr,
1365
+ environmentId,
1366
+ selectors: expectations.forbidden,
1367
+ selectorRoot: "forbidden",
1368
+ });
1369
+ const metadataMismatches = positive.matchedByNonLabel && Object.keys(expectations.expected.labelValues).length > 0 && !positive.labelMatch
1370
+ ? collectMissingLabelSelectors(tr.labels, expectations.expected.labelValues)
1371
+ : [];
1372
+ if (forbidden.matched) {
1373
+ return {
1374
+ scopeMatch: "forbidden",
1375
+ reasons: forbidden.reasons,
1376
+ expectedReferences: forbidden.references,
1377
+ metadataMismatches,
1378
+ };
1379
+ }
1380
+ if (!positivePresent) {
1381
+ return {
1382
+ scopeMatch: "unknown",
1383
+ reasons: [],
1384
+ expectedReferences: [],
1385
+ metadataMismatches,
1386
+ };
1387
+ }
1388
+ if (positive.matched) {
1389
+ return {
1390
+ scopeMatch: "match",
1391
+ reasons: positive.reasons,
1392
+ expectedReferences: positive.references,
1393
+ metadataMismatches,
1394
+ };
1395
+ }
1396
+ return {
1397
+ scopeMatch: "unexpected",
1398
+ reasons: [],
1399
+ expectedReferences: [],
1400
+ metadataMismatches,
1401
+ };
1402
+ };
1403
+ const buildDurationSummary = (entries) => {
1404
+ const durations = entries.map(({ tr }) => tr.duration ?? 0);
1405
+ const total = durations.reduce((acc, value) => acc + value, 0);
1406
+ return {
1407
+ total,
1408
+ average: durations.length ? Math.round(total / durations.length) : 0,
1409
+ max: durations.length ? Math.max(...durations) : 0,
1410
+ };
1411
+ };
1412
+ const collectTestEvidencePaths = (entry) => {
1413
+ const paths = [entry.relativePath];
1414
+ for (const artifact of entry.allArtifacts) {
1415
+ if (artifact.relativePath) {
1416
+ paths.push(artifact.relativePath);
1417
+ }
1418
+ }
1419
+ return uniqueValues(paths);
1420
+ };
1421
+ const buildRunAndTestFindings = (params) => {
1422
+ const { entries, expectations, globalArtifacts, modelingSummary, createFinding } = params;
1423
+ const runFindings = [];
1424
+ const stdoutArtifact = globalArtifacts.find((artifact) => artifact.displayName === "stdout.txt");
1425
+ const stderrArtifact = globalArtifacts.find((artifact) => artifact.displayName === "stderr.txt");
1426
+ if (entries.length === 0) {
1427
+ runFindings.push(createFinding({
1428
+ subject: "run",
1429
+ subjectType: "run",
1430
+ severity: "high",
1431
+ category: "bootstrap",
1432
+ checkName: "no-visible-tests",
1433
+ message: "No visible test results were found in the run.",
1434
+ explanation: "The agent output was generated, but there were no visible logical test results to review.",
1435
+ evidencePaths: [],
1436
+ remediationHint: "Verify that Allure results are being generated and that the test command actually executed the intended tests.",
1437
+ }));
1438
+ }
1439
+ if (!stdoutArtifact && !stderrArtifact) {
1440
+ runFindings.push(createFinding({
1441
+ subject: "run",
1442
+ subjectType: "run",
1443
+ severity: "info",
1444
+ category: "bootstrap",
1445
+ checkName: "missing-global-logs",
1446
+ message: "The run does not include global stdout or stderr logs.",
1447
+ explanation: "Global process logs help agents debug bootstrap failures and compare the recorded results with console output.",
1448
+ evidencePaths: [],
1449
+ remediationHint: "Run tests through `allure run -- <command>` without disabling log capture when you need bootstrap diagnostics.",
1450
+ confidence: 0.9,
1451
+ }));
1452
+ }
1453
+ if (modelingSummary.runnerFailures.total > 0) {
1454
+ runFindings.push(createFinding({
1455
+ subject: "run",
1456
+ subjectType: "run",
1457
+ severity: "high",
1458
+ category: "bootstrap",
1459
+ checkName: "runner-failures-outside-logical-results",
1460
+ message: "Runner-level failures were detected outside logical test results.",
1461
+ explanation: "Global errors or high-signal stderr messages suggest suite-load, import, or setup failures that are not represented in `manifest/tests.jsonl`.",
1462
+ evidencePaths: stderrArtifact?.relativePath ? [stderrArtifact.relativePath] : [],
1463
+ remediationHint: "Inspect global stderr and global errors before accepting the run, then rerun once the missing failures are understood.",
1464
+ confidence: 0.92,
1465
+ }));
1466
+ }
1467
+ if (modelingSummary.unmodeledFromStats.total > 0) {
1468
+ const severity = modelingSummary.unmodeledFromStats.failed > 0 ||
1469
+ modelingSummary.unmodeledFromStats.broken > 0 ||
1470
+ modelingSummary.unmodeledFromStats.unknown > 0
1471
+ ? "warning"
1472
+ : "info";
1473
+ runFindings.push(createFinding({
1474
+ subject: "run",
1475
+ subjectType: "run",
1476
+ severity,
1477
+ category: "bootstrap",
1478
+ checkName: "unmodeled-visible-results",
1479
+ message: "The run summary includes visible results that were not rendered as logical test files.",
1480
+ explanation: `The store statistics reported ${modelingSummary.compact.visible_results} visible results, but agent mode rendered only ${modelingSummary.compact.logical_tests} logical test files. Missing counts: ${summarizeStatusCounts(modelingSummary.unmodeledFromStats)}.`,
1481
+ evidencePaths: stderrArtifact?.relativePath ? [stderrArtifact.relativePath] : [],
1482
+ remediationHint: "Treat the run as partially modeled, inspect the global logs, and call out any missing skipped or failing results before final review.",
1483
+ confidence: 0.8,
1484
+ }));
1485
+ }
1486
+ const actualEnvironments = uniqueValues(entries.map(({ environmentId }) => environmentId));
1487
+ if (expectations) {
1488
+ const allFullNames = entries.map(({ tr }) => tr.fullName ?? tr.name);
1489
+ expectations.expected.fullNames.forEach((fullName, index) => {
1490
+ if (!allFullNames.includes(fullName)) {
1491
+ runFindings.push(createFinding({
1492
+ subject: "run",
1493
+ subjectType: "run",
1494
+ severity: "high",
1495
+ category: "scope",
1496
+ checkName: "missing-expected-test",
1497
+ message: `Expected test did not run: ${fullName}`,
1498
+ explanation: "The expectations file explicitly listed this test, but it did not appear in the agentic output.",
1499
+ evidencePaths: expectations.relativePath ? [expectations.relativePath] : [],
1500
+ remediationHint: "Check the test selection, environment, and feature branch scope before rerunning.",
1501
+ expectedReference: `expected.full_names[${index}]`,
1502
+ }));
1503
+ }
1504
+ });
1505
+ expectations.expected.fullNamePrefixes.forEach((prefix, index) => {
1506
+ if (!allFullNames.some((fullName) => fullName.startsWith(prefix))) {
1507
+ runFindings.push(createFinding({
1508
+ subject: "run",
1509
+ subjectType: "run",
1510
+ severity: "warning",
1511
+ category: "scope",
1512
+ checkName: "missing-expected-prefix",
1513
+ message: `No executed test matched the expected prefix: ${prefix}`,
1514
+ explanation: "The expectations file asked for tests within this name prefix, but none were recorded.",
1515
+ evidencePaths: expectations.relativePath ? [expectations.relativePath] : [],
1516
+ remediationHint: "Check the expected selector or adjust the executed test target so the intended scope is covered.",
1517
+ expectedReference: `expected.full_name_prefixes[${index}]`,
1518
+ }));
1519
+ }
1520
+ });
1521
+ expectations.expected.environments.forEach((environment, index) => {
1522
+ if (!actualEnvironments.includes(environment)) {
1523
+ runFindings.push(createFinding({
1524
+ subject: "run",
1525
+ subjectType: "run",
1526
+ severity: "warning",
1527
+ category: "scope",
1528
+ checkName: "missing-expected-environment",
1529
+ message: `Expected environment did not appear in the run: ${environment}`,
1530
+ explanation: "The expectations file scoped the run to this environment, but no logical test result matched it.",
1531
+ evidencePaths: expectations.relativePath ? [expectations.relativePath] : [],
1532
+ remediationHint: "Check the environment selector or rerun the intended environment explicitly.",
1533
+ expectedReference: `expected.environments[${index}]`,
1534
+ }));
1535
+ }
1536
+ });
1537
+ Object.entries(expectations.expected.labelValues).forEach(([labelName, values]) => {
1538
+ const matched = entries.some(({ tr }) => matchesLabelSelectors(tr.labels, { [labelName]: values }));
1539
+ if (!matched) {
1540
+ runFindings.push(createFinding({
1541
+ subject: "run",
1542
+ subjectType: "run",
1543
+ severity: "warning",
1544
+ category: "scope",
1545
+ checkName: "missing-expected-label-selector",
1546
+ message: `No executed test matched ${formatLabelRequirement(labelName, values)}`,
1547
+ explanation: "The expectations file defined a label selector for the intended scope, but no logical test result satisfied it.",
1548
+ evidencePaths: expectations.relativePath ? [expectations.relativePath] : [],
1549
+ remediationHint: "Add the expected label metadata to the intended tests or adjust the expectations selector.",
1550
+ expectedReference: `expected.label_values/${escapeJsonPointerSegment(labelName)}`,
1551
+ }));
1552
+ }
1553
+ });
1554
+ if (expectations.expected.environments.length > 0) {
1555
+ actualEnvironments
1556
+ .filter((environment) => !expectations.expected.environments.includes(environment))
1557
+ .forEach((environment) => {
1558
+ runFindings.push(createFinding({
1559
+ subject: "run",
1560
+ subjectType: "run",
1561
+ severity: "warning",
1562
+ category: "scope",
1563
+ checkName: "unexpected-environment",
1564
+ message: `Unexpected environment ran: ${environment}`,
1565
+ explanation: "The run included an environment outside the expected scope.",
1566
+ evidencePaths: [],
1567
+ remediationHint: "Narrow the environment selector or remove unrelated results before rerunning.",
1568
+ confidence: 0.95,
1569
+ }));
1570
+ });
1571
+ }
1572
+ }
1573
+ for (const entry of entries) {
1574
+ const currentAttempt = entry.attempts[0];
1575
+ const attemptSignatures = uniqueValues(entry.attempts.map(buildAttemptSignature));
1576
+ const testEvidencePaths = collectTestEvidencePaths(entry);
1577
+ const allStepSummary = mergeStepSummaries(entry.attempts.map((attempt) => mergeStepSummaries([attempt.stepSummary, attempt.fixtureStepSummary])));
1578
+ const hasUsefulSteps = currentAttempt.stepSummary.meaningfulSteps + currentAttempt.fixtureStepSummary.meaningfulSteps > 0;
1579
+ const hasAnyAttachments = entry.allArtifacts.some((artifact) => !artifact.missing);
1580
+ const noopRatio = allStepSummary.totalSteps > 0 ? allStepSummary.noopSteps / allStepSummary.totalSteps : 0;
1581
+ if (entry.scope.scopeMatch === "forbidden") {
1582
+ entry.findings.push(createFinding({
1583
+ subject: entry.key,
1584
+ subjectType: "test",
1585
+ severity: "high",
1586
+ category: "scope",
1587
+ checkName: "forbidden-selector-match",
1588
+ message: "This test matched a forbidden selector from the expectations file.",
1589
+ explanation: "The logical test belongs to a scope that the expectations file explicitly marked as forbidden.",
1590
+ evidencePaths: expectations?.relativePath
1591
+ ? [entry.relativePath, expectations.relativePath]
1592
+ : [entry.relativePath],
1593
+ remediationHint: "Tighten the test selection or update the expectations file before accepting the run.",
1594
+ expectedReference: entry.scope.expectedReferences[0],
1595
+ }));
1596
+ }
1597
+ else if (entry.scope.scopeMatch === "unexpected") {
1598
+ entry.findings.push(createFinding({
1599
+ subject: entry.key,
1600
+ subjectType: "test",
1601
+ severity: "warning",
1602
+ category: "scope",
1603
+ checkName: "unexpected-test",
1604
+ message: "This test ran outside the expected scope.",
1605
+ explanation: "The expectations file defined positive scope selectors, but this logical test did not match any of them.",
1606
+ evidencePaths: expectations?.relativePath
1607
+ ? [entry.relativePath, expectations.relativePath]
1608
+ : [entry.relativePath],
1609
+ remediationHint: "Rerun only the intended tests or broaden the expectations file if this test is part of the plan.",
1610
+ }));
1611
+ }
1612
+ if (entry.scope.metadataMismatches.length > 0) {
1613
+ entry.findings.push(createFinding({
1614
+ subject: entry.key,
1615
+ subjectType: "test",
1616
+ severity: "warning",
1617
+ category: "metadata",
1618
+ checkName: "metadata-mismatch",
1619
+ message: "Test metadata does not fully match the expected label selectors.",
1620
+ explanation: entry.scope.metadataMismatches.join("; "),
1621
+ evidencePaths: expectations?.relativePath
1622
+ ? [entry.relativePath, expectations.relativePath]
1623
+ : [entry.relativePath],
1624
+ remediationHint: "Add or repair the labels that identify the intended feature, task, or environment.",
1625
+ expectedReference: entry.scope.expectedReferences.find((reference) => reference.startsWith("expected.label_values")),
1626
+ confidence: 0.9,
1627
+ }));
1628
+ }
1629
+ if (entry.historyCollision) {
1630
+ entry.findings.push(createFinding({
1631
+ subject: entry.key,
1632
+ subjectType: "test",
1633
+ severity: "warning",
1634
+ category: "metadata",
1635
+ checkName: "history-id-collision",
1636
+ message: "Multiple visible tests shared the same history ID in this environment.",
1637
+ explanation: "The output had to suffix the markdown file name because the logical test key was not unique within the environment.",
1638
+ evidencePaths: [entry.relativePath],
1639
+ remediationHint: "Ensure the test metadata produces unique history IDs for distinct logical tests.",
1640
+ confidence: 0.85,
1641
+ }));
1642
+ }
1643
+ if (isFailedLikeStatus(currentAttempt.tr.status) && !hasUsefulSteps) {
1644
+ entry.findings.push(createFinding({
1645
+ subject: entry.key,
1646
+ subjectType: "test",
1647
+ severity: "warning",
1648
+ category: "evidence",
1649
+ checkName: "failed-without-useful-steps",
1650
+ message: "A failed or broken test has no useful runtime steps.",
1651
+ explanation: "The failure is recorded, but the step tree does not explain the state transitions that led to it.",
1652
+ evidencePaths: testEvidencePaths,
1653
+ remediationHint: "Add meaningful steps around the important actions and checks before rerunning.",
1654
+ confidence: 0.92,
1655
+ }));
1656
+ }
1657
+ if (isFailedLikeStatus(currentAttempt.tr.status) && !hasAnyAttachments) {
1658
+ entry.findings.push(createFinding({
1659
+ subject: entry.key,
1660
+ subjectType: "test",
1661
+ severity: "warning",
1662
+ category: "evidence",
1663
+ checkName: "failed-without-attachments",
1664
+ message: "A failed or broken test has no test-scoped attachments.",
1665
+ explanation: "Attachments often provide the fastest route to understanding why the test failed.",
1666
+ evidencePaths: testEvidencePaths,
1667
+ remediationHint: "Attach targeted logs, payloads, screenshots, or DOM snapshots around the failing point.",
1668
+ confidence: 0.88,
1669
+ }));
1670
+ }
1671
+ if ((currentAttempt.tr.duration ?? 0) >= NONTRIVIAL_DURATION_MS &&
1672
+ currentAttempt.stepSummary.totalSteps === 0 &&
1673
+ currentAttempt.fixtureStepSummary.totalSteps === 0) {
1674
+ entry.findings.push(createFinding({
1675
+ subject: entry.key,
1676
+ subjectType: "test",
1677
+ severity: "warning",
1678
+ category: "evidence",
1679
+ checkName: "nontrivial-run-with-empty-trace",
1680
+ message: "A nontrivial test run recorded no steps or fixture activity.",
1681
+ explanation: "The duration suggests real work happened, but the trace contains no step-level evidence.",
1682
+ evidencePaths: testEvidencePaths,
1683
+ remediationHint: "Add runtime steps or attachments so the execution path is observable on the next run.",
1684
+ confidence: 0.8,
1685
+ }));
1686
+ }
1687
+ if (entry.attempts.length > 1 && attemptSignatures.length === 1) {
1688
+ entry.findings.push(createFinding({
1689
+ subject: entry.key,
1690
+ subjectType: "test",
1691
+ severity: "info",
1692
+ category: "evidence",
1693
+ checkName: "retries-without-new-evidence",
1694
+ message: "Retries did not add any new observable evidence.",
1695
+ explanation: "The recorded status, error, steps, and attachments stayed effectively unchanged across retries.",
1696
+ evidencePaths: testEvidencePaths,
1697
+ remediationHint: "Add retry-specific diagnostics or targeted attachments so reruns can show what changed.",
1698
+ confidence: 0.7,
1699
+ }));
1700
+ }
1701
+ if (allStepSummary.totalSteps >= 3 && noopRatio >= NOOP_RATIO_THRESHOLD && !hasAnyAttachments) {
1702
+ entry.findings.push(createFinding({
1703
+ subject: entry.key,
1704
+ subjectType: "test",
1705
+ severity: "warning",
1706
+ category: "smells",
1707
+ checkName: "noop-dominated-steps",
1708
+ message: "The step tree is dominated by low-signal steps.",
1709
+ explanation: "Most recorded steps are leaf steps without parameters, nested actions, attachments, or error context.",
1710
+ evidencePaths: testEvidencePaths,
1711
+ remediationHint: "Collapse repetitive event-style steps into a smaller set of meaningful steps or attach a text log instead.",
1712
+ confidence: 0.75,
1713
+ }));
1714
+ }
1715
+ if (allStepSummary.totalSteps >= STEP_SPAM_THRESHOLD && !hasAnyAttachments) {
1716
+ entry.findings.push(createFinding({
1717
+ subject: entry.key,
1718
+ subjectType: "test",
1719
+ severity: "info",
1720
+ category: "smells",
1721
+ checkName: "step-spam",
1722
+ message: "The trace records many steps but no compact artifact.",
1723
+ explanation: "A large number of small steps can be harder to review than a focused log attachment.",
1724
+ evidencePaths: testEvidencePaths,
1725
+ remediationHint: "Consider attaching a structured text log when the trace is mostly event reporting.",
1726
+ confidence: 0.7,
1727
+ }));
1728
+ }
1729
+ if (isFailedLikeStatus(currentAttempt.tr.status) && !hasAnyAttachments && globalArtifacts.length > 0) {
1730
+ entry.findings.push(createFinding({
1731
+ subject: entry.key,
1732
+ subjectType: "test",
1733
+ severity: "info",
1734
+ category: "smells",
1735
+ checkName: "global-only-artifacts",
1736
+ message: "Only run-level artifacts are available for this failed test.",
1737
+ explanation: "The run captured global logs, but the failed test has no test-scoped attachments to pinpoint the failure.",
1738
+ evidencePaths: uniqueValues(testEvidencePaths.concat(globalArtifacts.flatMap((artifact) => (artifact.relativePath ? [artifact.relativePath] : [])))),
1739
+ remediationHint: "Prefer step-scoped or test-scoped attachments near the failing action when debugging targeted failures.",
1740
+ confidence: 0.78,
1741
+ }));
1742
+ }
1743
+ if (currentAttempt.tr.status === "passed" &&
1744
+ (currentAttempt.tr.duration ?? 0) >= NONTRIVIAL_DURATION_MS &&
1745
+ currentAttempt.stepSummary.totalSteps === 0 &&
1746
+ currentAttempt.fixtureStepSummary.totalSteps === 0 &&
1747
+ !hasAnyAttachments) {
1748
+ entry.findings.push(createFinding({
1749
+ subject: entry.key,
1750
+ subjectType: "test",
1751
+ severity: "info",
1752
+ category: "smells",
1753
+ checkName: "passed-without-observable-evidence",
1754
+ message: "A nontrivial passing test recorded no observable evidence.",
1755
+ explanation: "The test passed, but the agentic output contains no steps, fixture activity, or attachments showing what was verified.",
1756
+ evidencePaths: testEvidencePaths,
1757
+ remediationHint: "Add a few meaningful verification steps or attachments so the success path is reviewable.",
1758
+ confidence: 0.65,
1759
+ }));
1760
+ }
1761
+ }
1762
+ return {
1763
+ runFindings,
1764
+ allFindings: sortFindings(runFindings.concat(entries.flatMap((entry) => entry.findings))),
1765
+ };
1766
+ };
1767
+ const listVisibleTestLayouts = async (params) => {
1768
+ const { outputDir, store } = params;
1769
+ const tests = (await store.allTestResults({ includeHidden: false })).sort(compareTestResultsByStatusThenName);
1770
+ const layouts = [];
1771
+ const slugsByEnvironment = new Map();
1772
+ for (const tr of tests) {
1773
+ const rawEnvironmentId = (await store.environmentIdByTrId(tr.id)) ?? "default";
1774
+ const environmentId = rawEnvironmentId;
1775
+ const environmentPath = sanitizePathSegment(rawEnvironmentId, "default");
1776
+ const slugSeed = sanitizePathSegment(tr.historyId ?? tr.id, sanitizePathSegment(tr.id, "test"));
1777
+ const usedSlugs = slugsByEnvironment.get(environmentPath) ?? new Set();
1778
+ slugsByEnvironment.set(environmentPath, usedSlugs);
1779
+ const slugResult = createUniqueSlug(slugSeed, tr.id, usedSlugs);
1780
+ const slug = slugResult.slug;
1781
+ const filePath = join(outputDir, "tests", environmentPath, `${slug}.md`);
1782
+ const relativePath = normalizeMarkdownPath(relative(outputDir, filePath));
1783
+ const assetDir = join(outputDir, "tests", environmentPath, `${slug}.assets`);
1784
+ const relativeAssetDir = normalizeMarkdownPath(relative(outputDir, assetDir));
1785
+ layouts.push({
1786
+ tr,
1787
+ environmentId,
1788
+ environmentPath,
1789
+ slug,
1790
+ relativePath,
1791
+ filePath,
1792
+ assetDir,
1793
+ relativeAssetDir,
1794
+ historyCollision: slugResult.collision,
1795
+ });
1796
+ }
1797
+ return layouts;
1798
+ };
1799
+ const buildEntryFromLayout = async (params) => {
1800
+ const { layout, store, expectations } = params;
1801
+ const { tr, environmentId, environmentPath, slug, relativePath, filePath, assetDir, relativeAssetDir, historyCollision, } = layout;
1802
+ const retries = sortByNewestAttempt(await store.retriesByTr(tr));
1803
+ const attemptsToRender = [tr, ...retries];
1804
+ const attempts = [];
1805
+ const copiedById = new Map();
1806
+ const usedNames = new Set();
1807
+ for (const [index, attemptTr] of attemptsToRender.entries()) {
1808
+ const heading = index === 0 ? "Current Attempt" : `Retry ${index}`;
1809
+ const fixtures = await store.fixturesByTrId(attemptTr.id);
1810
+ const artifacts = await buildAttemptArtifacts({
1811
+ heading,
1812
+ tr: attemptTr,
1813
+ fixtures,
1814
+ store,
1815
+ artifactDir: assetDir,
1816
+ filePath,
1817
+ copiedById,
1818
+ usedNames,
1819
+ });
1820
+ attempts.push({
1821
+ heading,
1822
+ tr: attemptTr,
1823
+ fixtures,
1824
+ artifacts,
1825
+ stepSummary: analyzeStepTree(attemptTr.steps),
1826
+ fixtureStepSummary: mergeStepSummaries(fixtures.map((fixture) => analyzeStepTree(fixture.steps))),
1827
+ });
1828
+ }
1829
+ const allArtifacts = Array.from(copiedById.values()).sort((left, right) => left.displayName.localeCompare(right.displayName));
1830
+ return {
1831
+ key: relativePath,
1832
+ tr,
1833
+ environmentId,
1834
+ environmentPath,
1835
+ slug,
1836
+ relativePath,
1837
+ filePath,
1838
+ relativeAssetDir,
1839
+ attempts,
1840
+ allArtifacts,
1841
+ findings: [],
1842
+ scope: computeScopeEvaluation({
1843
+ tr,
1844
+ environmentId,
1845
+ expectations,
1846
+ }),
1847
+ packageName: getPackageName(tr),
1848
+ historyCollision,
1849
+ };
1850
+ };
1851
+ const buildEntries = async (params) => {
1852
+ const { outputDir, store, expectations } = params;
1853
+ const layouts = await listVisibleTestLayouts({
1854
+ outputDir,
1855
+ store,
1856
+ });
1857
+ const entries = [];
1858
+ for (const layout of layouts) {
1859
+ entries.push(await buildEntryFromLayout({
1860
+ layout,
1861
+ store,
1862
+ expectations,
1863
+ }));
1864
+ }
1865
+ return entries;
1866
+ };
1867
+ const buildSnapshot = async (params) => {
1868
+ const { outputDir, store, expectations, expectationLoadFindings, createFinding } = params;
1869
+ const stats = await store.testsStatistic((testResult) => !testResult.hidden);
1870
+ const entries = await buildEntries({
1871
+ outputDir,
1872
+ store,
1873
+ expectations,
1874
+ });
1875
+ const globalArtifacts = await buildGlobalArtifacts(outputDir, store);
1876
+ const globalErrors = await store.allGlobalErrors();
1877
+ const globalExitCode = await store.globalExitCode();
1878
+ const qualityGateResults = await store.qualityGateResults();
1879
+ const stderrArtifact = globalArtifacts.find((artifact) => artifact.displayName === "stderr.txt");
1880
+ const stderrContent = await readMaterializedArtifactText(outputDir, stderrArtifact);
1881
+ const modelingSummary = buildModelingSummary({
1882
+ entries,
1883
+ stats,
1884
+ globalErrors,
1885
+ stderrContent,
1886
+ });
1887
+ const { runFindings } = buildRunAndTestFindings({
1888
+ entries,
1889
+ expectations,
1890
+ globalArtifacts,
1891
+ modelingSummary,
1892
+ createFinding,
1893
+ });
1894
+ const combinedRunFindings = sortFindings(expectationLoadFindings.concat(runFindings));
1895
+ const combinedAllFindings = sortFindings(combinedRunFindings.concat(entries.flatMap((entry) => entry.findings)));
1896
+ return {
1897
+ stats,
1898
+ entries,
1899
+ globalArtifacts,
1900
+ globalErrors,
1901
+ globalExitCode,
1902
+ qualityGateResults,
1903
+ modelingSummary,
1904
+ durationSummary: buildDurationSummary(entries),
1905
+ combinedRunFindings,
1906
+ combinedAllFindings,
1907
+ };
1908
+ };
1909
+ const temporaryWritePath = (path) => `${path}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`;
1910
+ const writeTextAtomic = async (path, value) => {
1911
+ await mkdir(dirname(path), { recursive: true });
1912
+ const tempPath = temporaryWritePath(path);
1913
+ await writeFile(tempPath, value, "utf-8");
1914
+ await rename(tempPath, path);
1915
+ };
1916
+ const writeJson = async (path, value) => {
1917
+ await writeTextAtomic(path, `${JSON.stringify(value, null, 2)}\n`);
1918
+ };
1919
+ const writeJsonlSnapshot = async (path, items) => {
1920
+ const content = items.map((item) => JSON.stringify(item)).join("\n");
1921
+ await writeTextAtomic(path, content.length ? `${content}\n` : "");
1922
+ };
1923
+ const initializeJsonlStream = async (path) => {
1924
+ await writeTextAtomic(path, "");
1925
+ };
1926
+ const appendJsonlLine = async (path, item) => {
1927
+ await mkdir(dirname(path), { recursive: true });
1928
+ await appendFile(path, `${JSON.stringify(item)}\n`, "utf-8");
1929
+ };
1930
+ const toRunManifest = (params) => {
1931
+ const { context, command, generatedAt, phase, expectations, projectGuide, snapshot } = params;
1932
+ const stdoutArtifact = snapshot.globalArtifacts.find((artifact) => artifact.displayName === "stdout.txt");
1933
+ const stderrArtifact = snapshot.globalArtifacts.find((artifact) => artifact.displayName === "stderr.txt");
1934
+ const originalExitCode = snapshot.globalExitCode?.original ?? null;
1935
+ const actualExitCode = snapshot.globalExitCode?.actual ?? snapshot.globalExitCode?.original ?? null;
1936
+ return {
1937
+ schema_version: AGENT_SCHEMA_VERSION,
1938
+ report_uuid: context.reportUuid,
1939
+ generated_at: generatedAt,
1940
+ phase,
1941
+ command: command ?? null,
1942
+ actual_exit_code: actualExitCode,
1943
+ original_exit_code: originalExitCode,
1944
+ exit_code: snapshot.globalExitCode
1945
+ ? {
1946
+ original: snapshot.globalExitCode.original,
1947
+ actual: snapshot.globalExitCode.actual ?? null,
1948
+ }
1949
+ : null,
1950
+ summary: {
1951
+ stats: snapshot.stats,
1952
+ modeled_stats: snapshot.modelingSummary.modeledStats,
1953
+ unmodeled_from_stats: snapshot.modelingSummary.unmodeledFromStats,
1954
+ compact: {
1955
+ ...snapshot.modelingSummary.compact,
1956
+ findings: snapshot.combinedAllFindings.length,
1957
+ },
1958
+ duration_ms: snapshot.durationSummary,
1959
+ environments: buildEnvironmentSummary(snapshot.entries),
1960
+ },
1961
+ modeling: snapshot.modelingSummary,
1962
+ paths: {
1963
+ index_md: "index.md",
1964
+ agents_md: "AGENTS.md",
1965
+ tests_manifest: "manifest/tests.jsonl",
1966
+ findings_manifest: "manifest/findings.jsonl",
1967
+ test_events_manifest: "manifest/test-events.jsonl",
1968
+ expected_manifest: expectations?.relativePath ?? null,
1969
+ project_guide: projectGuide?.relativePath ?? null,
1970
+ process_logs: {
1971
+ stdout: stdoutArtifact?.relativePath ?? null,
1972
+ stderr: stderrArtifact?.relativePath ?? null,
1973
+ },
1974
+ },
1975
+ expectations_present: Boolean(expectations),
1976
+ check_summary: buildCheckSummary(snapshot.combinedAllFindings),
1977
+ agent_context: {
1978
+ agent_name: env[AGENT_NAME_ENV] ?? null,
1979
+ loop_id: env[AGENT_LOOP_ID_ENV] ?? null,
1980
+ task_id: env[AGENT_TASK_ID_ENV] ?? expectations?.taskId ?? null,
1981
+ conversation_id: env[AGENT_CONVERSATION_ID_ENV] ?? null,
1982
+ },
1983
+ };
1984
+ };
1985
+ const writeSnapshotFiles = async (params) => {
1986
+ const { runtime, snapshot, phase } = params;
1987
+ const { outputDir, context, command, generatedAt, expectations, projectGuide } = runtime;
1988
+ const nextTestPaths = new Set(snapshot.entries.map((entry) => entry.filePath));
1989
+ const nextAssetDirs = new Set(snapshot.entries.map((entry) => join(outputDir, entry.relativeAssetDir)));
1990
+ for (const stalePath of runtime.currentTestPaths) {
1991
+ if (!nextTestPaths.has(stalePath)) {
1992
+ await rm(stalePath, { force: true });
1993
+ }
1994
+ }
1995
+ for (const staleDir of runtime.currentAssetDirs) {
1996
+ if (!nextAssetDirs.has(staleDir)) {
1997
+ await rm(staleDir, { recursive: true, force: true });
1998
+ }
1999
+ }
2000
+ runtime.currentTestPaths = nextTestPaths;
2001
+ runtime.currentAssetDirs = nextAssetDirs;
2002
+ await Promise.all(snapshot.entries.map(async (entry) => {
2003
+ const content = renderTestFile({
2004
+ entry,
2005
+ outputDir,
2006
+ });
2007
+ await writeTextAtomic(entry.filePath, content);
2008
+ }));
2009
+ await Promise.all([
2010
+ writeJson(join(outputDir, "manifest", "run.json"), toRunManifest({
2011
+ context,
2012
+ command,
2013
+ generatedAt,
2014
+ phase,
2015
+ expectations,
2016
+ projectGuide,
2017
+ snapshot,
2018
+ })),
2019
+ writeJsonlSnapshot(join(outputDir, "manifest", "tests.jsonl"), snapshot.entries.map(toTestsManifestLine)),
2020
+ writeJsonlSnapshot(join(outputDir, "manifest", "findings.jsonl"), snapshot.combinedAllFindings.map(toFindingManifestLine)),
2021
+ writeTextAtomic(join(outputDir, "index.md"), renderIndex({
2022
+ context,
2023
+ command,
2024
+ generatedAt,
2025
+ phase,
2026
+ stats: snapshot.stats,
2027
+ durationSummary: snapshot.durationSummary,
2028
+ environmentSummary: buildEnvironmentSummary(snapshot.entries),
2029
+ modelingSummary: snapshot.modelingSummary,
2030
+ expectations,
2031
+ tests: snapshot.entries,
2032
+ globalArtifacts: snapshot.globalArtifacts,
2033
+ globalErrors: snapshot.globalErrors,
2034
+ globalExitCode: snapshot.globalExitCode,
2035
+ qualityGateResults: snapshot.qualityGateResults,
2036
+ findings: snapshot.combinedAllFindings,
2037
+ })),
2038
+ writeTextAtomic(join(outputDir, "AGENTS.md"), renderAgentsGuide(projectGuide?.relativePath)),
2039
+ ]);
2040
+ };
2041
+ const createBootstrapSnapshot = () => ({
2042
+ stats: { total: 0 },
2043
+ entries: [],
2044
+ globalArtifacts: [],
2045
+ globalErrors: [],
2046
+ globalExitCode: undefined,
2047
+ qualityGateResults: [],
2048
+ modelingSummary: {
2049
+ completeness: "complete",
2050
+ reasons: [],
2051
+ modeledStats: emptyStatusCounts(),
2052
+ unmodeledFromStats: emptyStatusCounts(),
2053
+ runnerFailures: {
2054
+ total: 0,
2055
+ globalErrors: 0,
2056
+ stderrActionable: 0,
2057
+ samples: [],
2058
+ },
2059
+ stderr: {
2060
+ actionableCount: 0,
2061
+ actionableSamples: [],
2062
+ noisyWarningCount: 0,
2063
+ noisyWarningSamples: [],
2064
+ },
2065
+ compact: {
2066
+ visible_results: 0,
2067
+ logical_tests: 0,
2068
+ unmodeled_visible_results: 0,
2069
+ runner_failures_outside_logical_tests: 0,
2070
+ completeness: "complete",
2071
+ },
2072
+ },
2073
+ durationSummary: {
2074
+ total: 0,
2075
+ average: 0,
2076
+ max: 0,
2077
+ },
2078
+ combinedRunFindings: [],
2079
+ combinedAllFindings: [],
2080
+ });
2081
+ const writeBootstrapFiles = async (runtime) => {
2082
+ await writeTextAtomic(join(runtime.outputDir, "AGENTS.md"), renderAgentsGuide(runtime.projectGuide?.relativePath));
2083
+ await initializeJsonlStream(join(runtime.outputDir, "manifest", "test-events.jsonl"));
2084
+ await writeSnapshotFiles({
2085
+ runtime,
2086
+ snapshot: createBootstrapSnapshot(),
2087
+ phase: "running",
2088
+ });
2089
+ };
2090
+ const toTestsManifestLine = (entry) => ({
2091
+ environment_id: entry.environmentId,
2092
+ history_id: entry.tr.historyId ?? null,
2093
+ test_result_id: entry.tr.id,
2094
+ full_name: entry.tr.fullName ?? entry.tr.name,
2095
+ package: entry.packageName ?? null,
2096
+ labels: toLabelEntries(entry.tr.labels),
2097
+ status: entry.tr.status,
2098
+ duration_ms: entry.tr.duration ?? 0,
2099
+ retries: Math.max(entry.attempts.length - 1, 0),
2100
+ flaky: entry.tr.flaky,
2101
+ scope_match: entry.scope.scopeMatch,
2102
+ scope_reasons: entry.scope.reasons,
2103
+ finding_counts: toFindingCounts(entry.findings),
2104
+ markdown_path: entry.relativePath,
2105
+ assets_dir: entry.relativeAssetDir,
2106
+ });
2107
+ const toFindingManifestLine = (finding) => ({
2108
+ finding_id: finding.findingId,
2109
+ subject: finding.subject,
2110
+ severity: finding.severity,
2111
+ category: finding.category,
2112
+ check_name: finding.checkName,
2113
+ message: finding.message,
2114
+ explanation: finding.explanation,
2115
+ evidence_paths: finding.evidencePaths,
2116
+ remediation_hint: finding.remediationHint,
2117
+ expected_reference: finding.expectedReference,
2118
+ confidence: finding.confidence,
2119
+ });
2120
+ const queueRuntimeTask = (runtime, task) => {
2121
+ runtime.queue = runtime.queue
2122
+ .catch(() => undefined)
2123
+ .then(async () => {
2124
+ try {
2125
+ await task();
2126
+ }
2127
+ catch (error) {
2128
+ runtime.lastError = error;
2129
+ throw error;
2130
+ }
2131
+ });
2132
+ return runtime.queue;
2133
+ };
2134
+ const appendRuntimeEvent = async (runtime, eventType, payload) => {
2135
+ runtime.eventCounter += 1;
2136
+ await appendJsonlLine(join(runtime.outputDir, "manifest", "test-events.jsonl"), {
2137
+ sequence: runtime.eventCounter,
2138
+ at: new Date().toISOString(),
2139
+ event_type: eventType,
2140
+ ...payload,
2141
+ });
2142
+ };
2143
+ const removeStaleLiveFiles = async (runtime, liveTestIds) => {
2144
+ for (const [testId, path] of Array.from(runtime.currentEntryPathByTestId.entries())) {
2145
+ if (!liveTestIds.has(testId)) {
2146
+ await rm(path, { force: true });
2147
+ runtime.currentEntryPathByTestId.delete(testId);
2148
+ runtime.currentTestPaths.delete(path);
2149
+ }
2150
+ }
2151
+ for (const [testId, assetDir] of Array.from(runtime.currentAssetDirByTestId.entries())) {
2152
+ if (!liveTestIds.has(testId)) {
2153
+ await rm(assetDir, { recursive: true, force: true });
2154
+ runtime.currentAssetDirByTestId.delete(testId);
2155
+ runtime.currentAssetDirs.delete(assetDir);
2156
+ }
2157
+ }
2158
+ };
2159
+ const writeLiveEntry = async (runtime, entry) => {
2160
+ const previousPath = runtime.currentEntryPathByTestId.get(entry.tr.id);
2161
+ const previousAssetDir = runtime.currentAssetDirByTestId.get(entry.tr.id);
2162
+ if (previousPath && previousPath !== entry.filePath) {
2163
+ await rm(previousPath, { force: true });
2164
+ runtime.currentTestPaths.delete(previousPath);
2165
+ }
2166
+ if (previousAssetDir && previousAssetDir !== join(runtime.outputDir, entry.relativeAssetDir)) {
2167
+ await rm(previousAssetDir, { recursive: true, force: true });
2168
+ runtime.currentAssetDirs.delete(previousAssetDir);
2169
+ }
2170
+ await writeTextAtomic(entry.filePath, renderTestFile({
2171
+ entry,
2172
+ outputDir: runtime.outputDir,
2173
+ }));
2174
+ runtime.currentEntryPathByTestId.set(entry.tr.id, entry.filePath);
2175
+ runtime.currentAssetDirByTestId.set(entry.tr.id, join(runtime.outputDir, entry.relativeAssetDir));
2176
+ runtime.currentTestPaths.add(entry.filePath);
2177
+ runtime.currentAssetDirs.add(join(runtime.outputDir, entry.relativeAssetDir));
2178
+ };
2179
+ const buildImpactedLiveEntries = async (runtime, trIds) => {
2180
+ const changedIds = new Set(trIds);
2181
+ const layouts = await listVisibleTestLayouts({
2182
+ outputDir: runtime.outputDir,
2183
+ store: runtime.store,
2184
+ });
2185
+ const liveTestIds = new Set(layouts.map(({ tr }) => tr.id));
2186
+ const impactedLayouts = [];
2187
+ await removeStaleLiveFiles(runtime, liveTestIds);
2188
+ for (const layout of layouts) {
2189
+ const previousPath = runtime.currentEntryPathByTestId.get(layout.tr.id);
2190
+ const previousAssetDir = runtime.currentAssetDirByTestId.get(layout.tr.id);
2191
+ const nextAssetDir = join(runtime.outputDir, layout.relativeAssetDir);
2192
+ let impacted = changedIds.has(layout.tr.id) || previousPath !== layout.filePath || previousAssetDir !== nextAssetDir;
2193
+ if (!impacted) {
2194
+ const retries = await runtime.store.retriesByTr(layout.tr);
2195
+ impacted = retries.some((retry) => changedIds.has(retry.id));
2196
+ }
2197
+ if (impacted) {
2198
+ impactedLayouts.push(layout);
2199
+ }
2200
+ }
2201
+ const entries = [];
2202
+ for (const layout of impactedLayouts) {
2203
+ entries.push(await buildEntryFromLayout({
2204
+ layout,
2205
+ store: runtime.store,
2206
+ expectations: runtime.expectations,
2207
+ }));
2208
+ }
2209
+ return entries;
2210
+ };
2211
+ const streamLiveTestUpdates = async (runtime, trIds) => {
2212
+ const entries = await buildImpactedLiveEntries(runtime, trIds);
2213
+ for (const entry of entries) {
2214
+ await writeLiveEntry(runtime, entry);
2215
+ const eventType = runtime.seenLogicalKeys.has(entry.key) ? "test_updated" : "test_completed";
2216
+ runtime.seenLogicalKeys.add(entry.key);
2217
+ await appendRuntimeEvent(runtime, eventType, {
2218
+ logical_key: entry.key,
2219
+ markdown_path: entry.relativePath,
2220
+ assets_dir: entry.relativeAssetDir,
2221
+ environment_id: entry.environmentId,
2222
+ test_result_id: entry.tr.id,
2223
+ related_test_result_ids: entry.attempts.map((attempt) => attempt.tr.id),
2224
+ full_name: entry.tr.fullName ?? entry.tr.name,
2225
+ status: entry.tr.status,
2226
+ retries: Math.max(entry.attempts.length - 1, 0),
2227
+ finding_counts: toFindingCounts(entry.findings),
2228
+ });
2229
+ }
2230
+ };
2231
+ const createRuntimeState = async (params) => {
2232
+ const { options, context, store } = params;
2233
+ const outputDir = resolveOutputDir(options);
2234
+ if (!outputDir) {
2235
+ return undefined;
2236
+ }
2237
+ await mkdir(outputDir, { recursive: true });
2238
+ await cleanupManagedEntries(outputDir);
2239
+ const generatedAt = new Date().toISOString();
2240
+ const createFinding = createFindingFactory();
2241
+ const expectationLoadResult = await loadExpectations(outputDir, createFinding);
2242
+ const projectGuide = await loadProjectGuide(outputDir);
2243
+ const runtime = {
2244
+ outputDir,
2245
+ context,
2246
+ store,
2247
+ generatedAt,
2248
+ command: env[AGENT_COMMAND_ENV],
2249
+ createFinding,
2250
+ expectations: expectationLoadResult.expectations,
2251
+ expectationLoadFindings: expectationLoadResult.findings,
2252
+ projectGuide,
2253
+ unsubscribers: [],
2254
+ queue: Promise.resolve(),
2255
+ seenLogicalKeys: new Set(),
2256
+ currentTestPaths: new Set(),
2257
+ currentAssetDirs: new Set(),
2258
+ currentEntryPathByTestId: new Map(),
2259
+ currentAssetDirByTestId: new Map(),
2260
+ finalized: false,
2261
+ eventCounter: 0,
2262
+ };
2263
+ await writeBootstrapFiles(runtime);
2264
+ return runtime;
2265
+ };
2266
+ export class AgentPlugin {
2267
+ constructor(options = {}) {
2268
+ this.options = options;
2269
+ _AgentPlugin_runtime.set(this, void 0);
2270
+ this.start = async (context, store, realtime) => {
2271
+ if (__classPrivateFieldGet(this, _AgentPlugin_runtime, "f")) {
2272
+ return;
2273
+ }
2274
+ const runtime = await createRuntimeState({
2275
+ options: this.options,
2276
+ context,
2277
+ store,
2278
+ });
2279
+ if (!runtime) {
2280
+ return;
2281
+ }
2282
+ runtime.unsubscribers.push(realtime.onTestResults(async (trIds) => {
2283
+ await queueRuntimeTask(runtime, async () => {
2284
+ await streamLiveTestUpdates(runtime, trIds);
2285
+ });
2286
+ }, { maxTimeout: 0 }));
2287
+ runtime.unsubscribers.push(realtime.onGlobalAttachment(async () => {
2288
+ await queueRuntimeTask(runtime, async () => {
2289
+ });
2290
+ }));
2291
+ runtime.unsubscribers.push(realtime.onGlobalError(async (error) => {
2292
+ await queueRuntimeTask(runtime, async () => {
2293
+ await appendRuntimeEvent(runtime, "run_error", {
2294
+ message: error.message ?? "Captured global error",
2295
+ });
2296
+ });
2297
+ }));
2298
+ runtime.unsubscribers.push(realtime.onGlobalExitCode(async (payload) => {
2299
+ await queueRuntimeTask(runtime, async () => {
2300
+ if ((payload.actual ?? payload.original) !== 0) {
2301
+ await appendRuntimeEvent(runtime, "run_error", {
2302
+ message: `Observed exit code ${payload.actual ?? payload.original}`,
2303
+ exit_code: payload,
2304
+ });
2305
+ }
2306
+ });
2307
+ }));
2308
+ runtime.unsubscribers.push(realtime.onQualityGateResults(async (results) => {
2309
+ await queueRuntimeTask(runtime, async () => {
2310
+ if (results.some(({ success }) => !success)) {
2311
+ await appendRuntimeEvent(runtime, "run_warning", {
2312
+ message: "Quality gate reported failing rules during the run.",
2313
+ failed_rules: results
2314
+ .filter(({ success }) => !success)
2315
+ .map(({ rule, environment, message }) => ({
2316
+ rule,
2317
+ environment,
2318
+ message,
2319
+ })),
2320
+ });
2321
+ }
2322
+ });
2323
+ }));
2324
+ __classPrivateFieldSet(this, _AgentPlugin_runtime, runtime, "f");
2325
+ };
2326
+ this.done = async (context, store) => {
2327
+ const runtime = __classPrivateFieldGet(this, _AgentPlugin_runtime, "f") ??
2328
+ (await createRuntimeState({
2329
+ options: this.options,
2330
+ context,
2331
+ store,
2332
+ }));
2333
+ if (!runtime) {
2334
+ return;
2335
+ }
2336
+ __classPrivateFieldSet(this, _AgentPlugin_runtime, runtime, "f");
2337
+ if (runtime.finalized) {
2338
+ return;
2339
+ }
2340
+ await runtime.queue.catch(() => undefined);
2341
+ for (const unsubscribe of runtime.unsubscribers.splice(0)) {
2342
+ unsubscribe();
2343
+ }
2344
+ const snapshot = await buildSnapshot({
2345
+ outputDir: runtime.outputDir,
2346
+ store: runtime.store,
2347
+ expectations: runtime.expectations,
2348
+ expectationLoadFindings: runtime.expectationLoadFindings,
2349
+ createFinding: runtime.createFinding,
2350
+ });
2351
+ await writeSnapshotFiles({
2352
+ runtime,
2353
+ snapshot,
2354
+ phase: "done",
2355
+ });
2356
+ await appendRuntimeEvent(runtime, "run_finished", {
2357
+ completeness: snapshot.modelingSummary.completeness,
2358
+ findings: snapshot.combinedAllFindings.length,
2359
+ logical_tests: snapshot.entries.length,
2360
+ });
2361
+ runtime.finalized = true;
2362
+ if (runtime.lastError) {
2363
+ throw runtime.lastError;
2364
+ }
2365
+ };
2366
+ }
2367
+ }
2368
+ _AgentPlugin_runtime = new WeakMap();