@crvouga/atlas 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/playwright-reporter/src/evidence.d.ts +122 -0
- package/dist/playwright-reporter/src/evidence.js +327 -0
- package/dist/playwright-reporter/src/index.d.ts +114 -0
- package/dist/playwright-reporter/src/index.js +312 -0
- package/dist/reports/src/compose.js +7 -14
- package/dist/reports/src/recorded-specs.d.ts +33 -0
- package/dist/reports/src/recorded-specs.js +42 -0
- package/dist/reports/src/report-index.d.ts +6 -0
- package/dist/reports/src/report-index.js +14 -2
- package/dist/reports/src/test-catalog.d.ts +386 -0
- package/dist/reports/src/test-catalog.js +655 -0
- package/dist/schema/src/api.d.ts +585 -0
- package/dist/schema/src/api.js +105 -0
- package/dist/server/src/app.js +82 -4
- package/dist/server/src/llms.js +14 -0
- package/dist/server/src/openapi.d.ts +1 -1
- package/dist/server/src/openapi.js +60 -3
- package/docs/agents.md +1 -0
- package/docs/integrating.md +1 -0
- package/docs/reference/http-api.md +134 -2
- package/docs/reference/playwright-reporter.md +332 -0
- package/docs/reference/reports.md +632 -0
- package/docs/reference/schema.md +57 -0
- package/docs/rich-evidence.md +198 -0
- package/docs/server.md +41 -0
- package/llms-full.txt +378 -2
- package/llms.txt +2 -0
- package/package.json +6 -1
- package/packages/schema/schemas/report-run.schema.json +251 -0
- package/packages/schema/schemas/report-runs.schema.json +177 -0
- package/packages/schema/schemas/test-catalog.schema.json +1177 -0
- package/packages/schema/schemas/test-history.schema.json +696 -0
- package/packages/server/openapi.json +8676 -5351
- package/packages/visualizer/src/app/AppShell.tsx +7 -3
- package/packages/visualizer/src/app/router.tsx +26 -3
- package/packages/visualizer/src/components/Header.tsx +2 -2
- package/packages/visualizer/src/evidence/evidence-context.tsx +10 -1
- package/packages/visualizer/src/explorer/Explorer.tsx +433 -0
- package/packages/visualizer/src/explorer/ExplorerIndex.tsx +120 -0
- package/packages/visualizer/src/explorer/RunsView.tsx +164 -0
- package/packages/visualizer/src/explorer/TestDetail.tsx +214 -0
- package/packages/visualizer/src/explorer/catalog.tsx +274 -0
- package/packages/visualizer/src/explorer/explorer.module.css +1238 -0
- package/packages/visualizer/src/explorer/groups.test.ts +87 -0
- package/packages/visualizer/src/explorer/groups.ts +126 -0
- package/packages/visualizer/src/explorer/navigate.ts +36 -0
- package/packages/visualizer/src/explorer/parts.tsx +59 -0
- package/packages/visualizer/src/explorer/search.ts +97 -0
- package/packages/visualizer/src/find/Finder.tsx +2 -0
- package/packages/visualizer/src/find/corpus.ts +13 -6
- package/packages/visualizer/src/find/rank.ts +2 -0
- package/packages/visualizer/src/evidence/project-home.tsx +0 -45
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog: @crvouga/atlas
|
|
2
2
|
|
|
3
|
+
## 0.10.0 (2026-10-21)
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
|
|
7
|
+
- explore tests by spec, environment and run, readable by agents (#15) ([cb2b8da](https://github.com/crvouga/atlas/commit/cb2b8da7a3c7ec8ca8e3849021ea43358eb593a3))
|
|
8
|
+
|
|
3
9
|
## 0.9.0 (2026-10-15)
|
|
4
10
|
|
|
5
11
|
### Features
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import type { FullProject, TestCase, TestError, TestResult, TestStep } from 'playwright/types/testReporter';
|
|
2
|
+
import type { PublishChecksInput } from '../../schema/src/layer.js';
|
|
3
|
+
/** One check as handed to the streaming reporter: the input type of a check in `PublishChecksSchema`. */
|
|
4
|
+
export type CheckInput = PublishChecksInput['checks'][number];
|
|
5
|
+
type AttachmentInput = NonNullable<CheckInput['attachments']>[number];
|
|
6
|
+
type StepInput = NonNullable<CheckInput['steps']>[number];
|
|
7
|
+
type AnnotationInput = NonNullable<CheckInput['annotations']>[number];
|
|
8
|
+
/** Which results upload their attachments: every one, only failed checks and failed attempts, or none. */
|
|
9
|
+
export type AttachmentMode = 'all' | 'failures' | 'none';
|
|
10
|
+
/** What to upload of the files Playwright attached: the `mode`, and the largest single file in bytes (`maxBytes`). */
|
|
11
|
+
export type AttachmentPolicy = {
|
|
12
|
+
/** `all`, `failures` (failed checks and failed attempts only) or `none`. */
|
|
13
|
+
readonly mode: AttachmentMode;
|
|
14
|
+
/** A larger file is not uploaded; the check gets an `attachment-skipped` annotation instead. */
|
|
15
|
+
readonly maxBytes: number;
|
|
16
|
+
};
|
|
17
|
+
/** Where a check comes from and how it is recorded, as worked out by the reporter before its evidence is collected. */
|
|
18
|
+
export type CheckContext = {
|
|
19
|
+
/** The check's stable id: `<specPath>#<describe titles > test title>`. */
|
|
20
|
+
readonly id: string;
|
|
21
|
+
readonly title: string;
|
|
22
|
+
/** The spec the check belongs to, relative to the repository. */
|
|
23
|
+
readonly specPath: string;
|
|
24
|
+
/** Facts about where it ran: browser, device, viewport, os, playwright version. */
|
|
25
|
+
readonly labels: Readonly<Record<string, string>>;
|
|
26
|
+
/** Relative paths (source locations) are written relative to this directory. */
|
|
27
|
+
readonly root: string;
|
|
28
|
+
readonly attachments: AttachmentPolicy;
|
|
29
|
+
/** The most timeline steps recorded per result; the rest are counted in an annotation. */
|
|
30
|
+
readonly maxSteps: number;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* Removes terminal colour and cursor codes, which Playwright puts in error messages and step titles.
|
|
34
|
+
* @example
|
|
35
|
+
* stripAnsi('\u001b[31mExpected\u001b[39m'); // 'Expected'
|
|
36
|
+
*/
|
|
37
|
+
export declare function stripAnsi(text: string): string;
|
|
38
|
+
/**
|
|
39
|
+
* The failure detail of an error: its message and stack without terminal codes, the message once, cut to `maxChars`.
|
|
40
|
+
* An absent error is the empty string.
|
|
41
|
+
* @param error A Playwright `TestError` (a result's or a step's).
|
|
42
|
+
* @param maxChars The longest detail kept. Default 8000.
|
|
43
|
+
*/
|
|
44
|
+
export declare function errorDetail(error: TestError | undefined, maxChars?: number): string;
|
|
45
|
+
/**
|
|
46
|
+
* The Atlas status of one Playwright result: skipped is `not-run`; the status the test expected (`passed`, or `failed`
|
|
47
|
+
* for a `test.fail()`) is `passed`; anything else (failed, timed out, interrupted, an expected failure that passed) is
|
|
48
|
+
* `failed`.
|
|
49
|
+
*/
|
|
50
|
+
export declare function resultStatus(test: Pick<TestCase, 'expectedStatus'>, result: Pick<TestResult, 'status'>): 'passed' | 'failed' | 'not-run';
|
|
51
|
+
/** Whether Playwright will run this test again after this result: it did not get what it expected and has retries left. */
|
|
52
|
+
export declare function willRetry(test: Pick<TestCase, 'expectedStatus' | 'retries'>, result: Pick<TestResult, 'status' | 'retry'>): boolean;
|
|
53
|
+
/**
|
|
54
|
+
* The timeline kind of a Playwright step, by its category: `test.step` is `step` (or the Gherkin keyword, `given`,
|
|
55
|
+
* `when`, `then`, `and`, `but`, for a playwright-bdd step), `expect` is `assertion`, `pw:api` is `action`, `hook`,
|
|
56
|
+
* `fixture`; any other category is kept as it is.
|
|
57
|
+
*/
|
|
58
|
+
export declare function stepKind(step: Pick<TestStep, 'category' | 'title'>): string;
|
|
59
|
+
/**
|
|
60
|
+
* Playwright's step tree as Atlas's flat timeline: each step gets an id (`s1`, `s2`, …), its parent's id, its kind,
|
|
61
|
+
* when it started after the result did (`offsetMs`), how long it took, and `failed` with the error as detail when it
|
|
62
|
+
* failed. At most `max` steps are kept, in order; `dropped` counts the rest.
|
|
63
|
+
* @param steps `result.steps`.
|
|
64
|
+
* @param startedAtMs When the result started (`result.startTime`), in epoch milliseconds.
|
|
65
|
+
* @param max The most steps kept.
|
|
66
|
+
*/
|
|
67
|
+
export declare function flattenSteps(steps: readonly TestStep[], startedAtMs: number, max: number): {
|
|
68
|
+
steps: StepInput[];
|
|
69
|
+
dropped: number;
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* The closest directory at or above `start` that holds `.git` (a folder, or a file in a worktree), or null. Reads the
|
|
73
|
+
* file system only; it never runs git.
|
|
74
|
+
*/
|
|
75
|
+
export declare function findRepoRoot(start: string, exists?: (file: string) => boolean): string | null;
|
|
76
|
+
/** `file` relative to `root`, with forward slashes on every OS. */
|
|
77
|
+
export declare function repoPath(file: string, root: string): string;
|
|
78
|
+
/**
|
|
79
|
+
* The `.feature` file a playwright-bdd generated test came from, or undefined for any other test file. playwright-bdd
|
|
80
|
+
* writes `<outputDir>/<path>.feature.spec.js` (its outputDir is `.features-gen` by default, mirroring the feature
|
|
81
|
+
* folders); this returns the first `<path>.feature` that exists beside `.features-gen` (dropping leading folders, such
|
|
82
|
+
* as a per-project output folder, until one does), else the mapped path even if no such file is found.
|
|
83
|
+
* @param file The absolute path of the test file (`test.location.file`).
|
|
84
|
+
* @example
|
|
85
|
+
* bddFeaturePath('/repo/e2e/.features-gen/features/login.feature.spec.js'); // '/repo/e2e/features/login.feature'
|
|
86
|
+
*/
|
|
87
|
+
export declare function bddFeaturePath(file: string, exists?: (file: string) => boolean): string | undefined;
|
|
88
|
+
/** The environment a project tests by default: `mobile-web` when it emulates a mobile device (`use.isMobile`), else `desktop-web`. */
|
|
89
|
+
export declare function projectEnvironment(project: Pick<FullProject, 'use'> | undefined): string;
|
|
90
|
+
/**
|
|
91
|
+
* What every check of a project carries as labels: `browser` (`use.browserName`, else the device's
|
|
92
|
+
* `defaultBrowserType`), `channel`, `device` (the project name), `viewport` (`<width>x<height>`), `os`
|
|
93
|
+
* (`process.platform`) and `playwright` (its version). Labels describe a result; they never split its slot.
|
|
94
|
+
* @param project The Playwright project, or undefined when it is not known.
|
|
95
|
+
* @param version Playwright's version (`config.version`).
|
|
96
|
+
*/
|
|
97
|
+
export declare function projectLabels(project: Pick<FullProject, 'name' | 'use'> | undefined, version?: string): Record<string, string>;
|
|
98
|
+
/**
|
|
99
|
+
* The files and output of one result as attachments with their bytes: every `result.attachments` entry (role `video`,
|
|
100
|
+
* `screenshot` or `trace` by its Playwright name, else the name itself; bytes from `body`, or read from `path`), and
|
|
101
|
+
* standard output and error as `log` text. A file over the size limit, or that cannot be read, is left out with an
|
|
102
|
+
* `attachment-skipped` annotation saying why. Never throws.
|
|
103
|
+
* @param result The Playwright result.
|
|
104
|
+
* @param failed Whether the result failed, for the `failures` mode.
|
|
105
|
+
* @param policy What to upload.
|
|
106
|
+
*/
|
|
107
|
+
export declare function resultAttachments(result: Pick<TestResult, 'attachments' | 'stdout' | 'stderr'>, failed: boolean, policy: AttachmentPolicy): Promise<{
|
|
108
|
+
attachments: AttachmentInput[];
|
|
109
|
+
annotations: AnnotationInput[];
|
|
110
|
+
}>;
|
|
111
|
+
/**
|
|
112
|
+
* The final check of a Playwright test from all its results (one per try): status of the last result (`flaky`, as a
|
|
113
|
+
* tag and a label, when it passed after a failed try), failure detail without terminal codes, duration, tags (the
|
|
114
|
+
* test's tags without `@`, and its annotation types), annotations, links (annotations whose description is a URL),
|
|
115
|
+
* timeline steps, one attempt per try when it was retried, attachments, and the test's location under the
|
|
116
|
+
* `playwright` extension. Reads attached files; never throws for a file it cannot read.
|
|
117
|
+
* @param test The Playwright test.
|
|
118
|
+
* @param results Its results, oldest first (`test.results`).
|
|
119
|
+
* @param context Its identity and what to record.
|
|
120
|
+
*/
|
|
121
|
+
export declare function checkOf(test: TestCase, results: readonly TestResult[], context: CheckContext): Promise<CheckInput>;
|
|
122
|
+
export {};
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Playwright results as Atlas evidence
|
|
3
|
+
*
|
|
4
|
+
* The pure half of the Playwright Test reporter: how one Playwright test and its results become one Atlas check, with
|
|
5
|
+
* its status, failure detail, duration, tags, annotations, links, timeline steps, retry attempts and attachments, and
|
|
6
|
+
* how a project becomes an environment and labels. Nothing here talks to a store; the only input/output is reading the
|
|
7
|
+
* files Playwright attached (videos, traces, screenshots) and looking for a repository root or a `.feature` file.
|
|
8
|
+
*
|
|
9
|
+
* @see docs/rich-evidence.md
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync } from 'node:fs';
|
|
12
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
13
|
+
import path from 'node:path';
|
|
14
|
+
const ANSI = /\u001b\[[0-9;?]*[ -/]*[@-~]/g;
|
|
15
|
+
/**
|
|
16
|
+
* Removes terminal colour and cursor codes, which Playwright puts in error messages and step titles.
|
|
17
|
+
* @example
|
|
18
|
+
* stripAnsi('\u001b[31mExpected\u001b[39m'); // 'Expected'
|
|
19
|
+
*/
|
|
20
|
+
export function stripAnsi(text) {
|
|
21
|
+
return text.replace(ANSI, '');
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The failure detail of an error: its message and stack without terminal codes, the message once, cut to `maxChars`.
|
|
25
|
+
* An absent error is the empty string.
|
|
26
|
+
* @param error A Playwright `TestError` (a result's or a step's).
|
|
27
|
+
* @param maxChars The longest detail kept. Default 8000.
|
|
28
|
+
*/
|
|
29
|
+
export function errorDetail(error, maxChars = 8000) {
|
|
30
|
+
if (!error)
|
|
31
|
+
return '';
|
|
32
|
+
const message = stripAnsi(error.message ?? error.value ?? '').trim();
|
|
33
|
+
const stack = stripAnsi(error.stack ?? '').trim();
|
|
34
|
+
const text = !stack ? message : !message || stack.includes(message) ? stack : `${message}\n${stack}`;
|
|
35
|
+
return text.length > maxChars ? `${text.slice(0, maxChars)}…` : text;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The Atlas status of one Playwright result: skipped is `not-run`; the status the test expected (`passed`, or `failed`
|
|
39
|
+
* for a `test.fail()`) is `passed`; anything else (failed, timed out, interrupted, an expected failure that passed) is
|
|
40
|
+
* `failed`.
|
|
41
|
+
*/
|
|
42
|
+
export function resultStatus(test, result) {
|
|
43
|
+
if (result.status === 'skipped')
|
|
44
|
+
return 'not-run';
|
|
45
|
+
if (result.status === test.expectedStatus)
|
|
46
|
+
return 'passed';
|
|
47
|
+
return 'failed';
|
|
48
|
+
}
|
|
49
|
+
/** Whether Playwright will run this test again after this result: it did not get what it expected and has retries left. */
|
|
50
|
+
export function willRetry(test, result) {
|
|
51
|
+
return result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== test.expectedStatus && result.retry < test.retries;
|
|
52
|
+
}
|
|
53
|
+
const GHERKIN = /^(given|when|then|and|but)\b/i;
|
|
54
|
+
const STEP_KINDS = { expect: 'assertion', 'pw:api': 'action', hook: 'hook', fixture: 'fixture', 'test.attach': 'attach' };
|
|
55
|
+
/**
|
|
56
|
+
* The timeline kind of a Playwright step, by its category: `test.step` is `step` (or the Gherkin keyword, `given`,
|
|
57
|
+
* `when`, `then`, `and`, `but`, for a playwright-bdd step), `expect` is `assertion`, `pw:api` is `action`, `hook`,
|
|
58
|
+
* `fixture`; any other category is kept as it is.
|
|
59
|
+
*/
|
|
60
|
+
export function stepKind(step) {
|
|
61
|
+
if (step.category === 'test.step')
|
|
62
|
+
return GHERKIN.exec(stripAnsi(step.title).trim())?.[1]?.toLowerCase() ?? 'step';
|
|
63
|
+
return STEP_KINDS[step.category] ?? step.category;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Playwright's step tree as Atlas's flat timeline: each step gets an id (`s1`, `s2`, …), its parent's id, its kind,
|
|
67
|
+
* when it started after the result did (`offsetMs`), how long it took, and `failed` with the error as detail when it
|
|
68
|
+
* failed. At most `max` steps are kept, in order; `dropped` counts the rest.
|
|
69
|
+
* @param steps `result.steps`.
|
|
70
|
+
* @param startedAtMs When the result started (`result.startTime`), in epoch milliseconds.
|
|
71
|
+
* @param max The most steps kept.
|
|
72
|
+
*/
|
|
73
|
+
export function flattenSteps(steps, startedAtMs, max) {
|
|
74
|
+
const out = [];
|
|
75
|
+
let dropped = 0;
|
|
76
|
+
const count = (list) => list.reduce((sum, step) => sum + 1 + count(step.steps ?? []), 0);
|
|
77
|
+
const visit = (list, parentId) => {
|
|
78
|
+
for (const step of list) {
|
|
79
|
+
if (out.length >= max) {
|
|
80
|
+
dropped += 1 + count(step.steps ?? []);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const id = `s${out.length + 1}`;
|
|
84
|
+
const skipped = (step.annotations ?? []).some(annotation => annotation.type === 'skip');
|
|
85
|
+
const started = step.startTime instanceof Date ? step.startTime.getTime() : Number.NaN;
|
|
86
|
+
out.push({
|
|
87
|
+
id,
|
|
88
|
+
...(parentId ? { parentId } : {}),
|
|
89
|
+
name: stripAnsi(step.title),
|
|
90
|
+
kind: stepKind(step),
|
|
91
|
+
status: step.error ? 'failed' : skipped ? 'skipped' : 'passed',
|
|
92
|
+
...(Number.isFinite(started) ? { offsetMs: Math.max(0, started - startedAtMs) } : {}),
|
|
93
|
+
...(Number.isFinite(step.duration) && step.duration >= 0 ? { durationMs: step.duration } : {}),
|
|
94
|
+
...(step.error ? { detail: errorDetail(step.error, 2000) } : {})
|
|
95
|
+
});
|
|
96
|
+
visit(step.steps ?? [], id);
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
visit(steps, undefined);
|
|
100
|
+
return { steps: out, dropped };
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* The closest directory at or above `start` that holds `.git` (a folder, or a file in a worktree), or null. Reads the
|
|
104
|
+
* file system only; it never runs git.
|
|
105
|
+
*/
|
|
106
|
+
export function findRepoRoot(start, exists = existsSync) {
|
|
107
|
+
for (let dir = path.resolve(start);; dir = path.dirname(dir)) {
|
|
108
|
+
if (exists(path.join(dir, '.git')))
|
|
109
|
+
return dir;
|
|
110
|
+
if (path.dirname(dir) === dir)
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/** `file` relative to `root`, with forward slashes on every OS. */
|
|
115
|
+
export function repoPath(file, root) {
|
|
116
|
+
return path.relative(root, path.resolve(root, file)).replaceAll('\\', '/');
|
|
117
|
+
}
|
|
118
|
+
const GENERATED = /\.feature\.spec\.[cm]?[jt]s$/;
|
|
119
|
+
/**
|
|
120
|
+
* The `.feature` file a playwright-bdd generated test came from, or undefined for any other test file. playwright-bdd
|
|
121
|
+
* writes `<outputDir>/<path>.feature.spec.js` (its outputDir is `.features-gen` by default, mirroring the feature
|
|
122
|
+
* folders); this returns the first `<path>.feature` that exists beside `.features-gen` (dropping leading folders, such
|
|
123
|
+
* as a per-project output folder, until one does), else the mapped path even if no such file is found.
|
|
124
|
+
* @param file The absolute path of the test file (`test.location.file`).
|
|
125
|
+
* @example
|
|
126
|
+
* bddFeaturePath('/repo/e2e/.features-gen/features/login.feature.spec.js'); // '/repo/e2e/features/login.feature'
|
|
127
|
+
*/
|
|
128
|
+
export function bddFeaturePath(file, exists = existsSync) {
|
|
129
|
+
const normalized = file.replaceAll('\\', '/');
|
|
130
|
+
if (!GENERATED.test(normalized))
|
|
131
|
+
return undefined;
|
|
132
|
+
const marker = normalized.lastIndexOf('/.features-gen/');
|
|
133
|
+
if (marker < 0)
|
|
134
|
+
return normalized.replace(GENERATED, '.feature');
|
|
135
|
+
const base = normalized.slice(0, marker);
|
|
136
|
+
const rest = normalized.slice(marker + '/.features-gen/'.length).replace(GENERATED, '.feature').split('/');
|
|
137
|
+
for (let skip = 0; skip < rest.length; skip++) {
|
|
138
|
+
const candidate = `${base}/${rest.slice(skip).join('/')}`;
|
|
139
|
+
if (exists(candidate))
|
|
140
|
+
return candidate;
|
|
141
|
+
}
|
|
142
|
+
return `${base}/${rest.join('/')}`;
|
|
143
|
+
}
|
|
144
|
+
/** The environment a project tests by default: `mobile-web` when it emulates a mobile device (`use.isMobile`), else `desktop-web`. */
|
|
145
|
+
export function projectEnvironment(project) {
|
|
146
|
+
return project?.use?.isMobile ? 'mobile-web' : 'desktop-web';
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* What every check of a project carries as labels: `browser` (`use.browserName`, else the device's
|
|
150
|
+
* `defaultBrowserType`), `channel`, `device` (the project name), `viewport` (`<width>x<height>`), `os`
|
|
151
|
+
* (`process.platform`) and `playwright` (its version). Labels describe a result; they never split its slot.
|
|
152
|
+
* @param project The Playwright project, or undefined when it is not known.
|
|
153
|
+
* @param version Playwright's version (`config.version`).
|
|
154
|
+
*/
|
|
155
|
+
export function projectLabels(project, version) {
|
|
156
|
+
const use = (project?.use ?? {});
|
|
157
|
+
const labels = {};
|
|
158
|
+
const browser = use.browserName ?? use.defaultBrowserType;
|
|
159
|
+
if (browser)
|
|
160
|
+
labels.browser = browser;
|
|
161
|
+
if (use.channel)
|
|
162
|
+
labels.channel = use.channel;
|
|
163
|
+
labels.device = project?.name || 'default';
|
|
164
|
+
if (use.viewport && Number.isFinite(use.viewport.width) && Number.isFinite(use.viewport.height))
|
|
165
|
+
labels.viewport = `${use.viewport.width}x${use.viewport.height}`;
|
|
166
|
+
labels.os = process.platform;
|
|
167
|
+
if (version)
|
|
168
|
+
labels.playwright = version;
|
|
169
|
+
return labels;
|
|
170
|
+
}
|
|
171
|
+
const ROLES = { video: 'video', screenshot: 'screenshot', trace: 'trace' };
|
|
172
|
+
function wanted(policy, failed) {
|
|
173
|
+
return policy.mode === 'all' || (policy.mode === 'failures' && failed);
|
|
174
|
+
}
|
|
175
|
+
function skippedNote(name, message) {
|
|
176
|
+
return { type: 'attachment-skipped', level: 'warning', message: `${name} ${message}` };
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* The files and output of one result as attachments with their bytes: every `result.attachments` entry (role `video`,
|
|
180
|
+
* `screenshot` or `trace` by its Playwright name, else the name itself; bytes from `body`, or read from `path`), and
|
|
181
|
+
* standard output and error as `log` text. A file over the size limit, or that cannot be read, is left out with an
|
|
182
|
+
* `attachment-skipped` annotation saying why. Never throws.
|
|
183
|
+
* @param result The Playwright result.
|
|
184
|
+
* @param failed Whether the result failed, for the `failures` mode.
|
|
185
|
+
* @param policy What to upload.
|
|
186
|
+
*/
|
|
187
|
+
export async function resultAttachments(result, failed, policy) {
|
|
188
|
+
const attachments = [];
|
|
189
|
+
const annotations = [];
|
|
190
|
+
if (!wanted(policy, failed))
|
|
191
|
+
return { attachments, annotations };
|
|
192
|
+
for (const given of result.attachments ?? []) {
|
|
193
|
+
const role = ROLES[given.name] ?? given.name;
|
|
194
|
+
const name = given.path ? path.basename(given.path) : given.name;
|
|
195
|
+
const meta = { name, role, ...(given.contentType ? { contentType: given.contentType } : {}) };
|
|
196
|
+
try {
|
|
197
|
+
if (given.body) {
|
|
198
|
+
if (given.body.length > policy.maxBytes) {
|
|
199
|
+
annotations.push(skippedNote(name, `(${given.body.length} bytes) was not uploaded: larger than maxAttachmentBytes (${policy.maxBytes})`));
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
attachments.push({ ...meta, data: new Uint8Array(given.body) });
|
|
203
|
+
}
|
|
204
|
+
else if (given.path) {
|
|
205
|
+
const size = (await stat(given.path)).size;
|
|
206
|
+
if (size > policy.maxBytes) {
|
|
207
|
+
annotations.push(skippedNote(name, `(${size} bytes) was not uploaded: larger than maxAttachmentBytes (${policy.maxBytes})`));
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
attachments.push({ ...meta, data: new Uint8Array(await readFile(given.path)) });
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
annotations.push(skippedNote(name, `could not be read: ${error instanceof Error ? error.message : String(error)}`));
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
for (const [stream, chunks] of [['stdout', result.stdout], ['stderr', result.stderr]]) {
|
|
218
|
+
const text = (chunks ?? []).map(chunk => (typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk))).join('');
|
|
219
|
+
if (!text.trim())
|
|
220
|
+
continue;
|
|
221
|
+
const bytes = new TextEncoder().encode(text).length;
|
|
222
|
+
if (bytes > policy.maxBytes) {
|
|
223
|
+
annotations.push(skippedNote(stream, `(${bytes} bytes) was not uploaded: larger than maxAttachmentBytes (${policy.maxBytes})`));
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
attachments.push({ name: stream, role: 'log', contentType: 'text/plain', text });
|
|
227
|
+
}
|
|
228
|
+
return { attachments, annotations };
|
|
229
|
+
}
|
|
230
|
+
const startOf = (result) => (result.startTime instanceof Date ? result.startTime.getTime() : 0);
|
|
231
|
+
function annotationsOf(test, result, root) {
|
|
232
|
+
const seen = new Set();
|
|
233
|
+
const annotations = [];
|
|
234
|
+
const links = [];
|
|
235
|
+
const all = [...(test.annotations ?? []), ...((result?.annotations) ?? [])];
|
|
236
|
+
for (const annotation of all) {
|
|
237
|
+
const key = `${annotation.type}\u0000${annotation.description ?? ''}`;
|
|
238
|
+
if (seen.has(key))
|
|
239
|
+
continue;
|
|
240
|
+
seen.add(key);
|
|
241
|
+
annotations.push({
|
|
242
|
+
type: annotation.type,
|
|
243
|
+
message: annotation.description ?? '',
|
|
244
|
+
...(annotation.location ? { source: { file: repoPath(annotation.location.file, root), line: annotation.location.line, column: annotation.location.column } } : {})
|
|
245
|
+
});
|
|
246
|
+
if (annotation.description && /^https?:\/\//.test(annotation.description))
|
|
247
|
+
links.push({ url: annotation.description, rel: annotation.type });
|
|
248
|
+
}
|
|
249
|
+
return { annotations, types: [...new Set(all.map(annotation => annotation.type))], links };
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* The final check of a Playwright test from all its results (one per try): status of the last result (`flaky`, as a
|
|
253
|
+
* tag and a label, when it passed after a failed try), failure detail without terminal codes, duration, tags (the
|
|
254
|
+
* test's tags without `@`, and its annotation types), annotations, links (annotations whose description is a URL),
|
|
255
|
+
* timeline steps, one attempt per try when it was retried, attachments, and the test's location under the
|
|
256
|
+
* `playwright` extension. Reads attached files; never throws for a file it cannot read.
|
|
257
|
+
* @param test The Playwright test.
|
|
258
|
+
* @param results Its results, oldest first (`test.results`).
|
|
259
|
+
* @param context Its identity and what to record.
|
|
260
|
+
*/
|
|
261
|
+
export async function checkOf(test, results, context) {
|
|
262
|
+
const final = results.at(-1);
|
|
263
|
+
if (!final)
|
|
264
|
+
return { id: context.id, title: context.title, status: 'not-run', specPath: context.specPath, labels: { ...context.labels } };
|
|
265
|
+
const status = resultStatus(test, final);
|
|
266
|
+
const retried = results.length > 1;
|
|
267
|
+
const flaky = retried && status === 'passed';
|
|
268
|
+
const { annotations: noted, types, links } = annotationsOf(test, final, context.root);
|
|
269
|
+
const annotations = [...noted];
|
|
270
|
+
const timeline = flattenSteps(final.steps ?? [], startOf(final), context.maxSteps);
|
|
271
|
+
if (timeline.dropped)
|
|
272
|
+
annotations.push({ type: 'steps-truncated', level: 'info', message: `${timeline.dropped} more steps were not recorded (maxSteps ${context.maxSteps})` });
|
|
273
|
+
const files = await resultAttachments(final, status === 'failed', context.attachments);
|
|
274
|
+
annotations.push(...files.annotations);
|
|
275
|
+
const attempts = [];
|
|
276
|
+
if (retried) {
|
|
277
|
+
for (const result of results) {
|
|
278
|
+
const attemptStatus = resultStatus(test, result);
|
|
279
|
+
const last = result === final;
|
|
280
|
+
const evidence = last ? null : await resultAttachments(result, attemptStatus === 'failed', context.attachments);
|
|
281
|
+
const steps = last ? null : flattenSteps(result.steps ?? [], startOf(result), context.maxSteps).steps;
|
|
282
|
+
attempts.push({
|
|
283
|
+
index: result.retry,
|
|
284
|
+
status: attemptStatus,
|
|
285
|
+
...(Number.isFinite(result.duration) && result.duration >= 0 ? { durationMs: result.duration } : {}),
|
|
286
|
+
...(result.startTime instanceof Date ? { startedAtMs: Math.max(0, startOf(result)) } : {}),
|
|
287
|
+
...(result.error ? { detail: errorDetail(result.error) } : {}),
|
|
288
|
+
...(steps?.length ? { steps } : {}),
|
|
289
|
+
...(evidence?.attachments.length ? { attachments: evidence.attachments } : {}),
|
|
290
|
+
...(evidence?.annotations.length ? { annotations: evidence.annotations } : {})
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const firstFailure = results.find(result => result.error)?.error;
|
|
295
|
+
const skipReason = (test.annotations ?? []).find(annotation => annotation.type === 'skip' || annotation.type === 'fixme')?.description;
|
|
296
|
+
const detail = status === 'failed' ? (errorDetail(final.error ?? final.errors?.[0]) || `${final.status}`)
|
|
297
|
+
: flaky ? `Passed on retry ${final.retry} (flaky)${firstFailure ? `: ${errorDetail(firstFailure, 500).split('\n')[0]}` : ''}`
|
|
298
|
+
: status === 'not-run' ? (skipReason ? `Skipped: ${skipReason}` : 'Skipped')
|
|
299
|
+
: '';
|
|
300
|
+
const tags = [...new Set([...(test.tags ?? []).map(tag => tag.replace(/^@/, '')), ...types, ...(flaky ? ['flaky'] : [])])].filter(Boolean);
|
|
301
|
+
const labels = { ...context.labels, ...(retried ? { retries: String(results.length - 1) } : {}), ...(flaky ? { flaky: 'true' } : {}) };
|
|
302
|
+
return {
|
|
303
|
+
id: context.id,
|
|
304
|
+
title: context.title,
|
|
305
|
+
status,
|
|
306
|
+
detail,
|
|
307
|
+
specPath: context.specPath,
|
|
308
|
+
...(Number.isFinite(final.duration) && final.duration >= 0 ? { durationMs: final.duration } : {}),
|
|
309
|
+
...(tags.length ? { tags } : {}),
|
|
310
|
+
labels,
|
|
311
|
+
...(links.length ? { links } : {}),
|
|
312
|
+
...(annotations.length ? { annotations } : {}),
|
|
313
|
+
...(timeline.steps.length ? { steps: timeline.steps } : {}),
|
|
314
|
+
...(attempts.length ? { attempts } : {}),
|
|
315
|
+
...(files.attachments.length ? { attachments: files.attachments } : {}),
|
|
316
|
+
extensions: {
|
|
317
|
+
playwright: {
|
|
318
|
+
testId: test.id,
|
|
319
|
+
...(test.location ? { location: { file: repoPath(test.location.file, context.root), line: test.location.line, column: test.location.column } } : {}),
|
|
320
|
+
expectedStatus: test.expectedStatus,
|
|
321
|
+
status: final.status,
|
|
322
|
+
retries: test.retries,
|
|
323
|
+
...(Number.isInteger(final.workerIndex) ? { workerIndex: final.workerIndex } : {})
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
};
|
|
327
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import type { FullConfig, FullProject, Reporter as PlaywrightReporter, Suite, TestCase, TestResult } from 'playwright/types/testReporter';
|
|
2
|
+
import type { ObjectStore } from '../../reports/src/object-store.js';
|
|
3
|
+
import { type AttachmentMode } from './evidence.js';
|
|
4
|
+
export { bddFeaturePath, checkOf, errorDetail, findRepoRoot, flattenSteps, projectEnvironment, projectLabels, repoPath, resultAttachments, resultStatus, stepKind, stripAnsi, willRetry } from './evidence.js';
|
|
5
|
+
export type { AttachmentMode, AttachmentPolicy, CheckContext, CheckInput } from './evidence.js';
|
|
6
|
+
/** A value given once for every project, or worked out from each project. */
|
|
7
|
+
export type PerProject<T> = T | ((project: FullProject) => T);
|
|
8
|
+
/**
|
|
9
|
+
* Options of the Playwright Test reporter, given as the second element of its entry in `reporter` in
|
|
10
|
+
* `playwright.config.ts`. Every one is optional: with `ATLAS_SERVER_URL`, `ATLAS_TOKEN` and `ATLAS_PROJECT` set, none
|
|
11
|
+
* is needed.
|
|
12
|
+
*/
|
|
13
|
+
export type AtlasPlaywrightReporterOptions = {
|
|
14
|
+
/** Project namespace on a shared Atlas server. Default `ATLAS_PROJECT`. */
|
|
15
|
+
readonly project?: string;
|
|
16
|
+
/** The Atlas server (`https://atlas.example.com`), or any store location (`./atlas-reports`, `s3://bucket/prefix`). Default `ATLAS_SERVER_URL`, else `ATLAS_STORE`. */
|
|
17
|
+
readonly url?: string;
|
|
18
|
+
/** A publish token. Default `ATLAS_PUBLISH_TOKEN`, else `ATLAS_TOKEN` (or the file `ATLAS_TOKEN_FILE` names). */
|
|
19
|
+
readonly token?: string;
|
|
20
|
+
/** Publish to this store instead of a URL: a directory store, an in-memory store in tests, anything that implements `ObjectStore`. */
|
|
21
|
+
readonly store?: ObjectStore;
|
|
22
|
+
/** A stable name for this suite. With the check id, environment and parameters it picks the slot a result fills. Default `playwright`. */
|
|
23
|
+
readonly producer?: string;
|
|
24
|
+
/** The part kind of every layer. Default `checks`. */
|
|
25
|
+
readonly kind?: string;
|
|
26
|
+
/** The environment of a project's checks. Default `mobile-web` for a project with `use.isMobile`, else `desktop-web`. */
|
|
27
|
+
readonly environment?: PerProject<string>;
|
|
28
|
+
/** Choices that make a result its own slot (a locale, a tenant). Empty by default: labels never split slots, parameters do. */
|
|
29
|
+
readonly parameters?: PerProject<Readonly<Record<string, string>>>;
|
|
30
|
+
/** More labels for every check of a project, beside browser, device, viewport, os and playwright. */
|
|
31
|
+
readonly labels?: PerProject<Readonly<Record<string, string>>>;
|
|
32
|
+
/** The commit tested. Default `ATLAS_SHA`, then the CI's own variables (`GITHUB_SHA`…), then git. */
|
|
33
|
+
readonly sha?: string;
|
|
34
|
+
/** The branch tested. Default `ATLAS_BRANCH`, then `GITHUB_HEAD_REF`, `GITHUB_REF_NAME`…, then git. */
|
|
35
|
+
readonly branch?: string;
|
|
36
|
+
/** Where to read the CI run. Default the GitHub Actions run (or the CI's own run URL variable). */
|
|
37
|
+
readonly runUrl?: string;
|
|
38
|
+
/** The spec a test belongs to, relative to the repository. Default: the `.feature` of a playwright-bdd test, else the test file. */
|
|
39
|
+
readonly specPath?: (test: TestCase) => string | undefined;
|
|
40
|
+
/** Paths are relative to this directory. Default the git root above Playwright's `rootDir`, else `rootDir`. */
|
|
41
|
+
readonly rootDir?: string;
|
|
42
|
+
/** Which results upload their attachments (videos, screenshots, traces, output): `all`, `failures` or `none`. Default `all`. */
|
|
43
|
+
readonly attachments?: AttachmentMode;
|
|
44
|
+
/** A larger file is not uploaded; the check says so in an annotation. Default 50 MiB. */
|
|
45
|
+
readonly maxAttachmentBytes?: number;
|
|
46
|
+
/** The most timeline steps recorded per result. Default 500. */
|
|
47
|
+
readonly maxSteps?: number;
|
|
48
|
+
/** Record the project's spec files (path, domain, kind, title) in every layer, so a server can group checks by spec without a published spec set. Default true. */
|
|
49
|
+
readonly specs?: boolean;
|
|
50
|
+
/** Send what has finished this often, in ms. Default 5000 (every layer repeats the spec list). */
|
|
51
|
+
readonly flushMs?: number;
|
|
52
|
+
/** Send at once when this many checks are waiting. Default 50 (their attachments are held in memory until sent). */
|
|
53
|
+
readonly batchSize?: number;
|
|
54
|
+
/** Tries for each layer at the end of the run. Default 5. */
|
|
55
|
+
readonly attempts?: number;
|
|
56
|
+
/** Print nothing, not even the summary line. */
|
|
57
|
+
readonly quiet?: boolean;
|
|
58
|
+
/** Where messages go. Default: the summary to standard output, problems to standard error. */
|
|
59
|
+
readonly log?: (line: string) => void;
|
|
60
|
+
/** The environment variables to read. Default `process.env`. */
|
|
61
|
+
readonly env?: Readonly<Record<string, string | undefined>>;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* The Playwright Test reporter of Atlas. Add it beside your console reporter; it streams each test as `running` when
|
|
65
|
+
* it starts and with its full evidence when its last try ends, one stream of layers per Playwright project, and
|
|
66
|
+
* prints one summary line when the run ends. It never fails or slows the run on account of Atlas.
|
|
67
|
+
*
|
|
68
|
+
* @example
|
|
69
|
+
* ```ts
|
|
70
|
+
* // playwright.config.ts
|
|
71
|
+
* import { defineConfig } from '@playwright/test';
|
|
72
|
+
*
|
|
73
|
+
* export default defineConfig({
|
|
74
|
+
* reporter: [['list'], ['@crvouga/atlas/playwright-reporter', { project: 'checkout', producer: 'e2e-web' }]],
|
|
75
|
+
* use: { video: 'retain-on-failure', trace: 'retain-on-failure', screenshot: 'only-on-failure' }
|
|
76
|
+
* });
|
|
77
|
+
* ```
|
|
78
|
+
*/
|
|
79
|
+
export default class AtlasPlaywrightReporter implements PlaywrightReporter {
|
|
80
|
+
private readonly options;
|
|
81
|
+
private config;
|
|
82
|
+
private store;
|
|
83
|
+
private label;
|
|
84
|
+
private root;
|
|
85
|
+
private identity;
|
|
86
|
+
private readonly streams;
|
|
87
|
+
private readonly identities;
|
|
88
|
+
private readonly specPaths;
|
|
89
|
+
private readonly running;
|
|
90
|
+
private readonly pending;
|
|
91
|
+
private warned;
|
|
92
|
+
private recorded;
|
|
93
|
+
/** Playwright passes the options given in `playwright.config.ts`. */
|
|
94
|
+
constructor(options?: AtlasPlaywrightReporterOptions);
|
|
95
|
+
/** This reporter prints only a summary line, so Playwright keeps its own console output. */
|
|
96
|
+
printsToStdio(): boolean;
|
|
97
|
+
private say;
|
|
98
|
+
/** Said once per run: Atlas problems never flood the test output. */
|
|
99
|
+
private warn;
|
|
100
|
+
/** Resolves where layers go, where the run came from, and the directory paths are relative to. Never throws. */
|
|
101
|
+
onBegin(config: FullConfig, _suite: Suite): void;
|
|
102
|
+
private specPathOf;
|
|
103
|
+
private titleOf;
|
|
104
|
+
private specsOf;
|
|
105
|
+
private streamOf;
|
|
106
|
+
private identityOf;
|
|
107
|
+
private send;
|
|
108
|
+
/** Reports the test as `running`, so the report shows it while it runs. */
|
|
109
|
+
onTestBegin(test: TestCase, _result: TestResult): void;
|
|
110
|
+
/** On a test's last try, collects its evidence (reading its attached files) and reports it. A try Playwright will repeat leaves the test `running`. */
|
|
111
|
+
onTestEnd(test: TestCase, result: TestResult): void;
|
|
112
|
+
/** Waits for every result, marks tests the run did not finish as `not-run`, sends the rest, and prints one summary line. */
|
|
113
|
+
onEnd(): Promise<void>;
|
|
114
|
+
}
|