@crvouga/atlas 0.9.0 → 0.10.0

package/dist/playwright-reporter/src/index.js

@@ -0,0 +1,312 @@
+/**
+ * @module Playwright Test reporter
+ *
+ * `@crvouga/atlas/playwright-reporter` is a Playwright Test reporter that streams every test to an Atlas server (or any
+ * store) as it runs, with the evidence Playwright already has: durations, failure details, steps as a timeline, retries
+ * as attempts (and `flaky`), tags, annotations, and the videos, screenshots and traces it attached. Each Playwright
+ * project is its own stream of layers, so a desktop and a mobile project are two environments of the same checks, and
+ * every check is labelled with its browser, device, viewport, operating system and Playwright version.
+ *
+ * It never throws into the test run: a destination that is not configured, a store that is down or a file that cannot
+ * be read is said once, and the run goes on. Playwright is only imported for its types, so importing this module loads
+ * nothing of Playwright.
+ *
+ * @see docs/rich-evidence.md
+ */
+import path from 'node:path';
+import { createReporter } from '../../reports/src/reporter.js';
+import { detectFreshness, detectOrigin } from '../../reports/src/report-cli.js';
+import { openStore } from '../../reports/src/store-url.js';
+import { bddFeaturePath, checkOf, findRepoRoot, projectEnvironment, projectLabels, repoPath, willRetry } from './evidence.js';
+export { bddFeaturePath, checkOf, errorDetail, findRepoRoot, flattenSteps, projectEnvironment, projectLabels, repoPath, resultAttachments, resultStatus, stepKind, stripAnsi, willRetry } from './evidence.js';
+const MiB = 1024 * 1024;
+function perProject(value, project) {
+    if (typeof value !== 'function')
+        return value;
+    return project ? value(project) : undefined;
+}
+function projectSuiteOf(test) {
+    for (let suite = test.parent; suite; suite = suite.parent)
+        if (suite.type === 'project')
+            return suite;
+    return undefined;
+}
+/**
+ * The Playwright Test reporter of Atlas. Add it beside your console reporter; it streams each test as `running` when
+ * it starts and with its full evidence when its last try ends, one stream of layers per Playwright project, and
+ * prints one summary line when the run ends. It never fails or slows the run on account of Atlas.
+ *
+ * @example
+ * ```ts
+ * // playwright.config.ts
+ * import { defineConfig } from '@playwright/test';
+ *
+ * export default defineConfig({
+ *   reporter: [['list'], ['@crvouga/atlas/playwright-reporter', { project: 'checkout', producer: 'e2e-web' }]],
+ *   use: { video: 'retain-on-failure', trace: 'retain-on-failure', screenshot: 'only-on-failure' }
+ * });
+ * ```
+ */
+export default class AtlasPlaywrightReporter {
+    options;
+    config;
+    store = null;
+    label = '';
+    root = process.cwd();
+    identity = { sha: null, branch: null, runUrl: null };
+    streams = new Map();
+    identities = new WeakMap();
+    specPaths = new Map();
+    running = new Set();
+    pending = new Set();
+    warned = false;
+    recorded = 0;
+    /** Playwright passes the options given in `playwright.config.ts`. */
+    constructor(options = {}) {
+        this.options = options ?? {};
+    }
+    /** This reporter prints only a summary line, so Playwright keeps its own console output. */
+    printsToStdio() {
+        return false;
+    }
+    say(line) {
+        if (this.options.quiet)
+            return;
+        if (this.options.log)
+            this.options.log(line);
+        else
+            process.stdout.write(`${line}\n`);
+    }
+    /** Said once per run: Atlas problems never flood the test output. */
+    warn(message) {
+        if (this.warned || this.options.quiet)
+            return;
+        this.warned = true;
+        const line = `Atlas: ${message}`;
+        if (this.options.log)
+            this.options.log(line);
+        else
+            process.stderr.write(`${line}\n`);
+    }
+    /** Resolves where layers go, where the run came from, and the directory paths are relative to. Never throws. */
+    onBegin(config, _suite) {
+        this.config = config;
+        const options = this.options;
+        const env = options.env ?? process.env;
+        try {
+            const base = config?.rootDir ?? process.cwd();
+            this.root = options.rootDir ? path.resolve(base, options.rootDir) : (findRepoRoot(base) ?? base);
+        }
+        catch { /* keep the working directory */ }
+        try {
+            if (options.store) {
+                this.store = options.store;
+                this.label = 'the given store';
+            }
+            else {
+                const url = options.url ?? env.ATLAS_SERVER_URL ?? env.ATLAS_STORE;
+                if (!url) {
+                    this.warn('not publishing: set ATLAS_SERVER_URL (and ATLAS_TOKEN, ATLAS_PROJECT), or give the reporter { url, token, project }');
+                }
+                else {
+                    const token = options.token ?? env.ATLAS_PUBLISH_TOKEN;
+                    const project = options.project ?? env.ATLAS_PROJECT;
+                    const opened = openStore(url, { base: this.root, env: { ...env, ...(token ? { ATLAS_TOKEN: token } : {}), ...(project ? { ATLAS_PROJECT: project } : {}) } });
+                    this.store = opened.store;
+                    this.label = opened.label;
+                }
+            }
+        }
+        catch (error) {
+            this.store = null;
+            this.warn(`not publishing: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if (!this.store)
+            return;
+        try {
+            const explicitSha = options.sha ?? env.ATLAS_SHA;
+            const origin = detectOrigin(env);
+            const sha = explicitSha ?? origin.sha;
+            const freshness = detectFreshness({ sha, ci: origin.ci && !explicitSha });
+            this.identity = { sha, branch: options.branch ?? env.ATLAS_BRANCH ?? origin.branch, runUrl: options.runUrl ?? origin.runUrl, ...(freshness ? { freshness } : {}) };
+        }
+        catch {
+            this.identity = { sha: options.sha ?? env.ATLAS_SHA ?? null, branch: options.branch ?? env.ATLAS_BRANCH ?? null, runUrl: options.runUrl ?? null, freshness: 'unknown' };
+        }
+    }
+    specPathOf(test) {
+        try {
+            const given = this.options.specPath?.(test);
+            if (given)
+                return given.replaceAll('\\', '/');
+        }
+        catch { /* fall back to the file */ }
+        const file = test.location?.file;
+        if (!file)
+            return '';
+        let known = this.specPaths.get(file);
+        if (known === undefined) {
+            known = repoPath(bddFeaturePath(file) ?? file, this.root);
+            this.specPaths.set(file, known);
+        }
+        return known;
+    }
+    titleOf(test, specPath) {
+        // playwright-bdd names a Scenario Outline's examples `Example #1`; the outline's title says which scenario it is.
+        if (specPath.endsWith('.feature') && /^Example #\d+/.test(test.title) && test.parent?.type === 'describe' && test.parent.title)
+            return `${test.parent.title}: ${test.title}`;
+        return test.title;
+    }
+    specsOf(projectSuite) {
+        if (this.options.specs === false || !projectSuite)
+            return [];
+        const specs = new Map();
+        for (const test of projectSuite.allTests?.() ?? []) {
+            const specPath = this.specPathOf(test);
+            if (!specPath || specs.has(specPath))
+                continue;
+            let top = test.parent;
+            while (top?.parent && top.parent.type === 'describe')
+                top = top.parent;
+            const segments = specPath.split('/');
+            const file = segments.at(-1) ?? specPath;
+            const title = specPath.endsWith('.feature') && top?.type === 'describe' && top.title ? top.title : file;
+            specs.set(specPath, { path: specPath, domain: segments.length > 1 ? segments.at(-2) : 'Overview', kind: file.includes('.') ? file.slice(file.lastIndexOf('.') + 1) : 'document', title });
+        }
+        return [...specs.values()];
+    }
+    streamOf(projectSuite) {
+        if (this.streams.has(projectSuite))
+            return this.streams.get(projectSuite);
+        let stream = null;
+        const store = this.store;
+        if (store) {
+            try {
+                const options = this.options;
+                const project = projectSuite?.project?.();
+                const environment = perProject(options.environment, project) ?? projectEnvironment(project);
+                const parameters = perProject(options.parameters, project) ?? {};
+                const shard = this.config?.shard;
+                const labels = { ...projectLabels(project, this.config?.version), ...(shard ? { shard: `${shard.current}/${shard.total}` } : {}), ...(perProject(options.labels, project) ?? {}) };
+                const reporter = createReporter({
+                    producer: options.producer ?? 'playwright',
+                    kind: options.kind ?? 'checks',
+                    title: `Playwright${project?.name ? `: ${project.name}` : ''}`,
+                    sha: this.identity.sha,
+                    branch: this.identity.branch,
+                    runUrl: this.identity.runUrl,
+                    ...(this.identity.freshness ? { freshness: this.identity.freshness } : {}),
+                    environment,
+                    parameters: { ...parameters },
+                    specs: this.specsOf(projectSuite),
+                    labels
+                }, {
+                    store,
+                    flushMs: options.flushMs ?? 5000,
+                    batchSize: options.batchSize ?? 50,
+                    attempts: options.attempts ?? 5,
+                    onError: (error, willRetryLayer) => { if (!willRetryLayer)
+                        this.warn(`could not send results to ${this.label}: ${error.message}`); }
+                });
+                stream = { reporter, labels };
+            }
+            catch (error) {
+                this.warn(`not publishing project ${projectSuite?.title ?? ''}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        this.streams.set(projectSuite, stream);
+        return stream;
+    }
+    identityOf(test) {
+        const known = this.identities.get(test);
+        if (known)
+            return known;
+        const specPath = this.specPathOf(test);
+        const titles = [test.title];
+        for (let suite = test.parent; suite && suite.type === 'describe'; suite = suite.parent)
+            titles.unshift(suite.title);
+        const repeat = test.repeatEachIndex ? ` [repeat ${test.repeatEachIndex}]` : '';
+        const identity = { stream: this.streamOf(projectSuiteOf(test)), id: `${specPath || test.id}#${titles.join(' > ')}${repeat}`, title: this.titleOf(test, specPath), specPath };
+        this.identities.set(test, identity);
+        return identity;
+    }
+    send(stream, check) {
+        try {
+            stream.reporter.check(check);
+            if (check.status !== 'running')
+                this.recorded += 1;
+        }
+        catch (error) {
+            this.warn(`a result could not be recorded (${check.id}): ${error instanceof Error ? error.message.split('\n')[0] : String(error)}`);
+        }
+    }
+    /** Reports the test as `running`, so the report shows it while it runs. */
+    onTestBegin(test, _result) {
+        if (!this.store)
+            return;
+        try {
+            const { stream, id, title, specPath } = this.identityOf(test);
+            if (!stream)
+                return;
+            this.running.add(test);
+            this.send(stream, { id, title, status: 'running', ...(specPath ? { specPath } : {}), labels: { ...stream.labels } });
+        }
+        catch (error) {
+            this.warn(`a result could not be recorded: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /** On a test's last try, collects its evidence (reading its attached files) and reports it. A try Playwright will repeat leaves the test `running`. */
+    onTestEnd(test, result) {
+        if (!this.store)
+            return;
+        try {
+            if (willRetry(test, result))
+                return;
+            this.running.delete(test);
+            const { stream, id, title, specPath } = this.identityOf(test);
+            if (!stream)
+                return;
+            const options = this.options;
+            const results = test.results?.length ? test.results : [result];
+            const work = checkOf(test, results, {
+                id, title, specPath, labels: stream.labels, root: this.root,
+                attachments: { mode: options.attachments ?? 'all', maxBytes: options.maxAttachmentBytes ?? 50 * MiB },
+                maxSteps: options.maxSteps ?? 500
+            }).then(check => this.send(stream, check), error => {
+                this.warn(`evidence of ${id} could not be collected: ${error instanceof Error ? error.message : String(error)}`);
+                this.send(stream, { id, title, status: result.status === 'skipped' ? 'not-run' : result.status === test.expectedStatus ? 'passed' : 'failed', ...(specPath ? { specPath } : {}), labels: { ...stream.labels } });
+            });
+            this.pending.add(work);
+            void work.finally(() => this.pending.delete(work));
+        }
+        catch (error) {
+            this.warn(`a result could not be recorded: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /** Waits for every result, marks tests the run did not finish as `not-run`, sends the rest, and prints one summary line. */
+    async onEnd() {
+        if (!this.store)
+            return;
+        try {
+            await Promise.allSettled([...this.pending]);
+            for (const test of this.running) {
+                const { stream, id, title, specPath } = this.identityOf(test);
+                if (stream)
+                    this.send(stream, { id, title, status: 'not-run', detail: 'The run ended before this test finished', ...(specPath ? { specPath } : {}), labels: { ...stream.labels } });
+            }
+            this.running.clear();
+            let layers = 0, lost = 0;
+            for (const stream of this.streams.values()) {
+                if (!stream)
+                    continue;
+                const result = await stream.reporter.close();
+                layers += result.layers;
+                lost += result.lost;
+            }
+            this.say(`Atlas: ${this.recorded} checks in ${layers} layer${layers === 1 ? '' : 's'} to ${this.label}${lost ? `; ${lost} could not be sent` : ''}`);
+        }
+        catch (error) {
+            this.warn(`could not finish publishing: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+}

package/dist/reports/src/compose.js

@@ -11,7 +11,8 @@
  */
 import { eachLimited } from './completion-order.js';
 import { catalogEntries, catalogLayers } from './layer-catalog.js';
-import { combineEvidence, distinctHeaders, emptyCounts, isDiagnosticKind, layerEvidence, specPath } from './layer-merge.js';
+import { combineEvidence, distinctHeaders, emptyCounts, isDiagnosticKind, layerEvidence } from './layer-merge.js';
+import { RecordedSpecs } from './recorded-specs.js';
 const BACKEND_ID = /^[a-z0-9-]+$/;
 /**
  * Counts results by status. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
@@ -98,19 +99,11 @@ export async function combineBackends(backends, options = {}) {
     const kept = include ? loaded.filter(item => include(item.evidence.layer)) : loaded;
     const layers = kept.map(item => item.evidence);
     // The specs a report is laid over come from the layers that are in it, not from ones left out.
-    const recorded = new Map();
-    if (!options.outline) {
-        for (const { evidence, specs } of kept) {
-            for (const spec of specs) {
-                const key = specPath(spec.path);
-                const prior = recorded.get(key);
-                const at = evidence.layer.recordedAtMs;
-                if (!prior || at > prior.at || (at === prior.at && (spec.fingerprint ?? '') > (prior.entry.fingerprint ?? '')))
-                    recorded.set(key, { entry: { ...spec, path: key }, at });
-            }
-        }
-    }
-    const outline = options.outline ?? { specs: [...recorded.values()].map(item => item.entry).sort((a, b) => (a.path < b.path ? -1 : 1)), sha: null, changedFiles: [] };
+    const recorded = new RecordedSpecs();
+    if (!options.outline)
+        for (const { evidence, specs } of kept)
+            recorded.add(specs, evidence.layer.recordedAtMs);
+    const outline = options.outline ?? { specs: recorded.entries(), sha: null, changedFiles: [] };
     const combined = combineEvidence(outline, layers);
     const headers = distinctHeaders(layers.map(evidence => evidence.layer)).sort((a, b) => b.recordedAtMs - a.recordedAtMs || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
     return { combined, layers: headers, backends: info.sort((a, b) => (a.id < b.id ? -1 : 1)), notices: [...new Set(notices)].sort(), summary: summarize(combined) };

package/dist/reports/src/recorded-specs.d.ts

@@ -0,0 +1,33 @@
+/**
+ * @module Specs the layers recorded
+ *
+ * When no spec set was published, a report is laid over the specs its layers say they ran against (each layer's
+ * `specs[]`). `RecordedSpecs` keeps one entry per spec path: the one from the newest layer, and at the same time the
+ * one with the larger fingerprint, so the outline it gives does not depend on the order layers are read in. The CLI's
+ * composition and the server's live report use it alike, so the two cannot disagree about which specs a report has.
+ *
+ * @see docs/report-contract.md#how-layers-compose
+ */
+import type { SpecEntry } from './explorer-types.js';
+/**
+ * The specs layers recorded, newest first per path. Add every included layer's `specs[]` with its `recordedAtMs`;
+ * `entries()` is the outline, sorted by path.
+ *
+ * @example
+ * ```ts
+ * const recorded = new RecordedSpecs();
+ * for (const layer of layers) recorded.add(layer.specs, layer.recordedAtMs);
+ * const outline = { specs: recorded.entries(), sha: null, changedFiles: [] };
+ * ```
+ */
+export declare class RecordedSpecs {
+    #private;
+    /** Takes in the specs one layer recorded at `at` (epoch ms). A spec whose path is empty is ignored. */
+    add(specs: readonly SpecEntry[], at: number): void;
+    /** The recorded spec at a path (as `specPath` normalises it), or undefined. */
+    get(path: string): SpecEntry | undefined;
+    /** How many distinct specs were recorded. */
+    get size(): number;
+    /** Every recorded spec, sorted by path. */
+    entries(): readonly SpecEntry[];
+}

package/dist/reports/src/recorded-specs.js

@@ -0,0 +1,42 @@
+import { specPath } from './layer-merge.js';
+/**
+ * The specs layers recorded, newest first per path. Add every included layer's `specs[]` with its `recordedAtMs`;
+ * `entries()` is the outline, sorted by path.
+ *
+ * @example
+ * ```ts
+ * const recorded = new RecordedSpecs();
+ * for (const layer of layers) recorded.add(layer.specs, layer.recordedAtMs);
+ * const outline = { specs: recorded.entries(), sha: null, changedFiles: [] };
+ * ```
+ */
+export class RecordedSpecs {
+    #byPath = new Map();
+    #sorted = null;
+    /** Takes in the specs one layer recorded at `at` (epoch ms). A spec whose path is empty is ignored. */
+    add(specs, at) {
+        for (const spec of specs) {
+            if (!spec.path)
+                continue;
+            const key = specPath(spec.path);
+            const prior = this.#byPath.get(key);
+            if (!prior || at > prior.at || (at === prior.at && (spec.fingerprint ?? '') > (prior.entry.fingerprint ?? ''))) {
+                this.#byPath.set(key, { entry: { ...spec, path: key }, at });
+                this.#sorted = null;
+            }
+        }
+    }
+    /** The recorded spec at a path (as `specPath` normalises it), or undefined. */
+    get(path) {
+        return this.#byPath.get(specPath(path))?.entry;
+    }
+    /** How many distinct specs were recorded. */
+    get size() {
+        return this.#byPath.size;
+    }
+    /** Every recorded spec, sorted by path. */
+    entries() {
+        this.#sorted ??= [...this.#byPath.values()].map(item => item.entry).sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+        return this.#sorted;
+    }
+}

package/dist/reports/src/report-index.d.ts

@@ -14,6 +14,7 @@
 import type { BackendInfo, LayerEvidence, LayerHeader } from './evidence-types.js';
 import type { BackendInput } from './compose.js';
 import type { ReportLayer } from './layer-types.js';
+import { TestHistory } from './test-catalog.js';
 /** What a viewer needs to show the report, and nothing that is only history. */
 export type EffectiveEvidence = {
     /** A hash of the report itself: the same on every replica, and different exactly when the report is. Use it as an ETag. */
@@ -87,6 +88,11 @@ export declare class ReportIndex {
         readonly text: string;
         readonly version: string;
     } | null;
+    /**
+     * Every recorded result of every test and every run, as layers were folded in: what the test catalog, a test's history
+     * and the run list are built from. Read-only for callers; `history.rev` changes whenever a layer is taken in.
+     */
+    get history(): TestHistory;
     /** For `/readyz` and logs. */
     stats(): {
         backends: number;

package/dist/reports/src/report-index.js

@@ -17,6 +17,7 @@ import { LAYER_INDEX, layerSummary, readLayer } from './layer-store.js';
 import { layerEvidence, layerResultKeys, replaces } from './layer-merge.js';
 import { readUtf8 } from './object-io.js';
 import { isPaged } from './object-store.js';
+import { TestHistory } from './test-catalog.js';
 const RETRY_FAILED_MS = 5 * 60_000;
 /** Versions a viewer may still be holding, remembered so it can be sent only what changed. */
 const REMEMBERED_VERSIONS = 256;
@@ -34,6 +35,8 @@ export class ReportIndex {
     #concurrency;
     #now;
     #winners = new Map();
+    /** Every result of every test (bounded per test) and every run, for the test catalog. It never decides a winner. */
+    #history = new TestHistory();
     #held = new Map();
     /** The smallest backend id a layer was found in, so attribution does not depend on read order. */
     #attribution = new Map();
@@ -195,7 +198,8 @@ export class ReportIndex {
             }
             if (!id || !layer)
                 throw new Error('an indexed layer is missing');
-            this.#fold(layerEvidence(layerSummary(layer), backend.id), backend.id);
+            const summary = layerSummary(layer);
+            this.#fold(layerEvidence(summary, backend.id), backend.id, summary.specs);
             this.#folded.add(mark);
             this.#failed.delete(mark);
             this.#refreshedAt ||= this.#now();
@@ -208,7 +212,7 @@ export class ReportIndex {
         const failure = this.#failed.get(mark);
         return failure !== undefined && this.#now() - failure.at < RETRY_FAILED_MS;
     }
-    #fold(evidence, backend) {
+    #fold(evidence, backend, specs = []) {
         const id = evidence.id;
         const known = this.#attribution.get(id);
         if (known !== undefined) {
@@ -227,6 +231,7 @@ export class ReportIndex {
         if (!this.#include(evidence.layer))
             return;
         this.#added.push({ ...evidence.layer, backend });
+        this.#history.add(evidence.layer, evidence.rows, specs, backend);
         const wins = new Set();
         const keyOf = layerResultKeys(evidence.layer);
         for (const row of evidence.rows) {
@@ -330,6 +335,13 @@ export class ReportIndex {
             text: `{"version":${JSON.stringify(value.version)},"base":${JSON.stringify(since)},"delta":true,"layers":[${changed.map(item => (item.json ??= JSON.stringify(this.#entryOf(item)))).join(',')}],"removed":${JSON.stringify(gone)},"backends":${JSON.stringify(value.backends)},"notices":${JSON.stringify(value.notices)},"totalLayers":${value.totalLayers},"active":[]}`
         };
     }
+    /**
+     * Every recorded result of every test and every run, as layers were folded in: what the test catalog, a test's history
+     * and the run list are built from. Read-only for callers; `history.rev` changes whenever a layer is taken in.
+     */
+    get history() {
+        return this.#history;
+    }
     /** For `/readyz` and logs. */
     stats() {
         return { backends: this.#backends.length, layers: this.#attribution.size, slots: this.#winners.size, holding: this.#held.size, failed: this.#failed.size, refreshedAt: this.#refreshedAt };