@crvouga/atlas 0.16.0 → 0.17.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog: @crvouga/atlas
 
+## 0.17.0 (2026-10-28)
+
+### Features
+
+- report recording completeness separately from test success ([a8dd9d4](https://github.com/crvouga/atlas/commit/a8dd9d4b12d0c9cfbb7cb94c48d571e4aa5050ef))
+
 ## 0.16.0 (2026-10-28)
 
 ### Features

package/dist/reports/src/attachment-files.d.ts

@@ -34,7 +34,9 @@ export type ReadOptions = {
  * attachment gave none.
  *
  * A file that is missing, not allowed or too large is left out with a warning: evidence that could not be
- * found never stops the results from being published. Attachments without a `path` are kept untouched.
+ * found never stops the results from being published. The exception in kind is a video of a check that declared
+ * `recordingExpectation: "required"`: it stays as an attachment without bytes whose `omitted` says why (`not-found`,
+ * `not-allowed`, `too-large`, `over-budget`), so the recording receipt can name the gap. Attachments without a `path` are kept untouched.
  *
  * @param holder A check, step, attempt or any object that may have `attachments`, `steps` and `attempts`.
  * @param roots Folders a relative path is resolved against (in order), and the only folders an untrusted path may name.

package/dist/reports/src/attachment-files.js

@@ -11,13 +11,16 @@
 import { readFile, realpath, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { contentTypeOf } from './media-types.js';
+import { isVideoAttachment } from './recording-receipt.js';
 /**
  * Reads the files that attachments name (`path`), at every level (the holder, its `steps` and its `attempts`,
  * recursively), and hands their bytes over as `data` with a `contentType` guessed from the file name when the
  * attachment gave none.
  *
  * A file that is missing, not allowed or too large is left out with a warning: evidence that could not be
- * found never stops the results from being published. Attachments without a `path` are kept untouched.
+ * found never stops the results from being published. The exception in kind is a video of a check that declared
+ * `recordingExpectation: "required"`: it stays as an attachment without bytes whose `omitted` says why (`not-found`,
+ * `not-allowed`, `too-large`, `over-budget`), so the recording receipt can name the gap. Attachments without a `path` are kept untouched.
  *
  * @param holder A check, step, attempt or any object that may have `attachments`, `steps` and `attempts`.
  * @param roots Folders a relative path is resolved against (in order), and the only folders an untrusted path may name.
@@ -27,7 +30,7 @@ export async function readAttachmentFiles(holder, roots, options = {}) {
     const warnings = [];
     const maxBytes = options.maxBytes ?? 512 * 1024 * 1024;
     let attached = 0, bytes = 0;
-    async function walk(value) {
+    async function walk(value, expectsRecording = value.recordingExpectation === 'required') {
         const out = { ...value };
         if (Array.isArray(out.attachments)) {
             const kept = [];
@@ -40,17 +43,24 @@ export async function readAttachmentFiles(holder, roots, options = {}) {
                 const name = String(given.name ?? file);
                 const { path: _path, ...rest } = given;
                 const found = await locate(file, roots, options.trusted === true);
+                // A video that a check was expected to have stays in the layer as metadata with the reason it was left out
+                // (never the path), so the recording receipt can name the gap instead of the file vanishing.
+                const left = (reason) => { if (expectsRecording && isVideoAttachment({ ...rest, contentType: rest.contentType ?? contentTypeOf(name) }))
+                    kept.push({ ...rest, omitted: reason }); };
                 if (found.problem) {
                     warnings.push(`${name}: ${found.problem}`);
+                    left(found.problem.startsWith('file not found') ? 'not-found' : 'not-allowed');
                     continue;
                 }
                 const size = (await stat(found.file)).size;
                 if (size > maxBytes) {
                     warnings.push(`${name}: ${(size / 1024 / 1024).toFixed(0)} MiB is over the limit of ${Math.round(maxBytes / 1024 / 1024)} MiB`);
+                    left('too-large');
                     continue;
                 }
                 if (options.budget && size > options.budget.remaining) {
                     warnings.push(`${name}: the total size of attachments would pass the budget (raise --max-total-attachment-mb, or attach a uri)`);
+                    left('over-budget');
                     continue;
                 }
                 if (options.budget)
@@ -64,13 +74,13 @@ export async function readAttachmentFiles(holder, roots, options = {}) {
         if (Array.isArray(out.steps)) {
             const steps = [];
             for (const step of out.steps)
-                steps.push(await walk(step));
+                steps.push(await walk(step, expectsRecording));
             out.steps = steps;
         }
         if (Array.isArray(out.attempts)) {
             const attempts = [];
             for (const attempt of out.attempts)
-                attempts.push(await walk(attempt));
+                attempts.push(await walk(attempt, expectsRecording));
             out.attempts = attempts;
         }
         return out;

package/dist/reports/src/evidence-types.d.ts

@@ -123,6 +123,23 @@ export type LayerHeader = {
     readonly execution: string;
     /** What the run as a whole carries: its logs, links and labels. */
     readonly evidence?: EvidenceSummary;
+    /** Counts from the layer's recording receipt: how many expected recordings were published. Absent when the layer declared no expectations: its completeness is unknown, not complete. Never changes a status. */
+    readonly recordings?: RecordingCounts;
+};
+/** How many recordings a layer expected and what became of them (see `RecordingReceipt`). */
+export type RecordingCounts = {
+    /** Checks that required a recording: `uploaded + referenced + skipped + unresolved`. */
+    readonly expected: number;
+    /** Stored in the layer, with a checksum. */
+    readonly uploaded: number;
+    /** Given only by `uri`: Atlas cannot vouch for it. */
+    readonly referenced: number;
+    /** Captured but left out, with a reason (too large, not allowed…). */
+    readonly skipped: number;
+    /** No video was attached at all. */
+    readonly unresolved: number;
+    /** Intentionally unrecorded checks: neither expected nor missing. */
+    readonly notApplicable: number;
 };
 /** One check of one report part. */
 export type LayerRow = {
@@ -145,6 +162,11 @@ export type LayerRow = {
     readonly durationMs: number | null;
     /** What this result carries beyond its outcome. Absent when nothing. */
     readonly evidence?: EvidenceSummary;
+    /** What became of this check's expected recording: `uploaded`, `referenced`, `skipped`, `unresolved` or `not-applicable`, with a reason when it is not uploaded. Absent when the check declared no expectation. Independent of `status`. */
+    readonly recording?: {
+        readonly state: string;
+        readonly reason?: string;
+    };
 };
 /** The checks of one layer, read separately from its header. */
 export type LayerRows = {

package/dist/reports/src/layer-merge.js

@@ -74,10 +74,13 @@ export function layerEvidence(summary, backend) {
         return found;
     };
     const rows = [];
+    const receipt = summary.recordings;
+    const expectations = receipt ? new Map(receipt.items.map(item => [item.check, item])) : null;
     for (const part of summary.parts) {
         for (const check of part.checks) {
             const test = tests.get(check.id);
             const evidence = summarizeEvidence(check);
+            const item = expectations?.get(check.id);
             rows.push({
                 kind: part.kind,
                 id: check.id,
@@ -90,6 +93,7 @@ export function layerEvidence(summary, backend) {
                 variant: test ? { system: test.system, runtime: test.runtime, seed: test.seed } : {},
                 durationMs: test?.durationMs ?? check.durationMs ?? null,
                 ...(evidence ? { evidence } : {}),
+                ...(item ? { recording: { state: item.state, ...(item.reason ? { reason: item.reason } : {}) } } : {}),
             });
         }
     }
@@ -113,6 +117,7 @@ export function layerEvidence(summary, backend) {
         counts: outcomeCounts(rows),
         execution: textFingerprint(layerExecutionKey(summary)),
         ...(layerSummaryEvidence ? { evidence: layerSummaryEvidence } : {}),
+        ...(receipt ? { recordings: { expected: receipt.expected, uploaded: receipt.uploaded, referenced: receipt.referenced, skipped: receipt.skipped, unresolved: receipt.unresolved, notApplicable: receipt.notApplicable } } : {}),
     };
     const fingerprints = {};
     for (const spec of summary.specs)

package/dist/reports/src/layer-types.d.ts

@@ -15,7 +15,7 @@ import type { SourceRecord } from './protocol.js';
 /** String choices (such as `{ device: 'iphone-15' }`) that make a result its own slot. Compared as sorted key/value pairs. */
 export type LayerParameters = Readonly<Record<string, string>>;
 /** What a layer may carry as a whole. All optional and open: see docs/extending.md. */
-export type LayerExtras = Partial<Pick<import('../../schema/src/layer.js').LayerDocument, 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions'>>;
+export type LayerExtras = Partial<Pick<import('../../schema/src/layer.js').LayerDocument, 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions' | 'recordings'>>;
 /** A layer without its captured files: what composition and the viewer's lists read. */
 export type LayerSummary = LayerExtras & {
     /** `modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout. Advisory; never changes a status. */

package/dist/reports/src/publish-checks.js

@@ -15,6 +15,7 @@ import { normalizeEnvironment } from './layer-axes.js';
 import { publishLayerBundle } from './layer-store.js';
 import { contentTypeOf } from './media-types.js';
 import { parseWith } from './parse-with.js';
+import { isVideoAttachment, recordingReceipt } from './recording-receipt.js';
 import { sha256Hex } from './sigv4.js';
 /** A small text value stays in the layer's own document; anything longer is stored as a file. */
 const INLINE_TEXT_BYTES = 8 * 1024;
@@ -28,7 +29,7 @@ function decodeBase64(text) {
  * screenshot attached ten times is stored once), referred to by `bundleKey`. What a producer wrote as `data`,
  * `base64` or a long `text` becomes a reference; a short `text` stays inline; a `uri` is left alone.
  */
-async function materialize(holder, entries, limit) {
+async function materialize(holder, entries, limit, expectsRecording = holder.recordingExpectation === 'required') {
     const out = { ...holder };
     if (Array.isArray(out.attachments)) {
         out.attachments = await Promise.all(out.attachments.map(async (given) => {
@@ -41,6 +42,11 @@ async function materialize(holder, entries, limit) {
                 bytes = encoder.encode(text);
             if (!bytes)
                 return { ...kept, ...(text !== undefined ? { bytes: encoder.encode(text).length } : {}) };
+            if (bytes.length > limit && expectsRecording && isVideoAttachment({ ...kept, ...(kept.contentType ? {} : { contentType: contentTypeOf(String(kept.name)) }) })) {
+                // A recording the check was expected to have is reported as left out (and why) instead of failing the whole publish.
+                const { text: _inline, ...rest } = kept;
+                return { ...rest, contentType: kept.contentType ?? contentTypeOf(String(kept.name)), bytes: bytes.length, sha256: await sha256Hex(bytes), omitted: 'too-large' };
+            }
             if (bytes.length > limit)
                 throw new Error(`publishChecks: attachment "${String(kept.name)}" is ${bytes.length} bytes, over the limit of ${limit}. Store it elsewhere and attach its uri`);
             const sha256 = await sha256Hex(bytes);
@@ -51,9 +57,9 @@ async function materialize(holder, entries, limit) {
         }));
     }
     if (Array.isArray(out.steps))
-        out.steps = await Promise.all(out.steps.map(step => materialize(step, entries, limit)));
+        out.steps = await Promise.all(out.steps.map(step => materialize(step, entries, limit, expectsRecording)));
     if (Array.isArray(out.attempts))
-        out.attempts = await Promise.all(out.attempts.map(attempt => materialize(attempt, entries, limit)));
+        out.attempts = await Promise.all(out.attempts.map(attempt => materialize(attempt, entries, limit, expectsRecording)));
     return out;
 }
 /**
@@ -81,6 +87,7 @@ export async function publishChecks(store, input, options = {}) {
     const entries = new Map();
     const checks = await Promise.all(args.checks.map(check => materialize(check, entries, limit)));
     const layerEvidence = await materialize({ attachments: args.attachments }, entries, limit);
+    const recordings = recordingReceipt(checks);
     const total = [...entries.values()].reduce((sum, entry) => sum + entry.bytes.length, 0);
     const totalLimit = options.maxTotalAttachmentBytes ?? 2 * 1024 * 1024 * 1024;
     if (total > totalLimit)
@@ -110,6 +117,7 @@ export async function publishChecks(store, input, options = {}) {
         parts: [part],
         source: null,
         recording: null,
+        ...(recordings ? { recordings } : {}),
         ...(args.tags?.length ? { tags: args.tags } : {}),
         ...(args.labels && Object.keys(args.labels).length ? { labels: Object.fromEntries(Object.entries(args.labels).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) } : {}),
         ...(args.links?.length ? { links: args.links } : {}),

package/dist/reports/src/recording-receipt.d.ts

@@ -0,0 +1,48 @@
+/**
+ * @module Recording receipts
+ *
+ * Answers "did the recordings we expected get published?" apart from "did the tests pass?". `recordingReceipt` reads the
+ * checks a producer is about to publish (after their files were stored) and describes, per check that declared a
+ * `recordingExpectation`, whether its video was uploaded, only referenced, left out (with a reason) or never attached.
+ * `recordingCompleteness` reduces the winning results of a report to the figures `atlas report status` prints. Neither
+ * reads a clock or the environment, and neither changes a status, a slot or a winner: a receipt describes publication, it
+ * does not decide a result. A layer with no receipt is Unknown, never Complete.
+ *
+ * @see docs/rich-evidence.md#recording-completeness
+ */
+import type { RecordingCompleteness, RecordingReceipt } from '../../schema/src/layer.js';
+type Loose = Record<string, unknown>;
+/** Is this attachment a screen recording: a role of `video`, or a `video/*` media type? */
+export declare function isVideoAttachment(attachment: Loose): boolean;
+/**
+ * The receipt for the checks of one layer, or `undefined` when no check declares a `recordingExpectation` (the layer then
+ * claims nothing). Call it with checks whose attachment bytes are already stored (`bundleKey`, `bytes` and `sha256` set).
+ *
+ * A `required` check is `uploaded` when any video of it has stored bytes, `referenced` when one has only a `uri`,
+ * `skipped` when its videos were left out (the attachment's `omitted` is the reason) and `unresolved` when it has no
+ * video at all. A `not-applicable` check is listed but is neither expected nor missing. Items are sorted by check id so the
+ * same input is always the same receipt.
+ */
+export declare function recordingReceipt(checks: readonly Loose[]): RecordingReceipt | undefined;
+/** One result as {@link recordingCompleteness} reads it. */
+export type RecordingResult = {
+    readonly spec: string | null;
+    readonly check: string;
+    readonly environment: string | null;
+    readonly layer: string;
+    /** Whether the layer the result came from carried a receipt. */
+    readonly hasReceipt: boolean;
+    /** The result's item in that receipt, when its check declared an expectation. */
+    readonly recording?: {
+        readonly state: string;
+        readonly reason?: string;
+    };
+};
+/**
+ * Reduces the winning results of a report to its recording completeness. `incomplete` when any expected recording is
+ * skipped or unresolved; otherwise `unknown` when a contributing layer has no receipt (nothing is claimed for it, and a
+ * report with no results at all is unknown too); otherwise `complete` when something was expected and `none-expected` when
+ * nothing was. Gaps are sorted so the output never depends on read order.
+ */
+export declare function recordingCompleteness(results: readonly RecordingResult[]): RecordingCompleteness;
+export {};

package/dist/reports/src/recording-receipt.js

@@ -0,0 +1,103 @@
+import { contentTypeOf } from './media-types.js';
+/** Is this attachment a screen recording: a role of `video`, or a `video/*` media type? */
+export function isVideoAttachment(attachment) {
+    if (attachment.role === 'video')
+        return true;
+    const type = typeof attachment.contentType === 'string' ? attachment.contentType : typeof attachment.name === 'string' ? contentTypeOf(attachment.name) : '';
+    return type.startsWith('video/');
+}
+/** Every video attachment of a holder, its steps and its attempts, at any depth. */
+function videosOf(holder) {
+    const found = [];
+    const take = (list) => { if (Array.isArray(list))
+        for (const item of list)
+            if (isVideoAttachment(item))
+                found.push(item); };
+    take(holder.attachments);
+    for (const step of holder.steps ?? [])
+        take(step.attachments);
+    for (const attempt of holder.attempts ?? []) {
+        take(attempt.attachments);
+        for (const step of attempt.steps ?? [])
+            take(step.attachments);
+    }
+    return found;
+}
+/**
+ * The receipt for the checks of one layer, or `undefined` when no check declares a `recordingExpectation` (the layer then
+ * claims nothing). Call it with checks whose attachment bytes are already stored (`bundleKey`, `bytes` and `sha256` set).
+ *
+ * A `required` check is `uploaded` when any video of it has stored bytes, `referenced` when one has only a `uri`,
+ * `skipped` when its videos were left out (the attachment's `omitted` is the reason) and `unresolved` when it has no
+ * video at all. A `not-applicable` check is listed but is neither expected nor missing. Items are sorted by check id so the
+ * same input is always the same receipt.
+ */
+export function recordingReceipt(checks) {
+    const items = [];
+    const count = { uploaded: 0, referenced: 0, skipped: 0, unresolved: 0, notApplicable: 0 };
+    for (const check of checks) {
+        const expectation = check.recordingExpectation;
+        if (expectation !== 'required' && expectation !== 'not-applicable')
+            continue;
+        const check_ = String(check.id);
+        if (expectation === 'not-applicable') {
+            count.notApplicable += 1;
+            items.push({ check: check_, state: 'not-applicable' });
+            continue;
+        }
+        const files = videosOf(check).map(video => {
+            const state = typeof video.bundleKey === 'string' ? 'uploaded' : typeof video.uri === 'string' ? 'referenced' : 'skipped';
+            return {
+                name: String(video.name),
+                state,
+                ...(typeof video.bytes === 'number' ? { bytes: video.bytes } : {}),
+                ...(typeof video.sha256 === 'string' ? { sha256: video.sha256 } : {}),
+                ...(state === 'skipped' ? { reason: typeof video.omitted === 'string' ? video.omitted : 'bytes-not-attached' } : {})
+            };
+        });
+        const first = (state) => files.find(file => file.state === state);
+        const state = first('uploaded') ? 'uploaded' : first('referenced') ? 'referenced' : first('skipped') ? 'skipped' : 'unresolved';
+        const reason = state === 'skipped' ? first('skipped')?.reason : state === 'unresolved' ? 'no-video-attached' : undefined;
+        count[state] += 1;
+        items.push({ check: check_, state, ...(reason ? { reason } : {}), ...(files.length ? { files } : {}) });
+    }
+    if (!items.length)
+        return undefined;
+    items.sort((a, b) => (a.check < b.check ? -1 : a.check > b.check ? 1 : 0));
+    return { expected: count.uploaded + count.referenced + count.skipped + count.unresolved, ...count, items };
+}
+/**
+ * Reduces the winning results of a report to its recording completeness. `incomplete` when any expected recording is
+ * skipped or unresolved; otherwise `unknown` when a contributing layer has no receipt (nothing is claimed for it, and a
+ * report with no results at all is unknown too); otherwise `complete` when something was expected and `none-expected` when
+ * nothing was. Gaps are sorted so the output never depends on read order.
+ */
+export function recordingCompleteness(results) {
+    const count = { expected: 0, uploaded: 0, referenced: 0, skipped: 0, unresolved: 0, notApplicable: 0 };
+    const gaps = [];
+    const without = new Set();
+    for (const result of results) {
+        if (!result.hasReceipt) {
+            without.add(result.layer);
+            continue;
+        }
+        const state = result.recording?.state;
+        if (state === undefined)
+            continue;
+        if (state === 'not-applicable') {
+            count.notApplicable += 1;
+            continue;
+        }
+        count.expected += 1;
+        if (state === 'uploaded' || state === 'referenced' || state === 'skipped' || state === 'unresolved')
+            count[state] += 1;
+        // A state this reader does not know is a gap: it is never counted as uploaded.
+        if (state !== 'uploaded' && state !== 'referenced') {
+            gaps.push({ spec: result.spec, check: result.check, layer: result.layer, environment: result.environment, state, ...(result.recording?.reason ? { reason: result.recording.reason } : {}) });
+        }
+    }
+    const text = (value) => value ?? '';
+    gaps.sort((a, b) => (text(a.spec) + a.check + text(a.environment) + a.layer < text(b.spec) + b.check + text(b.environment) + b.layer ? -1 : 1));
+    const state = gaps.length ? 'incomplete' : without.size || !results.length ? 'unknown' : count.expected ? 'complete' : 'none-expected';
+    return { state, ...count, layersWithoutReceipt: without.size, gaps };
+}

package/dist/reports/src/report-cli.d.ts

@@ -13,7 +13,7 @@
  * @see docs/agents.md
  */
 /** The text `atlas report --help` prints: every subcommand, its flags, what each does and the exit codes. Also the source of the command reference in `docs/reference/cli.md`. */
-export declare const REPORT_USAGE = "atlas report \u2014 layered evidence for any kind of spec\n\n  atlas report publish [--store <dir|s3://bucket/prefix>] [--producer <name>]\n                       ( --ctrf <file> | --junit <file> | --manifest <file> | --checks <file.json> )\n                       [--environment <name>] [--branch <b>] [--sha <s>] [--run-url <u>]\n                       [--run-id <id>] [--run-title <t>] [--run-started-at <iso|ms>] [--run-ended-at <iso|ms>]\n                       [--spec <path> | --specs <dir>] [--upload-run]\n                       [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]\n                       [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...\n                       [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]\n  atlas report stream  [--store <dir|s3://\u2026|https://atlas.example.com>] --producer <name> [--environment <name>]\n                       [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...\n                       [--run-id <id>] [--run-title <t>]\n                       [--flush-ms 1000] [--batch 100]       (checks as JSON lines on standard input)\n  atlas report status  [--store <dir|s3://\u2026|id=url>]... [--backends <module>]\n                       [--specs <dir>] [--format text|json|markdown]\n                       [--require-environments a,b] [--require-all-specs]\n                       [--branch <name>]... [--ignore-modified]\n  atlas report merge   [--store \u2026]... [--backends <module>] --into <dir|s3://\u2026>\n  atlas report doctor  [--store \u2026]... [--role publisher|reader|both]\n  atlas report policy  --store s3://bucket/prefix [--role publisher|reader|both]\n\n  A store is a directory or s3://bucket/prefix (add ?region=\u2026&endpoint=\u2026 for R2, MinIO and the\n  rest). Select a project on an Atlas node with --project <slug> or ATLAS_PROJECT. This scopes specs, reports and recordings together.\n  When no --store is given, ATLAS_STORE is used. Keys never go in the URL: set\n  AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or sign in with `aws sso login`, or use a role.\n\n  publish   Write one immutable layer: the checks of one producer, on one environment. Safe to repeat.\n            --producer is required, unless the --checks file names one\n            --spec <path>   every check in the report belongs to this spec (a report per feature file)\n            --specs <dir>   link checks to the spec files in <dir> by path, or by the suite, class or\n                            file name the report gives; also publishes the specs themselves, so the\n                            node can show them without a checkout\n            --run-id <id>   name this execution (a local or manual run with no CI run URL; also ATLAS_RUN_ID).\n                            Layers that share an id are one run, however many shards or batches uploaded them;\n                            another invocation at the same commit gets another id. Never part of what a check is\n            --run-title <t> a human name for the run (also ATLAS_RUN_TITLE); --run-started-at and --run-ended-at\n                            record when it ran. Without --run-id a run is its CI run URL, else branch@sha\n            --upload-run    with --manifest: also upload the run's screenshots and clips for the viewer\n            --attach <file>   attach a file to the whole layer: a CI log, a coverage report, a trace archive\n                              (screenshots, videos and traces of a single test go in its CTRF or checks file)\n            --label k=v       a fact about the run, not part of what it checks: build, os, browser\n            --link [n=]<url>  a link to the run, a dashboard, an issue\n            Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own\n            folder and each --attach-root, and never from a hidden file or a link out: a report is data from a\n            test run, not instructions. One that is missing or not allowed is skipped with a warning.\n            From a checkout with uncommitted changes the layer is marked \"modified\".\n  stream    Send checks as they finish: one JSON object per line on standard input (the fields of one\n            check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server\n            shows the run while it is still going. A check sent again with the same id (running, then\n            passed) is replaced by the later. Ends when the input does; exit 1 if anything could not be sent.\n            Point it at an Atlas server with --store https://\u2026 and a token in ATLAS_TOKEN.\n  status    Combine every layer of every store, and exit 1 unless the report passes.\n            --specs <dir>          lay the results over these spec files (default: the specs in the store)\n            --require-all-specs    fail while any spec has no results\n            --branch <name>        only layers recorded on this branch (repeatable)\n            --ignore-modified      leave out layers from modified checkouts and unknown origins\n  merge     Copy the layers of several stores into one.\n  doctor    Check that a store works: credentials, listing, writing, and what it holds.\n  policy    Print the least-privilege IAM policy for publishers, readers, or both.\n\n  --backends <module>  A TypeScript or JavaScript module whose default export is a list of\n                       { id, label?, store } (or a function returning it): a device lab, anything\n                       that implements ObjectStore.\n\n  --format json (alias --json)   publish, stream, merge, doctor and status print one JSON document instead of prose\n                       (schemas: publish-result, stream-result, merge-result, doctor-result, report-status). A command\n                       that is wrong is {\"ok\":false,\"error\":{\"message\":\u2026,\"code\":\"usage\"}} on standard error.\n  A store may be an Atlas server (https://\u2026): ATLAS_STORE=https://atlas.example.com, with ATLAS_TOKEN.\n\n  Exit codes: 0 ok, 1 the report does not pass (or a check failed), 2 the command was wrong.\n";
+export declare const REPORT_USAGE = "atlas report \u2014 layered evidence for any kind of spec\n\n  atlas report publish [--store <dir|s3://bucket/prefix>] [--producer <name>]\n                       ( --ctrf <file> | --junit <file> | --manifest <file> | --checks <file.json> )\n                       [--environment <name>] [--branch <b>] [--sha <s>] [--run-url <u>]\n                       [--run-id <id>] [--run-title <t>] [--run-started-at <iso|ms>] [--run-ended-at <iso|ms>]\n                       [--spec <path> | --specs <dir>] [--upload-run]\n                       [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]\n                       [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...\n                       [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]\n                       [--require-recordings]\n  atlas report stream  [--store <dir|s3://\u2026|https://atlas.example.com>] --producer <name> [--environment <name>]\n                       [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...\n                       [--run-id <id>] [--run-title <t>]\n                       [--flush-ms 1000] [--batch 100]       (checks as JSON lines on standard input)\n  atlas report status  [--store <dir|s3://\u2026|id=url>]... [--backends <module>]\n                       [--specs <dir>] [--format text|json|markdown]\n                       [--require-environments a,b] [--require-all-specs]\n                       [--branch <name>]... [--ignore-modified]\n  atlas report merge   [--store \u2026]... [--backends <module>] --into <dir|s3://\u2026>\n  atlas report doctor  [--store \u2026]... [--role publisher|reader|both]\n  atlas report policy  --store s3://bucket/prefix [--role publisher|reader|both]\n\n  A store is a directory or s3://bucket/prefix (add ?region=\u2026&endpoint=\u2026 for R2, MinIO and the\n  rest). Select a project on an Atlas node with --project <slug> or ATLAS_PROJECT. This scopes specs, reports and recordings together.\n  When no --store is given, ATLAS_STORE is used. Keys never go in the URL: set\n  AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or sign in with `aws sso login`, or use a role.\n\n  publish   Write one immutable layer: the checks of one producer, on one environment. Safe to repeat.\n            --producer is required, unless the --checks file names one\n            --spec <path>   every check in the report belongs to this spec (a report per feature file)\n            --specs <dir>   link checks to the spec files in <dir> by path, or by the suite, class or\n                            file name the report gives; also publishes the specs themselves, so the\n                            node can show them without a checkout\n            --run-id <id>   name this execution (a local or manual run with no CI run URL; also ATLAS_RUN_ID).\n                            Layers that share an id are one run, however many shards or batches uploaded them;\n                            another invocation at the same commit gets another id. Never part of what a check is\n            --run-title <t> a human name for the run (also ATLAS_RUN_TITLE); --run-started-at and --run-ended-at\n                            record when it ran. Without --run-id a run is its CI run URL, else branch@sha\n            --upload-run    with --manifest: also upload the run's screenshots and clips for the viewer\n            --attach <file>   attach a file to the whole layer: a CI log, a coverage report, a trace archive\n                              (screenshots, videos and traces of a single test go in its CTRF or checks file)\n            --label k=v       a fact about the run, not part of what it checks: build, os, browser\n            --link [n=]<url>  a link to the run, a dashboard, an issue\n            Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own\n            folder and each --attach-root, and never from a hidden file or a link out: a report is data from a\n            test run, not instructions. One that is missing or not allowed is skipped with a warning.\n            --require-recordings   exit 1 when a check that declared recordingExpectation \"required\" has no\n                              uploaded video (absent, left out as too large or not allowed, or refused). The layer is\n                              still written and the checks keep their results; only this command fails. The receipt of\n                              what was uploaded (counts, sizes, checksums) is in the JSON result under layer.recordings.\n            From a checkout with uncommitted changes the layer is marked \"modified\".\n  stream    Send checks as they finish: one JSON object per line on standard input (the fields of one\n            check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server\n            shows the run while it is still going. A check sent again with the same id (running, then\n            passed) is replaced by the later. Ends when the input does; exit 1 if anything could not be sent.\n            Point it at an Atlas server with --store https://\u2026 and a token in ATLAS_TOKEN.\n  status    Combine every layer of every store, and exit 1 unless the report passes.\n            --specs <dir>          lay the results over these spec files (default: the specs in the store)\n            --require-all-specs    fail while any spec has no results\n            --branch <name>        only layers recorded on this branch (repeatable)\n            --ignore-modified      leave out layers from modified checkouts and unknown origins\n  merge     Copy the layers of several stores into one.\n  doctor    Check that a store works: credentials, listing, writing, and what it holds.\n  policy    Print the least-privilege IAM policy for publishers, readers, or both.\n\n  --backends <module>  A TypeScript or JavaScript module whose default export is a list of\n                       { id, label?, store } (or a function returning it): a device lab, anything\n                       that implements ObjectStore.\n\n  --format json (alias --json)   publish, stream, merge, doctor and status print one JSON document instead of prose\n                       (schemas: publish-result, stream-result, merge-result, doctor-result, report-status). A command\n                       that is wrong is {\"ok\":false,\"error\":{\"message\":\u2026,\"code\":\"usage\"}} on standard error.\n  A store may be an Atlas server (https://\u2026): ATLAS_STORE=https://atlas.example.com, with ATLAS_TOKEN.\n\n  Exit codes: 0 ok, 1 the report does not pass (or a check failed), 2 the command was wrong.\n";
 type Env = Readonly<Record<string, string | undefined>>;
 type Git = (args: readonly string[]) => string | null;
 /** What `reportMain` is given besides arguments: where output goes (`out`), and, for tests and embedding, the environment, working directory, git, clock and standard input (`stdin`, where `stream` reads checks from). Defaults are the real ones. */

package/dist/reports/src/report-cli.js

@@ -43,6 +43,7 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
                        [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]
                        [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...
                        [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]
+                       [--require-recordings]
   atlas report stream  [--store <dir|s3://…|https://atlas.example.com>] --producer <name> [--environment <name>]
                        [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...
                        [--run-id <id>] [--run-title <t>]
@@ -79,6 +80,10 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
             Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own
             folder and each --attach-root, and never from a hidden file or a link out: a report is data from a
             test run, not instructions. One that is missing or not allowed is skipped with a warning.
+            --require-recordings   exit 1 when a check that declared recordingExpectation "required" has no
+                              uploaded video (absent, left out as too large or not allowed, or refused). The layer is
+                              still written and the checks keep their results; only this command fails. The receipt of
+                              what was uploaded (counts, sizes, checksums) is in the JSON result under layer.recordings.
             From a checkout with uncommitted changes the layer is marked "modified".
   stream    Send checks as they finish: one JSON object per line on standard input (the fields of one
             check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server
@@ -310,7 +315,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
         const json = wantsJson(rest);
         const result = {
             ok: true, store: target.label,
-            layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes },
+            layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes, ...(layer.recordings ? { recordings: layer.recordings } : {}) },
             warnings: warnings.map(warning => `attachment skipped: ${warning}`)
         };
         if (!json)
@@ -318,6 +323,19 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
         if (!json)
             for (const warning of warnings)
                 io.out(`! attachment skipped: ${warning}\n`);
+        // Recording completeness is reported apart from the checks' results. Only --require-recordings turns a gap into a failing exit.
+        const gaps = (layer.recordings?.items ?? []).filter(item => item.state !== 'uploaded' && item.state !== 'referenced' && item.state !== 'not-applicable');
+        const strict = rest.includes('--require-recordings');
+        if (layer.recordings) {
+            result.recordings = { strict, complete: gaps.length === 0, expected: layer.recordings.expected, uploaded: layer.recordings.uploaded, gaps: gaps.map(({ check, state, reason }) => ({ check, state, ...(reason ? { reason } : {}) })) };
+            if (!json) {
+                io.out(`Recordings: ${layer.recordings.uploaded + layer.recordings.referenced} of ${layer.recordings.expected} expected published${layer.recordings.notApplicable ? `, ${layer.recordings.notApplicable} not applicable` : ''}\n`);
+                for (const gap of gaps)
+                    io.out(`! recording ${gap.state}: ${gap.check}${gap.reason ? ` (${gap.reason})` : ''}\n`);
+            }
+            if (strict && gaps.length)
+                result.ok = false;
+        }
         if (specsDirectory) {
             const files = await collectSpecFiles(path.resolve(base, specsDirectory));
             // A spec named with --spec that the directory does not hold (or whose kind is not a spec file) is published too.
@@ -343,7 +361,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
         }
         if (json)
             io.out(`${JSON.stringify(result, null, 2)}\n`);
-        return 0;
+        return result.ok ? 0 : 1;
     }
     if (command === 'stream') {
         const target = writeStoreOf(value(rest, 'store'), base, env);

package/dist/reports/src/report-status.d.ts

@@ -26,6 +26,8 @@ export type StatusOptions = {
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
  * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
+ * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *
  * @example
  * const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });

package/dist/reports/src/report-status.js

@@ -12,6 +12,7 @@ import { REPORT_STATUS_SCHEMA_VERSION } from '../../schema/src/layer.js';
 import { countResults } from './compose.js';
 import { normalizeEnvironment } from './layer-axes.js';
 import { isDiagnosticKind } from './layer-merge.js';
+import { recordingCompleteness } from './recording-receipt.js';
 /** Total string order by code unit. */
 const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
 /**
@@ -23,6 +24,8 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
  * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
+ * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *
  * @example
  * const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });
@@ -85,7 +88,12 @@ export function reportStatus(report, options = {}) {
             ...report.combined.specs.map(entry => specOutcome(entry.spec.path, entry.spec.title, entry.spec.kind, entry.results)),
             ...(report.combined.unassigned.length ? [specOutcome(null, 'No spec', 'unassigned', report.combined.unassigned)] : [])
         ],
-        failing
+        failing,
+        // Apart from `ok`: whether the expected videos were published says nothing about whether the checks passed.
+        recordings: recordingCompleteness(outcomes.map(result => ({
+            spec: result.row.specPath, check: result.row.id, environment: result.layer.environment, layer: result.layer.id,
+            hasReceipt: result.layer.recordings !== undefined, ...(result.row.recording ? { recording: result.row.recording } : {})
+        })))
     };
 }
 /** `spec › check`, unless the check's id already starts with its spec. */
@@ -120,6 +128,14 @@ export function formatStatusText(status) {
         const where = [item.environment, item.branch, item.sha?.slice(0, 7)].filter(Boolean).join(' · ');
         lines.push(`✗ ${item.status === 'running' ? '[running] ' : ''}${label(item)}${where ? `  (${where})` : ''}${item.detail ? `\n    ${item.detail.split('\n')[0]}` : ''}`);
     }
+    const recordings = status.recordings;
+    // Quiet for a report that never mentioned recordings; the JSON says `unknown` explicitly.
+    if (recordings && !(recordings.state === 'unknown' && recordings.expected === 0 && recordings.notApplicable === 0)) {
+        const published = recordings.uploaded + recordings.referenced;
+        lines.push(`· recordings: ${recordings.state} — ${published} of ${recordings.expected} expected published${recordings.notApplicable ? `, ${recordings.notApplicable} not applicable` : ''}${recordings.layersWithoutReceipt ? `, ${recordings.layersWithoutReceipt} layer${recordings.layersWithoutReceipt === 1 ? '' : 's'} unknown` : ''}`);
+        for (const gap of recordings.gaps)
+            lines.push(`  ! recording ${gap.state}: ${label({ spec: gap.spec, check: gap.check })}${gap.reason ? ` (${gap.reason})` : ''}`);
+    }
     for (const notice of status.notices)
         lines.push(`! ${notice}`);
     for (const reason of status.reasons)

package/dist/reports/src/reports-explorer-route.d.ts

@@ -82,6 +82,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                         height?: number | undefined;
                         durationMs?: number | undefined;
                         description?: string | undefined;
+                        omitted?: string | undefined;
                         tags?: string[] | undefined;
                         at?: {
                             [x: string]: import("hono/utils/types").JSONValue;
@@ -126,6 +127,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;
@@ -183,6 +185,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -250,6 +253,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -356,6 +360,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -400,6 +405,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                     height?: number | undefined;
                                     durationMs?: number | undefined;
                                     description?: string | undefined;
+                                    omitted?: string | undefined;
                                     tags?: string[] | undefined;
                                     at?: {
                                         [x: string]: import("hono/utils/types").JSONValue;
@@ -457,6 +463,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;
@@ -524,6 +531,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;