@crvouga/atlas 0.16.0 → 1.0.0

package/dist/reports/src/recording-receipt.js

@@ -0,0 +1,103 @@
+import { contentTypeOf } from './media-types.js';
+/** Is this attachment a screen recording: a role of `video`, or a `video/*` media type? */
+export function isVideoAttachment(attachment) {
+    if (attachment.role === 'video')
+        return true;
+    const type = typeof attachment.contentType === 'string' ? attachment.contentType : typeof attachment.name === 'string' ? contentTypeOf(attachment.name) : '';
+    return type.startsWith('video/');
+}
+/** Every video attachment of a holder, its steps and its attempts, at any depth. */
+function videosOf(holder) {
+    const found = [];
+    const take = (list) => { if (Array.isArray(list))
+        for (const item of list)
+            if (isVideoAttachment(item))
+                found.push(item); };
+    take(holder.attachments);
+    for (const step of holder.steps ?? [])
+        take(step.attachments);
+    for (const attempt of holder.attempts ?? []) {
+        take(attempt.attachments);
+        for (const step of attempt.steps ?? [])
+            take(step.attachments);
+    }
+    return found;
+}
+/**
+ * The receipt for the checks of one layer, or `undefined` when no check declares a `recordingExpectation` (the layer then
+ * claims nothing). Call it with checks whose attachment bytes are already stored (`bundleKey`, `bytes` and `sha256` set).
+ *
+ * A `required` check is `uploaded` when any video of it has stored bytes, `referenced` when one has only a `uri`,
+ * `skipped` when its videos were left out (the attachment's `omitted` is the reason) and `unresolved` when it has no
+ * video at all. A `not-applicable` check is listed but is neither expected nor missing. Items are sorted by check id so the
+ * same input is always the same receipt.
+ */
+export function recordingReceipt(checks) {
+    const items = [];
+    const count = { uploaded: 0, referenced: 0, skipped: 0, unresolved: 0, notApplicable: 0 };
+    for (const check of checks) {
+        const expectation = check.recordingExpectation;
+        if (expectation !== 'required' && expectation !== 'not-applicable')
+            continue;
+        const check_ = String(check.id);
+        if (expectation === 'not-applicable') {
+            count.notApplicable += 1;
+            items.push({ check: check_, state: 'not-applicable' });
+            continue;
+        }
+        const files = videosOf(check).map(video => {
+            const state = typeof video.bundleKey === 'string' ? 'uploaded' : typeof video.uri === 'string' ? 'referenced' : 'skipped';
+            return {
+                name: String(video.name),
+                state,
+                ...(typeof video.bytes === 'number' ? { bytes: video.bytes } : {}),
+                ...(typeof video.sha256 === 'string' ? { sha256: video.sha256 } : {}),
+                ...(state === 'skipped' ? { reason: typeof video.omitted === 'string' ? video.omitted : 'bytes-not-attached' } : {})
+            };
+        });
+        const first = (state) => files.find(file => file.state === state);
+        const state = first('uploaded') ? 'uploaded' : first('referenced') ? 'referenced' : first('skipped') ? 'skipped' : 'unresolved';
+        const reason = state === 'skipped' ? first('skipped')?.reason : state === 'unresolved' ? 'no-video-attached' : undefined;
+        count[state] += 1;
+        items.push({ check: check_, state, ...(reason ? { reason } : {}), ...(files.length ? { files } : {}) });
+    }
+    if (!items.length)
+        return undefined;
+    items.sort((a, b) => (a.check < b.check ? -1 : a.check > b.check ? 1 : 0));
+    return { expected: count.uploaded + count.referenced + count.skipped + count.unresolved, ...count, items };
+}
+/**
+ * Reduces the winning results of a report to its recording completeness. `incomplete` when any expected recording is
+ * skipped or unresolved; otherwise `unknown` when a contributing layer has no receipt (nothing is claimed for it, and a
+ * report with no results at all is unknown too); otherwise `complete` when something was expected and `none-expected` when
+ * nothing was. Gaps are sorted so the output never depends on read order.
+ */
+export function recordingCompleteness(results) {
+    const count = { expected: 0, uploaded: 0, referenced: 0, skipped: 0, unresolved: 0, notApplicable: 0 };
+    const gaps = [];
+    const without = new Set();
+    for (const result of results) {
+        if (!result.hasReceipt) {
+            without.add(result.layer);
+            continue;
+        }
+        const state = result.recording?.state;
+        if (state === undefined)
+            continue;
+        if (state === 'not-applicable') {
+            count.notApplicable += 1;
+            continue;
+        }
+        count.expected += 1;
+        if (state === 'uploaded' || state === 'referenced' || state === 'skipped' || state === 'unresolved')
+            count[state] += 1;
+        // A state this reader does not know is a gap: it is never counted as uploaded.
+        if (state !== 'uploaded' && state !== 'referenced') {
+            gaps.push({ spec: result.spec, check: result.check, layer: result.layer, environment: result.environment, state, ...(result.recording?.reason ? { reason: result.recording.reason } : {}) });
+        }
+    }
+    const text = (value) => value ?? '';
+    gaps.sort((a, b) => (text(a.spec) + a.check + text(a.environment) + a.layer < text(b.spec) + b.check + text(b.environment) + b.layer ? -1 : 1));
+    const state = gaps.length ? 'incomplete' : without.size || !results.length ? 'unknown' : count.expected ? 'complete' : 'none-expected';
+    return { state, ...count, layersWithoutReceipt: without.size, gaps };
+}

package/dist/reports/src/report-cli.d.ts

@@ -13,7 +13,7 @@
  * @see docs/agents.md
  */
 /** The text `atlas report --help` prints: every subcommand, its flags, what each does and the exit codes. Also the source of the command reference in `docs/reference/cli.md`. */
-export declare const REPORT_USAGE = "atlas report \u2014 layered evidence for any kind of spec\n\n  atlas report publish [--store <dir|s3://bucket/prefix>] [--producer <name>]\n                       ( --ctrf <file> | --junit <file> | --manifest <file> | --checks <file.json> )\n                       [--environment <name>] [--branch <b>] [--sha <s>] [--run-url <u>]\n                       [--run-id <id>] [--run-title <t>] [--run-started-at <iso|ms>] [--run-ended-at <iso|ms>]\n                       [--spec <path> | --specs <dir>] [--upload-run]\n                       [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]\n                       [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...\n                       [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]\n  atlas report stream  [--store <dir|s3://\u2026|https://atlas.example.com>] --producer <name> [--environment <name>]\n                       [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...\n                       [--run-id <id>] [--run-title <t>]\n                       [--flush-ms 1000] [--batch 100]       (checks as JSON lines on standard input)\n  atlas report status  [--store <dir|s3://\u2026|id=url>]... [--backends <module>]\n                       [--specs <dir>] [--format text|json|markdown]\n                       [--require-environments a,b] [--require-all-specs]\n                       [--branch <name>]... [--ignore-modified]\n  atlas report merge   [--store \u2026]... [--backends <module>] --into <dir|s3://\u2026>\n  atlas report doctor  [--store \u2026]... [--role publisher|reader|both]\n  atlas report policy  --store s3://bucket/prefix [--role publisher|reader|both]\n\n  A store is a directory or s3://bucket/prefix (add ?region=\u2026&endpoint=\u2026 for R2, MinIO and the\n  rest). Select a project on an Atlas node with --project <slug> or ATLAS_PROJECT. This scopes specs, reports and recordings together.\n  When no --store is given, ATLAS_STORE is used. Keys never go in the URL: set\n  AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or sign in with `aws sso login`, or use a role.\n\n  publish   Write one immutable layer: the checks of one producer, on one environment. Safe to repeat.\n            --producer is required, unless the --checks file names one\n            --spec <path>   every check in the report belongs to this spec (a report per feature file)\n            --specs <dir>   link checks to the spec files in <dir> by path, or by the suite, class or\n                            file name the report gives; also publishes the specs themselves, so the\n                            node can show them without a checkout\n            --run-id <id>   name this execution (a local or manual run with no CI run URL; also ATLAS_RUN_ID).\n                            Layers that share an id are one run, however many shards or batches uploaded them;\n                            another invocation at the same commit gets another id. Never part of what a check is\n            --run-title <t> a human name for the run (also ATLAS_RUN_TITLE); --run-started-at and --run-ended-at\n                            record when it ran. Without --run-id a run is its CI run URL, else branch@sha\n            --upload-run    with --manifest: also upload the run's screenshots and clips for the viewer\n            --attach <file>   attach a file to the whole layer: a CI log, a coverage report, a trace archive\n                              (screenshots, videos and traces of a single test go in its CTRF or checks file)\n            --label k=v       a fact about the run, not part of what it checks: build, os, browser\n            --link [n=]<url>  a link to the run, a dashboard, an issue\n            Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own\n            folder and each --attach-root, and never from a hidden file or a link out: a report is data from a\n            test run, not instructions. One that is missing or not allowed is skipped with a warning.\n            From a checkout with uncommitted changes the layer is marked \"modified\".\n  stream    Send checks as they finish: one JSON object per line on standard input (the fields of one\n            check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server\n            shows the run while it is still going. A check sent again with the same id (running, then\n            passed) is replaced by the later. Ends when the input does; exit 1 if anything could not be sent.\n            Point it at an Atlas server with --store https://\u2026 and a token in ATLAS_TOKEN.\n  status    Combine every layer of every store, and exit 1 unless the report passes.\n            --specs <dir>          lay the results over these spec files (default: the specs in the store)\n            --require-all-specs    fail while any spec has no results\n            --branch <name>        only layers recorded on this branch (repeatable)\n            --ignore-modified      leave out layers from modified checkouts and unknown origins\n  merge     Copy the layers of several stores into one.\n  doctor    Check that a store works: credentials, listing, writing, and what it holds.\n  policy    Print the least-privilege IAM policy for publishers, readers, or both.\n\n  --backends <module>  A TypeScript or JavaScript module whose default export is a list of\n                       { id, label?, store } (or a function returning it): a device lab, anything\n                       that implements ObjectStore.\n\n  --format json (alias --json)   publish, stream, merge, doctor and status print one JSON document instead of prose\n                       (schemas: publish-result, stream-result, merge-result, doctor-result, report-status). A command\n                       that is wrong is {\"ok\":false,\"error\":{\"message\":\u2026,\"code\":\"usage\"}} on standard error.\n  A store may be an Atlas server (https://\u2026): ATLAS_STORE=https://atlas.example.com, with ATLAS_TOKEN.\n\n  Exit codes: 0 ok, 1 the report does not pass (or a check failed), 2 the command was wrong.\n";
+export declare const REPORT_USAGE = "atlas report \u2014 layered evidence for any kind of spec\n\n  atlas report publish [--store <dir|s3://bucket/prefix>] [--producer <name>]\n                       ( --ctrf <file> | --junit <file> | --manifest <file> | --checks <file.json> )\n                       [--environment <name>] [--branch <b>] [--sha <s>] [--run-url <u>]\n                       [--run-id <id>] [--run-title <t>] [--run-started-at <iso|ms>] [--run-ended-at <iso|ms>]\n                       [--spec <path> | --specs <dir>] [--upload-run]\n                       [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]\n                       [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...\n                       [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]\n                       [--require-recordings]\n  atlas report stream  [--store <dir|s3://\u2026|https://atlas.example.com>] --producer <name> [--environment <name>]\n                       [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...\n                       [--run-id <id>] [--run-title <t>]\n                       [--flush-ms 1000] [--batch 100]       (checks as JSON lines on standard input)\n  atlas report status  [--store <dir|s3://\u2026|id=url>]... [--backends <module>]\n                       [--specs <dir>] [--format text|json|markdown]\n                       [--require-environments a,b] [--require-all-specs]\n                       [--branch <name>]... [--ignore-modified]\n  atlas report merge   [--store \u2026]... [--backends <module>] --into <dir|s3://\u2026>\n  atlas report doctor  [--store \u2026]... [--role publisher|reader|both]\n  atlas report policy  --store s3://bucket/prefix [--role publisher|reader|both]\n\n  A store is a directory or s3://bucket/prefix (add ?region=\u2026&endpoint=\u2026 for R2, MinIO and the\n  rest). Select a project on an Atlas node with --project <slug> or ATLAS_PROJECT. This scopes specs, reports and recordings together.\n  When no --store is given, ATLAS_STORE is used. Keys never go in the URL: set\n  AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or sign in with `aws sso login`, or use a role.\n\n  publish   Write one immutable layer: the checks of one producer, on one environment. Safe to repeat.\n            --producer is required, unless the --checks file names one\n            --spec <path>   every check in the report belongs to this spec (a report per feature file)\n            --specs <dir>   link checks to the spec files in <dir> by path, or by the suite, class or\n                            file name the report gives; also publishes the specs themselves, so the\n                            node can show them without a checkout\n            --run-id <id>   name this execution (a local or manual run with no CI run URL; also ATLAS_RUN_ID).\n                            Layers that share an id are one run, however many shards or batches uploaded them;\n                            another invocation at the same commit gets another id. Never part of what a check is\n            --run-title <t> a human name for the run (also ATLAS_RUN_TITLE); --run-started-at and --run-ended-at\n                            record when it ran. Without --run-id a run is its CI run URL, else branch@sha\n            --upload-run    with --manifest: also upload the run's screenshots and clips for the viewer\n            --attach <file>   attach a file to the whole layer: a CI log, a coverage report, a trace archive\n                              (screenshots, videos and traces of a single test go in its CTRF or checks file)\n            --label k=v       a fact about the run, not part of what it checks: build, os, browser\n            --link [n=]<url>  a link to the run, a dashboard, an issue\n            Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own\n            folder and each --attach-root, and never from a hidden file or a link out: a report is data from a\n            test run, not instructions. One that is missing or not allowed is skipped with a warning.\n            --require-recordings   exit 1 when a check that declared recordingExpectation \"required\" has no\n                              uploaded video (absent, left out as too large or not allowed, or refused). The layer is\n                              still written and the checks keep their results; only this command fails. The receipt of\n                              what was uploaded (counts, sizes, checksums) is in the JSON result under layer.recordings.\n            From a checkout with uncommitted changes the layer is marked \"modified\".\n  stream    Send checks as they finish: one JSON object per line on standard input (the fields of one\n            check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server\n            shows the run while it is still going. A check sent again with the same id (running, then\n            passed) is replaced by the later. Ends when the input does; exit 1 if anything could not be sent.\n            Point it at an Atlas server with --store https://\u2026 and a token in ATLAS_TOKEN.\n  status    Combine every layer of every store, and exit 1 unless the report passes.\n            --specs <dir>          lay the results over these spec files (default: the specs in the store)\n            --require-all-specs    fail while any spec has no results\n            --branch <name>        only layers recorded on this branch (repeatable)\n            --ignore-modified      leave out layers from modified checkouts and unknown origins\n  merge     Copy the layers of several stores into one.\n  doctor    Check that a store works: credentials, listing, writing, and what it holds.\n  policy    Print the least-privilege IAM policy for publishers, readers, or both.\n\n  --backends <module>  A TypeScript or JavaScript module whose default export is a list of\n                       { id, label?, store } (or a function returning it): a device lab, anything\n                       that implements ObjectStore.\n\n  --format json (alias --json)   publish, stream, merge, doctor and status print one JSON document instead of prose\n                       (schemas: publish-result, stream-result, merge-result, doctor-result, report-status). A command\n                       that is wrong is {\"ok\":false,\"error\":{\"message\":\u2026,\"code\":\"usage\"}} on standard error.\n  A store may be an Atlas server (https://\u2026): ATLAS_STORE=https://atlas.example.com, with ATLAS_TOKEN.\n\n  Exit codes: 0 ok, 1 the report does not pass (or a check failed), 2 the command was wrong.\n";
 type Env = Readonly<Record<string, string | undefined>>;
 type Git = (args: readonly string[]) => string | null;
 /** What `reportMain` is given besides arguments: where output goes (`out`), and, for tests and embedding, the environment, working directory, git, clock and standard input (`stdin`, where `stream` reads checks from). Defaults are the real ones. */

package/dist/reports/src/report-cli.js

@@ -43,6 +43,7 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
                        [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]
                        [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...
                        [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]
+                       [--require-recordings]
   atlas report stream  [--store <dir|s3://…|https://atlas.example.com>] --producer <name> [--environment <name>]
                        [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...
                        [--run-id <id>] [--run-title <t>]
@@ -79,6 +80,10 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
             Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own
             folder and each --attach-root, and never from a hidden file or a link out: a report is data from a
             test run, not instructions. One that is missing or not allowed is skipped with a warning.
+            --require-recordings   exit 1 when a check that declared recordingExpectation "required" has no
+                              uploaded video (absent, left out as too large or not allowed, or refused). The layer is
+                              still written and the checks keep their results; only this command fails. The receipt of
+                              what was uploaded (counts, sizes, checksums) is in the JSON result under layer.recordings.
             From a checkout with uncommitted changes the layer is marked "modified".
   stream    Send checks as they finish: one JSON object per line on standard input (the fields of one
             check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server
@@ -310,7 +315,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
         const json = wantsJson(rest);
         const result = {
             ok: true, store: target.label,
-            layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes },
+            layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes, ...(layer.recordings ? { recordings: layer.recordings } : {}) },
             warnings: warnings.map(warning => `attachment skipped: ${warning}`)
         };
         if (!json)
@@ -318,6 +323,19 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
         if (!json)
             for (const warning of warnings)
                 io.out(`! attachment skipped: ${warning}\n`);
+        // Recording completeness is reported apart from the checks' results. Only --require-recordings turns a gap into a failing exit.
+        const gaps = (layer.recordings?.items ?? []).filter(item => item.state !== 'uploaded' && item.state !== 'referenced' && item.state !== 'not-applicable');
+        const strict = rest.includes('--require-recordings');
+        if (layer.recordings) {
+            result.recordings = { strict, complete: gaps.length === 0, expected: layer.recordings.expected, uploaded: layer.recordings.uploaded, gaps: gaps.map(({ check, state, reason }) => ({ check, state, ...(reason ? { reason } : {}) })) };
+            if (!json) {
+                io.out(`Recordings: ${layer.recordings.uploaded + layer.recordings.referenced} of ${layer.recordings.expected} expected published${layer.recordings.notApplicable ? `, ${layer.recordings.notApplicable} not applicable` : ''}\n`);
+                for (const gap of gaps)
+                    io.out(`! recording ${gap.state}: ${gap.check}${gap.reason ? ` (${gap.reason})` : ''}\n`);
+            }
+            if (strict && gaps.length)
+                result.ok = false;
+        }
         if (specsDirectory) {
             const files = await collectSpecFiles(path.resolve(base, specsDirectory));
             // A spec named with --spec that the directory does not hold (or whose kind is not a spec file) is published too.
@@ -343,7 +361,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
         }
         if (json)
             io.out(`${JSON.stringify(result, null, 2)}\n`);
-        return 0;
+        return result.ok ? 0 : 1;
     }
     if (command === 'stream') {
         const target = writeStoreOf(value(rest, 'store'), base, env);

package/dist/reports/src/report-index.d.ts

@@ -93,6 +93,12 @@ export declare class ReportIndex {
      * and the run list are built from. Read-only for callers; `history.rev` changes whenever a layer is taken in.
      */
     get history(): TestHistory;
+    /**
+     * The ids of layers this index has read and that no longer decide any result. Removing one from its store cannot
+     * change the report (every slot it filled is held by a newer layer), which is what makes it safe for garbage
+     * collection to delete. A layer that failed to load, is hidden by `include`, or still holds a result is not here.
+     */
+    superseded(): ReadonlySet<string>;
     /** For `/readyz` and logs. */
     stats(): {
         backends: number;

package/dist/reports/src/report-index.js

@@ -50,6 +50,8 @@ export class ReportIndex {
     /** Layers that stopped holding any winner, and when. Older removals are forgotten; `#forgotten` says how far. */
     #removed = [];
     #forgotten = 0;
+    /** Layers that were read, took part in the report, and no longer decide any result: all their checks were replaced by newer layers. */
+    #superseded = new Set();
     #refreshing = null;
     #timer = null;
     #refreshedAt = 0;
@@ -248,6 +250,9 @@ export class ReportIndex {
         if (wins.size) {
             this.#held.set(id, { evidence, wins, rev: ++this.#rev, entry: null, json: null, digest: null });
         }
+        else {
+            this.#superseded.add(id);
+        }
     }
     /** What this layer sends has changed. */
     #touch(held) {
@@ -264,6 +269,7 @@ export class ReportIndex {
             return;
         }
         this.#held.delete(layerId);
+        this.#superseded.add(layerId);
         this.#removed.push({ id: layerId, rev: ++this.#rev });
         if (this.#removed.length > REMEMBERED_REMOVALS)
             this.#forgotten = this.#removed.splice(0, this.#removed.length - REMEMBERED_REMOVALS).at(-1).rev;
@@ -342,6 +348,14 @@ export class ReportIndex {
     get history() {
         return this.#history;
     }
+    /**
+     * The ids of layers this index has read and that no longer decide any result. Removing one from its store cannot
+     * change the report (every slot it filled is held by a newer layer), which is what makes it safe for garbage
+     * collection to delete. A layer that failed to load, is hidden by `include`, or still holds a result is not here.
+     */
+    superseded() {
+        return this.#superseded;
+    }
     /** For `/readyz` and logs. */
     stats() {
         return { backends: this.#backends.length, layers: this.#attribution.size, slots: this.#winners.size, holding: this.#held.size, failed: this.#failed.size, refreshedAt: this.#refreshedAt };

package/dist/reports/src/report-status.d.ts

@@ -26,6 +26,8 @@ export type StatusOptions = {
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
  * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
+ * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *
  * @example
  * const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });

package/dist/reports/src/report-status.js

@@ -12,6 +12,7 @@ import { REPORT_STATUS_SCHEMA_VERSION } from '../../schema/src/layer.js';
 import { countResults } from './compose.js';
 import { normalizeEnvironment } from './layer-axes.js';
 import { isDiagnosticKind } from './layer-merge.js';
+import { recordingCompleteness } from './recording-receipt.js';
 /** Total string order by code unit. */
 const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
 /**
@@ -23,6 +24,8 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
  * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
+ * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *
  * @example
  * const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });
@@ -85,7 +88,12 @@ export function reportStatus(report, options = {}) {
             ...report.combined.specs.map(entry => specOutcome(entry.spec.path, entry.spec.title, entry.spec.kind, entry.results)),
             ...(report.combined.unassigned.length ? [specOutcome(null, 'No spec', 'unassigned', report.combined.unassigned)] : [])
         ],
-        failing
+        failing,
+        // Apart from `ok`: whether the expected videos were published says nothing about whether the checks passed.
+        recordings: recordingCompleteness(outcomes.map(result => ({
+            spec: result.row.specPath, check: result.row.id, environment: result.layer.environment, layer: result.layer.id,
+            hasReceipt: result.layer.recordings !== undefined, ...(result.row.recording ? { recording: result.row.recording } : {})
+        })))
     };
 }
 /** `spec › check`, unless the check's id already starts with its spec. */
@@ -120,6 +128,14 @@ export function formatStatusText(status) {
         const where = [item.environment, item.branch, item.sha?.slice(0, 7)].filter(Boolean).join(' · ');
         lines.push(`✗ ${item.status === 'running' ? '[running] ' : ''}${label(item)}${where ? `  (${where})` : ''}${item.detail ? `\n    ${item.detail.split('\n')[0]}` : ''}`);
     }
+    const recordings = status.recordings;
+    // Quiet for a report that never mentioned recordings; the JSON says `unknown` explicitly.
+    if (recordings && !(recordings.state === 'unknown' && recordings.expected === 0 && recordings.notApplicable === 0)) {
+        const published = recordings.uploaded + recordings.referenced;
+        lines.push(`· recordings: ${recordings.state} — ${published} of ${recordings.expected} expected published${recordings.notApplicable ? `, ${recordings.notApplicable} not applicable` : ''}${recordings.layersWithoutReceipt ? `, ${recordings.layersWithoutReceipt} layer${recordings.layersWithoutReceipt === 1 ? '' : 's'} unknown` : ''}`);
+        for (const gap of recordings.gaps)
+            lines.push(`  ! recording ${gap.state}: ${label({ spec: gap.spec, check: gap.check })}${gap.reason ? ` (${gap.reason})` : ''}`);
+    }
     for (const notice of status.notices)
         lines.push(`! ${notice}`);
     for (const reason of status.reasons)

package/dist/reports/src/reports-explorer-route.d.ts

@@ -82,6 +82,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                         height?: number | undefined;
                         durationMs?: number | undefined;
                         description?: string | undefined;
+                        omitted?: string | undefined;
                         tags?: string[] | undefined;
                         at?: {
                             [x: string]: import("hono/utils/types").JSONValue;
@@ -126,6 +127,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;
@@ -183,6 +185,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -250,6 +253,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -356,6 +360,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -400,6 +405,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                     height?: number | undefined;
                                     durationMs?: number | undefined;
                                     description?: string | undefined;
+                                    omitted?: string | undefined;
                                     tags?: string[] | undefined;
                                     at?: {
                                         [x: string]: import("hono/utils/types").JSONValue;
@@ -457,6 +463,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;
@@ -524,6 +531,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;

package/dist/reports/src/reports-route.d.ts

@@ -120,6 +120,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                         height?: number | undefined;
                         durationMs?: number | undefined;
                         description?: string | undefined;
+                        omitted?: string | undefined;
                         tags?: string[] | undefined;
                         at?: {
                             [x: string]: import("hono/utils/types").JSONValue;
@@ -164,6 +165,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;
@@ -221,6 +223,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -288,6 +291,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -394,6 +398,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                             height?: number | undefined;
                             durationMs?: number | undefined;
                             description?: string | undefined;
+                            omitted?: string | undefined;
                             tags?: string[] | undefined;
                             at?: {
                                 [x: string]: import("hono/utils/types").JSONValue;
@@ -438,6 +443,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                                     height?: number | undefined;
                                     durationMs?: number | undefined;
                                     description?: string | undefined;
+                                    omitted?: string | undefined;
                                     tags?: string[] | undefined;
                                     at?: {
                                         [x: string]: import("hono/utils/types").JSONValue;
@@ -495,6 +501,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;
@@ -562,6 +569,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
                                 height?: number | undefined;
                                 durationMs?: number | undefined;
                                 description?: string | undefined;
+                                omitted?: string | undefined;
                                 tags?: string[] | undefined;
                                 at?: {
                                     [x: string]: import("hono/utils/types").JSONValue;

package/dist/schema/src/api.d.ts

@@ -165,6 +165,14 @@ export declare const LayerHeaderSchema: z.ZodObject<{
             available: z.ZodBoolean;
         }, z.core.$loose>>>;
     }, z.core.$loose>>;
+    recordings: z.ZodOptional<z.ZodObject<{
+        expected: z.ZodNumber;
+        uploaded: z.ZodNumber;
+        referenced: z.ZodNumber;
+        skipped: z.ZodNumber;
+        unresolved: z.ZodNumber;
+        notApplicable: z.ZodNumber;
+    }, z.core.$loose>>;
 }, z.core.$loose>;
 /** One check of a layer as the report shows it. */
 export declare const LayerRowSchema: z.ZodObject<{
@@ -207,6 +215,10 @@ export declare const LayerRowSchema: z.ZodObject<{
             available: z.ZodBoolean;
         }, z.core.$loose>>>;
     }, z.core.$loose>>;
+    recording: z.ZodOptional<z.ZodObject<{
+        state: z.ZodString;
+        reason: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>;
 }, z.core.$loose>;
 /** A layer with its checks: the rows of one layer, read apart from its header. */
 export declare const LayerRowsSchema: z.ZodObject<{
@@ -251,6 +263,10 @@ export declare const LayerRowsSchema: z.ZodObject<{
                 available: z.ZodBoolean;
             }, z.core.$loose>>>;
         }, z.core.$loose>>;
+        recording: z.ZodOptional<z.ZodObject<{
+            state: z.ZodString;
+            reason: z.ZodOptional<z.ZodString>;
+        }, z.core.$loose>>;
     }, z.core.$loose>>;
     fingerprints: z.ZodRecord<z.ZodString, z.ZodString>;
 }, z.core.$loose>;
@@ -297,6 +313,10 @@ export declare const LayerEvidenceSchema: z.ZodObject<{
                 available: z.ZodBoolean;
             }, z.core.$loose>>>;
         }, z.core.$loose>>;
+        recording: z.ZodOptional<z.ZodObject<{
+            state: z.ZodString;
+            reason: z.ZodOptional<z.ZodString>;
+        }, z.core.$loose>>;
     }, z.core.$loose>>;
     fingerprints: z.ZodRecord<z.ZodString, z.ZodString>;
     layer: z.ZodObject<{
@@ -348,6 +368,14 @@ export declare const LayerEvidenceSchema: z.ZodObject<{
                 available: z.ZodBoolean;
             }, z.core.$loose>>>;
         }, z.core.$loose>>;
+        recordings: z.ZodOptional<z.ZodObject<{
+            expected: z.ZodNumber;
+            uploaded: z.ZodNumber;
+            referenced: z.ZodNumber;
+            skipped: z.ZodNumber;
+            unresolved: z.ZodNumber;
+            notApplicable: z.ZodNumber;
+        }, z.core.$loose>>;
     }, z.core.$loose>;
 }, z.core.$loose>;
 /**
@@ -399,6 +427,10 @@ export declare const EffectiveEvidenceSchema: z.ZodObject<{
                     available: z.ZodBoolean;
                 }, z.core.$loose>>>;
             }, z.core.$loose>>;
+            recording: z.ZodOptional<z.ZodObject<{
+                state: z.ZodString;
+                reason: z.ZodOptional<z.ZodString>;
+            }, z.core.$loose>>;
         }, z.core.$loose>>;
         fingerprints: z.ZodRecord<z.ZodString, z.ZodString>;
         layer: z.ZodObject<{
@@ -450,6 +482,14 @@ export declare const EffectiveEvidenceSchema: z.ZodObject<{
                     available: z.ZodBoolean;
                 }, z.core.$loose>>>;
             }, z.core.$loose>>;
+            recordings: z.ZodOptional<z.ZodObject<{
+                expected: z.ZodNumber;
+                uploaded: z.ZodNumber;
+                referenced: z.ZodNumber;
+                skipped: z.ZodNumber;
+                unresolved: z.ZodNumber;
+                notApplicable: z.ZodNumber;
+            }, z.core.$loose>>;
         }, z.core.$loose>;
     }, z.core.$loose>>;
     backends: z.ZodArray<z.ZodObject<{
@@ -522,6 +562,10 @@ export declare const EffectiveDeltaSchema: z.ZodObject<{
                     available: z.ZodBoolean;
                 }, z.core.$loose>>>;
             }, z.core.$loose>>;
+            recording: z.ZodOptional<z.ZodObject<{
+                state: z.ZodString;
+                reason: z.ZodOptional<z.ZodString>;
+            }, z.core.$loose>>;
         }, z.core.$loose>>;
         fingerprints: z.ZodRecord<z.ZodString, z.ZodString>;
         layer: z.ZodObject<{
@@ -573,6 +617,14 @@ export declare const EffectiveDeltaSchema: z.ZodObject<{
                     available: z.ZodBoolean;
                 }, z.core.$loose>>>;
             }, z.core.$loose>>;
+            recordings: z.ZodOptional<z.ZodObject<{
+                expected: z.ZodNumber;
+                uploaded: z.ZodNumber;
+                referenced: z.ZodNumber;
+                skipped: z.ZodNumber;
+                unresolved: z.ZodNumber;
+                notApplicable: z.ZodNumber;
+            }, z.core.$loose>>;
         }, z.core.$loose>;
     }, z.core.$loose>>;
     removed: z.ZodArray<z.ZodString>;
@@ -648,6 +700,14 @@ export declare const EvidencePageSchema: z.ZodObject<{
                 available: z.ZodBoolean;
             }, z.core.$loose>>>;
         }, z.core.$loose>>;
+        recordings: z.ZodOptional<z.ZodObject<{
+            expected: z.ZodNumber;
+            uploaded: z.ZodNumber;
+            referenced: z.ZodNumber;
+            skipped: z.ZodNumber;
+            unresolved: z.ZodNumber;
+            notApplicable: z.ZodNumber;
+        }, z.core.$loose>>;
     }, z.core.$loose>>;
     cursor: z.ZodNullable<z.ZodString>;
     notices: z.ZodArray<z.ZodString>;
@@ -687,6 +747,7 @@ export declare const CheckDetailSchema: z.ZodObject<{
                 height: z.ZodOptional<z.ZodNumber>;
                 durationMs: z.ZodOptional<z.ZodNumber>;
                 description: z.ZodOptional<z.ZodString>;
+                omitted: z.ZodOptional<z.ZodString>;
                 tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
                 at: z.ZodOptional<z.ZodObject<{
                     step: z.ZodOptional<z.ZodString>;
@@ -752,6 +813,7 @@ export declare const CheckDetailSchema: z.ZodObject<{
                     height: z.ZodOptional<z.ZodNumber>;
                     durationMs: z.ZodOptional<z.ZodNumber>;
                     description: z.ZodOptional<z.ZodString>;
+                    omitted: z.ZodOptional<z.ZodString>;
                     tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
                     at: z.ZodOptional<z.ZodObject<{
                         step: z.ZodOptional<z.ZodString>;
@@ -797,6 +859,7 @@ export declare const CheckDetailSchema: z.ZodObject<{
                 height: z.ZodOptional<z.ZodNumber>;
                 durationMs: z.ZodOptional<z.ZodNumber>;
                 description: z.ZodOptional<z.ZodString>;
+                omitted: z.ZodOptional<z.ZodString>;
                 tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
                 at: z.ZodOptional<z.ZodObject<{
                     step: z.ZodOptional<z.ZodString>;
@@ -849,6 +912,7 @@ export declare const CheckDetailSchema: z.ZodObject<{
             height: z.ZodOptional<z.ZodNumber>;
             durationMs: z.ZodOptional<z.ZodNumber>;
             description: z.ZodOptional<z.ZodString>;
+            omitted: z.ZodOptional<z.ZodString>;
             tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
             at: z.ZodOptional<z.ZodObject<{
                 step: z.ZodOptional<z.ZodString>;
@@ -896,6 +960,7 @@ export declare const CheckDetailSchema: z.ZodObject<{
             limit: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>>;
         durationMs: z.ZodOptional<z.ZodNumber>;
+        recordingExpectation: z.ZodOptional<z.ZodString>;
     }, z.core.$loose>;
     run: z.ZodObject<{}, z.core.$loose>;
 }, z.core.$loose>;

package/dist/schema/src/api.js

@@ -108,7 +108,11 @@ export const LayerHeaderSchema = z.looseObject({
     live: z.boolean(),
     counts: z.record(EvidenceStatusSchema, z.number().int().nonnegative()),
     execution: z.string(),
-    evidence: EvidenceSummarySchema.optional()
+    evidence: EvidenceSummarySchema.optional(),
+    recordings: z.looseObject({
+        expected: z.number().int().nonnegative(), uploaded: z.number().int().nonnegative(), referenced: z.number().int().nonnegative(),
+        skipped: z.number().int().nonnegative(), unresolved: z.number().int().nonnegative(), notApplicable: z.number().int().nonnegative()
+    }).optional().describe('How many of the recordings the layer expected were published. Absent: the layer declared no expectations, so its completeness is unknown, not complete')
 });
 /** One check of a layer as the report shows it. */
 export const LayerRowSchema = z.looseObject({
@@ -122,7 +126,8 @@ export const LayerRowSchema = z.looseObject({
     testId: z.string().nullable(),
     variant: z.record(z.string(), z.string()),
     durationMs: z.number().nullable(),
-    evidence: EvidenceSummarySchema.optional()
+    evidence: EvidenceSummarySchema.optional(),
+    recording: z.looseObject({ state: z.string(), reason: z.string().optional() }).optional().describe('What became of the check\'s expected recording: uploaded, referenced, skipped, unresolved or not-applicable. Independent of `status`')
 });
 /** A layer with its checks: the rows of one layer, read apart from its header. */
 export const LayerRowsSchema = z.looseObject({