@crvouga/atlas 0.16.0 → 0.17.0

package/docs/reference/reports.md

@@ -276,7 +276,9 @@ recursively), and hands their bytes over as `data` with a `contentType` guessed
 attachment gave none.
 
 A file that is missing, not allowed or too large is left out with a warning: evidence that could not be
-found never stops the results from being published. Attachments without a `path` are kept untouched.
+found never stops the results from being published. The exception in kind is a video of a check that declared
+`recordingExpectation: "required"`: it stays as an attachment without bytes whose `omitted` says why (`not-found`,
+`not-allowed`, `too-large`, `over-budget`), so the recording receipt can name the gap. Attachments without a `path` are kept untouched.
 
 - `holder`: A check, step, attempt or any object that may have `attachments`, `steps` and `attempts`.
 
@@ -1747,11 +1749,34 @@ export type LayerHeader = {
   readonly execution: string;
   /** What the run as a whole carries: its logs, links and labels. */
   readonly evidence?: EvidenceSummary;
+  /** Counts from the layer's recording receipt: how many expected recordings were published. Absent when the layer declared no expectations: its completeness is unknown, not complete. Never changes a status. */
+  readonly recordings?: RecordingCounts;
 };
 ```
 
 A report layer's origin and totals. Its checks are read separately, once.
 
+#### `RecordingCounts`
+
+```ts
+export type RecordingCounts = {
+  /** Checks that required a recording: `uploaded + referenced + skipped + unresolved`. */
+  readonly expected: number;
+  /** Stored in the layer, with a checksum. */
+  readonly uploaded: number;
+  /** Given only by `uri`: Atlas cannot vouch for it. */
+  readonly referenced: number;
+  /** Captured but left out, with a reason (too large, not allowed…). */
+  readonly skipped: number;
+  /** No video was attached at all. */
+  readonly unresolved: number;
+  /** Intentionally unrecorded checks: neither expected nor missing. */
+  readonly notApplicable: number;
+};
+```
+
+How many recordings a layer expected and what became of them (see `RecordingReceipt`).
+
 #### `LayerRow`
 
 ```ts
@@ -1775,6 +1800,8 @@ export type LayerRow = {
   readonly durationMs: number | null;
   /** What this result carries beyond its outcome. Absent when nothing. */
   readonly evidence?: EvidenceSummary;
+  /** What became of this check's expected recording: `uploaded`, `referenced`, `skipped`, `unresolved` or `not-applicable`, with a reason when it is not uploaded. Absent when the check declared no expectation. Independent of `status`. */
+  readonly recording?: { readonly state: string; readonly reason?: string };
 };
 ```
 
@@ -3983,7 +4010,7 @@ String choices (such as `{ device: 'iphone-15' }`) that make a result its own sl
 #### `LayerExtras`
 
 ```ts
-export type LayerExtras = Partial<Pick<import('../../schema/src/layer').LayerDocument, 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions'>>;
+export type LayerExtras = Partial<Pick<import('../../schema/src/layer').LayerDocument, 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions' | 'recordings'>>;
 ```
 
 What a layer may carry as a whole. All optional and open: see docs/extending.md.
@@ -5381,6 +5408,69 @@ export function contentTypeForMedia(media: string): string
 
 The media type for a snapshot media kind (`html`, `png`, `json`, `text`, `bytes`, `video`); an unknown kind is `application/octet-stream`.
 
+### Recording receipts
+
+Source: `packages/reports/src/recording-receipt.ts`. Import: `@crvouga/atlas/reports/recording-receipt`
+
+Answers "did the recordings we expected get published?" apart from "did the tests pass?". `recordingReceipt` reads the
+checks a producer is about to publish (after their files were stored) and describes, per check that declared a
+`recordingExpectation`, whether its video was uploaded, only referenced, left out (with a reason) or never attached.
+`recordingCompleteness` reduces the winning results of a report to the figures `atlas report status` prints. Neither
+reads a clock or the environment, and neither changes a status, a slot or a winner: a receipt describes publication, it
+does not decide a result. A layer with no receipt is Unknown, never Complete.
+
+See docs/rich-evidence.md#recording-completeness
+
+#### `isVideoAttachment`
+
+```ts
+export function isVideoAttachment(attachment: Loose): boolean
+```
+
+Is this attachment a screen recording: a role of `video`, or a `video/*` media type?
+
+#### `recordingReceipt`
+
+```ts
+export function recordingReceipt(checks: readonly Loose[]): RecordingReceipt | undefined
+```
+
+The receipt for the checks of one layer, or `undefined` when no check declares a `recordingExpectation` (the layer then
+claims nothing). Call it with checks whose attachment bytes are already stored (`bundleKey`, `bytes` and `sha256` set).
+
+A `required` check is `uploaded` when any video of it has stored bytes, `referenced` when one has only a `uri`,
+`skipped` when its videos were left out (the attachment's `omitted` is the reason) and `unresolved` when it has no
+video at all. A `not-applicable` check is listed but is neither expected nor missing. Items are sorted by check id so the
+same input is always the same receipt.
+
+#### `RecordingResult`
+
+```ts
+export type RecordingResult = {
+  readonly spec: string | null;
+  readonly check: string;
+  readonly environment: string | null;
+  readonly layer: string;
+  /** Whether the layer the result came from carried a receipt. */
+  readonly hasReceipt: boolean;
+  /** The result's item in that receipt, when its check declared an expectation. */
+  readonly recording?: { readonly state: string; readonly reason?: string };
+};
+```
+
+One result as {@link recordingCompleteness} reads it.
+
+#### `recordingCompleteness`
+
+```ts
+export function recordingCompleteness(results: readonly RecordingResult[]): RecordingCompleteness
+```
+
+Reduces the winning results of a report to its recording completeness. `incomplete` when any expected recording is
+skipped or unresolved; otherwise `unknown` when a contributing layer has no receipt (nothing is claimed for it, and a
+report with no results at all is unknown too); otherwise `complete` when something was expected and `none-expected` when
+nothing was. Gaps are sorted so the output never depends on read order.
+
 ### Remote commit report
 
 Source: `packages/reports/src/remote-report.ts`. Import: `@crvouga/atlas/reports/remote-report`
@@ -5756,6 +5846,8 @@ checks (or none completed), failed checks, running checks, specs without results
 each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
 report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
 not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+`recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
+completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
 
 **Example**
 

package/docs/reference/schema.md

@@ -764,6 +764,47 @@ export type Attachment = z.output<typeof AttachmentSchema>;
 
 An attachment as stored in a layer, parsed from `AttachmentSchema`.
 
+#### `RECORDING_EXPECTATIONS`
+
+```ts
+export const RECORDING_EXPECTATIONS = …
+```
+
+Whether a check is expected to have a screen recording (a video). `required`: the producer expects one, and the layer's
+recording receipt says whether it arrived. `not-applicable`: the check is intentionally unrecorded (an API-only check),
+so it counts as neither a recorded pass nor a missing video. An open string: a reader treats any other value, and its
+absence, as no expectation.
+
+#### `RecordingItemSchema`
+
+```ts
+export const RecordingItemSchema = …
+```
+
+The state of one check's recording in a receipt, in an open vocabulary. Conventions: `uploaded` (the video's bytes are in
+the layer, with a checksum), `referenced` (only a `uri` was given: Atlas cannot vouch for it), `skipped` (the video was
+captured but its bytes were left out, with a reason), `unresolved` (no video was attached at all), `not-applicable`
+(intentionally unrecorded). A reader shows a state it does not know as it is and never counts it as uploaded.
+
+#### `RecordingReceiptSchema`
+
+```ts
+export const RecordingReceiptSchema = …
+```
+
+What a layer's producer published about the recordings it expected: counts and one item per check that expected a
+recording (or declared itself not applicable). Written by `publishChecks` when any check declares a
+`recordingExpectation`; a layer without it says nothing, which a reader reports as Unknown, never Complete. It only
+describes publication: it never changes a check's status, its slot or which result wins.
+
+#### `RecordingReceipt`
+
+```ts
+export type RecordingReceipt = z.output<typeof RecordingReceiptSchema>;
+```
+
+A recording receipt as stored in a layer, parsed from `RecordingReceiptSchema`.
+
 #### `AnnotationSchema`
 
 ```ts
@@ -962,6 +1003,24 @@ export type PublishChecksInput = z.input<typeof PublishChecksSchema>;
 
 What a caller passes to `publishChecks`: the input type of `PublishChecksSchema`, before defaults.
 
+#### `RecordingCompletenessSchema`
+
+```ts
+export const RecordingCompletenessSchema = …
+```
+
+Recording completeness across the winning results of a report: `complete` (every expected recording was published),
+`incomplete` (at least one is skipped or unresolved), `none-expected` (every layer declared its expectations, and none
+required a recording) or `unknown` (a layer has no receipt, so nothing is claimed). Independent of the checks' outcomes.
+
+#### `RecordingCompleteness`
+
+```ts
+export type RecordingCompleteness = z.output<typeof RecordingCompletenessSchema>;
+```
+
+A parsed `RecordingCompletenessSchema` value.
+
 #### `REPORT_STATUS_SCHEMA_VERSION`
 
 ```ts

package/docs/rich-evidence.md

@@ -224,3 +224,32 @@ or the command line, an attachment may name a `path` instead; `atlas report publ
 ([attach evidence](extending.md#attach-evidence)). A CTRF report already carries most of this, and
 `atlas report publish --ctrf` keeps it ([native runner reporters](runners.md) cover CodeceptJS, Mocha, Jest and Node's
 test runner).
+
+## Recording completeness
+
+A green run does not say whether the videos you expected were kept and published. Atlas answers that separately from
+the checks' outcomes. Mark a check with `recordingExpectation`: `"required"` (expect a video of it) or
+`"not-applicable"` (an API-only or intentionally unrecorded check). A check with neither is not part of the answer.
+
+```json
+{ "id": "checkout#pays", "title": "Pays", "status": "passed", "recordingExpectation": "required",
+  "attachments": [{ "name": "pays.webm", "role": "video", "path": "test-results/pays.webm" }] }
+```
+
+When any check carries an expectation, `publishChecks` and `atlas report publish` write a receipt into the layer
+(`recordings`): counts of expected, uploaded, referenced (only a `uri`, which Atlas cannot vouch for), skipped and
+unresolved recordings, and for each expected check its state, a reason when it is not uploaded, and the size and SHA-256
+of every stored video. A video over the size limit, or a file that is missing or not allowed, is kept as an attachment
+without bytes whose `omitted` says why (`too-large`, `not-found`, `not-allowed`, `over-budget`) instead of failing the
+publish; it never contains a path or a token.
+
+- `atlas report publish --format json` prints the receipt under `layer.recordings` and a summary under `recordings`.
+  Add `--require-recordings` to exit 1 when an expected recording is skipped or unresolved. The layer and its test
+  results are still written and are not changed; only the command fails, so a CI step can report the evidence gap and
+  retry (publishing again reuses the stored files).
+- `atlas report status --format json` has `recordings`: `state` is `complete`, `incomplete`, `none-expected` or
+  `unknown`, with counts and each gap. It never changes `ok`. A layer without a receipt is `unknown`, never `complete`.
+- The viewer shows `Recordings 2 of 3: incomplete` beside the run's origin, apart from its green or red outcomes.
+
+This is a statement about publication. Atlas does not check that a stored video plays in a browser, and does not yet
+re-check later that a stored file is still readable.

package/llms-full.txt

@@ -1211,6 +1211,35 @@ or the command line, an attachment may name a `path` instead; `atlas report publ
 ([attach evidence](extending.md#attach-evidence)). A CTRF report already carries most of this, and
 `atlas report publish --ctrf` keeps it ([native runner reporters](runners.md) cover CodeceptJS, Mocha, Jest and Node's
 test runner).
+
+## Recording completeness
+
+A green run does not say whether the videos you expected were kept and published. Atlas answers that separately from
+the checks' outcomes. Mark a check with `recordingExpectation`: `"required"` (expect a video of it) or
+`"not-applicable"` (an API-only or intentionally unrecorded check). A check with neither is not part of the answer.
+
+```json
+{ "id": "checkout#pays", "title": "Pays", "status": "passed", "recordingExpectation": "required",
+  "attachments": [{ "name": "pays.webm", "role": "video", "path": "test-results/pays.webm" }] }
+```
+
+When any check carries an expectation, `publishChecks` and `atlas report publish` write a receipt into the layer
+(`recordings`): counts of expected, uploaded, referenced (only a `uri`, which Atlas cannot vouch for), skipped and
+unresolved recordings, and for each expected check its state, a reason when it is not uploaded, and the size and SHA-256
+of every stored video. A video over the size limit, or a file that is missing or not allowed, is kept as an attachment
+without bytes whose `omitted` says why (`too-large`, `not-found`, `not-allowed`, `over-budget`) instead of failing the
+publish; it never contains a path or a token.
+
+- `atlas report publish --format json` prints the receipt under `layer.recordings` and a summary under `recordings`.
+  Add `--require-recordings` to exit 1 when an expected recording is skipped or unresolved. The layer and its test
+  results are still written and are not changed; only the command fails, so a CI step can report the evidence gap and
+  retry (publishing again reuses the stored files).
+- `atlas report status --format json` has `recordings`: `state` is `complete`, `incomplete`, `none-expected` or
+  `unknown`, with counts and each gap. It never changes `ok`. A layer without a receipt is `unknown`, never `complete`.
+- The viewer shows `Recordings 2 of 3: incomplete` beside the run's origin, apart from its green or red outcomes.
+
+This is a statement about publication. Atlas does not check that a stored video plays in a browser, and does not yet
+re-check later that a stored file is still readable.
 </doc>
 
 <doc path="docs/server.md">
@@ -4315,6 +4344,7 @@ atlas report — layered evidence for any kind of spec
                        [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]
                        [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...
                        [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]
+                       [--require-recordings]
   atlas report stream  [--store <dir|s3://…|https://atlas.example.com>] --producer <name> [--environment <name>]
                        [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...
                        [--run-id <id>] [--run-title <t>]
@@ -4351,6 +4381,10 @@ atlas report — layered evidence for any kind of spec
             Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own
             folder and each --attach-root, and never from a hidden file or a link out: a report is data from a
             test run, not instructions. One that is missing or not allowed is skipped with a warning.
+            --require-recordings   exit 1 when a check that declared recordingExpectation "required" has no
+                              uploaded video (absent, left out as too large or not allowed, or refused). The layer is
+                              still written and the checks keep their results; only this command fails. The receipt of
+                              what was uploaded (counts, sizes, checksums) is in the JSON result under layer.recordings.
             From a checkout with uncommitted changes the layer is marked "modified".
   stream    Send checks as they finish: one JSON object per line on standard input (the fields of one
             check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crvouga/atlas",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Atlas: specs of any kind, reported as layered evidence. Publish checks from any runner (JUnit, CTRF, a script), combine them across nodes, branches, environments and backends, gate CI on the result, and run statecharts against the real app.",
   "license": "MIT",
   "type": "module",

package/packages/schema/schemas/check-detail.schema.json

@@ -73,6 +73,10 @@
           "type": "number",
           "minimum": 0
         },
+        "recordingExpectation": {
+          "description": "`required`: expect a video of this check, and report whether it was published. `not-applicable`: intentionally unrecorded. Absent: no expectation. Never changes the check's status",
+          "type": "string"
+        },
         "tags": {
           "description": "Free labels to filter and group by: smoke, slow, flaky…",
           "type": "array",
@@ -169,6 +173,10 @@
               "description": {
                 "type": "string"
               },
+              "omitted": {
+                "description": "Why the producer did not include its bytes, in an open vocabulary: too-large, not-found, not-allowed, over-budget, policy… Present on a metadata-only attachment, so a reader can tell \"left out\" from \"never recorded\"",
+                "type": "string"
+              },
               "tags": {
                 "description": "Free labels of this one attachment, such as `demo`: shown as `#demo`, searchable, and normalized by `normalizeTag` (a leading `#` and letter case never matter). Unlike the tags of a check they say nothing about its result",
                 "type": "array",
@@ -404,6 +412,10 @@
                     "description": {
                       "type": "string"
                     },
+                    "omitted": {
+                      "description": "Why the producer did not include its bytes, in an open vocabulary: too-large, not-found, not-allowed, over-budget, policy… Present on a metadata-only attachment, so a reader can tell \"left out\" from \"never recorded\"",
+                      "type": "string"
+                    },
                     "tags": {
                       "description": "Free labels of this one attachment, such as `demo`: shown as `#demo`, searchable, and normalized by `normalizeTag` (a leading `#` and letter case never matter). Unlike the tags of a check they say nothing about its result",
                       "type": "array",
@@ -676,6 +688,10 @@
                           "description": {
                             "type": "string"
                           },
+                          "omitted": {
+                            "description": "Why the producer did not include its bytes, in an open vocabulary: too-large, not-found, not-allowed, over-budget, policy… Present on a metadata-only attachment, so a reader can tell \"left out\" from \"never recorded\"",
+                            "type": "string"
+                          },
                           "tags": {
                             "description": "Free labels of this one attachment, such as `demo`: shown as `#demo`, searchable, and normalized by `normalizeTag` (a leading `#` and letter case never matter). Unlike the tags of a check they say nothing about its result",
                             "type": "array",
@@ -883,6 +899,10 @@
                     "description": {
                       "type": "string"
                     },
+                    "omitted": {
+                      "description": "Why the producer did not include its bytes, in an open vocabulary: too-large, not-found, not-allowed, over-budget, policy… Present on a metadata-only attachment, so a reader can tell \"left out\" from \"never recorded\"",
+                      "type": "string"
+                    },
                     "tags": {
                       "description": "Free labels of this one attachment, such as `demo`: shown as `#demo`, searchable, and normalized by `normalizeTag` (a leading `#` and letter case never matter). Unlike the tags of a check they say nothing about its result",
                       "type": "array",

package/packages/schema/schemas/effective-delta.schema.json

@@ -207,6 +207,22 @@
                     "roles"
                   ],
                   "additionalProperties": {}
+                },
+                "recording": {
+                  "description": "What became of the check's expected recording: uploaded, referenced, skipped, unresolved or not-applicable. Independent of `status`",
+                  "type": "object",
+                  "properties": {
+                    "state": {
+                      "type": "string"
+                    },
+                    "reason": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "state"
+                  ],
+                  "additionalProperties": {}
                 }
               },
               "required": [
@@ -481,6 +497,51 @@
                   "roles"
                 ],
                 "additionalProperties": {}
+              },
+              "recordings": {
+                "description": "How many of the recordings the layer expected were published. Absent: the layer declared no expectations, so its completeness is unknown, not complete",
+                "type": "object",
+                "properties": {
+                  "expected": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "uploaded": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "referenced": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "skipped": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "unresolved": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "notApplicable": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  }
+                },
+                "required": [
+                  "expected",
+                  "uploaded",
+                  "referenced",
+                  "skipped",
+                  "unresolved",
+                  "notApplicable"
+                ],
+                "additionalProperties": {}
               }
             },
             "required": [

package/packages/schema/schemas/effective-evidence.schema.json

@@ -207,6 +207,22 @@
                     "roles"
                   ],
                   "additionalProperties": {}
+                },
+                "recording": {
+                  "description": "What became of the check's expected recording: uploaded, referenced, skipped, unresolved or not-applicable. Independent of `status`",
+                  "type": "object",
+                  "properties": {
+                    "state": {
+                      "type": "string"
+                    },
+                    "reason": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "state"
+                  ],
+                  "additionalProperties": {}
                 }
               },
               "required": [
@@ -481,6 +497,51 @@
                   "roles"
                 ],
                 "additionalProperties": {}
+              },
+              "recordings": {
+                "description": "How many of the recordings the layer expected were published. Absent: the layer declared no expectations, so its completeness is unknown, not complete",
+                "type": "object",
+                "properties": {
+                  "expected": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "uploaded": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "referenced": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "skipped": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "unresolved": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "notApplicable": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  }
+                },
+                "required": [
+                  "expected",
+                  "uploaded",
+                  "referenced",
+                  "skipped",
+                  "unresolved",
+                  "notApplicable"
+                ],
+                "additionalProperties": {}
               }
             },
             "required": [

package/packages/schema/schemas/evidence-page.schema.json

@@ -252,6 +252,51 @@
               "roles"
             ],
             "additionalProperties": {}
+          },
+          "recordings": {
+            "description": "How many of the recordings the layer expected were published. Absent: the layer declared no expectations, so its completeness is unknown, not complete",
+            "type": "object",
+            "properties": {
+              "expected": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "uploaded": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "referenced": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "skipped": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "unresolved": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "notApplicable": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              }
+            },
+            "required": [
+              "expected",
+              "uploaded",
+              "referenced",
+              "skipped",
+              "unresolved",
+              "notApplicable"
+            ],
+            "additionalProperties": {}
           }
         },
         "required": [

package/packages/schema/schemas/layer-rows.schema.json

@@ -198,6 +198,22 @@
               "roles"
             ],
             "additionalProperties": {}
+          },
+          "recording": {
+            "description": "What became of the check's expected recording: uploaded, referenced, skipped, unresolved or not-applicable. Independent of `status`",
+            "type": "object",
+            "properties": {
+              "state": {
+                "type": "string"
+              },
+              "reason": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "state"
+            ],
+            "additionalProperties": {}
           }
         },
         "required": [