@crvouga/atlas 1.14.0 → 1.15.0

package/CHANGELOG.md

@@ -1,6 +1,13 @@
 # Changelog: @crvouga/atlas
 
-## 1.14.0 (2026-10-29)
+## 1.15.0 (2026-10-29)
+
+### Features
+
+- count a video whose bundle file was never stored as unavailable ([6e342e1](https://github.com/crvouga/atlas/commit/6e342e1c0994b78499e373496afa5035a387a43c))
+- compare behavior promises, recordings and outcomes between two pinned runs ([9bc0056](https://github.com/crvouga/atlas/commit/9bc0056de84899801377c0cc1c3bb70c80e3820a))
+
+## 1.14.0 (2026-10-28)
 
 ### Features
 

package/dist/reports/src/evidence-detail.d.ts

@@ -46,11 +46,19 @@ export declare function summarizeEvidence(...holders: readonly EvidenceHolder[])
  * `effectiveScope`). Without `inherited` it is the same as `summarizeEvidence`.
  */
 export declare function summarizeEvidenceIn(inherited: EvidenceScope | undefined, ...holders: readonly EvidenceHolder[]): EvidenceSummary | undefined;
+/**
+ * Like {@link summarizeEvidenceIn}, knowing which bundle keys the layer names but does not hold (`unstored`, see
+ * `unstoredKeys`): a video whose only bytes are such a key is `available: false`, never a recording. Without
+ * `unstored` it is the same as `summarizeEvidenceIn`.
+ */
+export declare function summarizeStoredEvidence(inherited: EvidenceScope | undefined, unstored: ReadonlySet<string> | undefined, ...holders: readonly EvidenceHolder[]): EvidenceSummary | undefined;
 /**
  * Like {@link summarizeEvidence} for a layer and its parts: the same summary, but a video attached to the layer or a
  * part is owned by the `layer`, not by a check.
  */
 export declare function summarizeLayerEvidence(...holders: readonly EvidenceHolder[]): EvidenceSummary | undefined;
+/** Like {@link summarizeLayerEvidence}, knowing which bundle keys the layer names but does not hold (see {@link summarizeStoredEvidence}). */
+export declare function summarizeStoredLayerEvidence(unstored: ReadonlySet<string> | undefined, ...holders: readonly EvidenceHolder[]): EvidenceSummary | undefined;
 /**
  * Whether an attachment is a video, by its effective media type: the producer's `contentType`, else the one its file
  * name implies. A role never decides this on its own (`demo` on a PNG is not a video), except that a `video` role
@@ -113,3 +121,14 @@ export declare function partsEvidence(layer: {
 }): PartEvidence[];
 /** Every distinct bundle key (`bundleKey`) that the attachments of a holder, its steps and its attempts point to, so a layer's files can be copied. Attachments held by `uri` or inline `text` have none. */
 export declare function bundleKeysOf(holder: EvidenceHolder): string[];
+/**
+ * The bundle keys that attachments of a layer, its parts and their checks name, but that the layer's `bundle` index does
+ * not hold: files the producer referenced and never stored. Empty for a layer whose every file is there. This reads only
+ * the index, never a blob, so it is cheap enough to ask of every layer as it is read.
+ */
+export declare function unstoredKeys(layer: EvidenceHolder & {
+    readonly parts: readonly (EvidenceHolder & {
+        readonly checks: readonly EvidenceHolder[];
+    })[];
+    readonly bundle: Readonly<Record<string, unknown>>;
+}): string[];

package/dist/reports/src/evidence-detail.js

@@ -26,6 +26,14 @@ export function summarizeEvidence(...holders) {
 export function summarizeEvidenceIn(inherited, ...holders) {
     return summarize(holders, 'check', inherited);
 }
+/**
+ * Like {@link summarizeEvidenceIn}, knowing which bundle keys the layer names but does not hold (`unstored`, see
+ * `unstoredKeys`): a video whose only bytes are such a key is `available: false`, never a recording. Without
+ * `unstored` it is the same as `summarizeEvidenceIn`.
+ */
+export function summarizeStoredEvidence(inherited, unstored, ...holders) {
+    return summarize(holders, 'check', inherited, false, unstored);
+}
 /**
  * Like {@link summarizeEvidence} for a layer and its parts: the same summary, but a video attached to the layer or a
  * part is owned by the `layer`, not by a check.
@@ -34,7 +42,11 @@ export function summarizeLayerEvidence(...holders) {
     // The first holder is the layer: its scope reaches the parts after it, which may declare their own.
     return summarize(holders, 'layer', undefined, true);
 }
-function summarize(holders, owner, inherited, firstIsParent = false) {
+/** Like {@link summarizeLayerEvidence}, knowing which bundle keys the layer names but does not hold (see {@link summarizeStoredEvidence}). */
+export function summarizeStoredLayerEvidence(unstored, ...holders) {
+    return summarize(holders, 'layer', undefined, true, unstored);
+}
+function summarize(holders, owner, inherited, firstIsParent = false, unstored) {
     if (!holders.some(holder => count(holder) > 0))
         return undefined;
     const roles = new Set();
@@ -50,7 +62,7 @@ function summarize(holders, owner, inherited, firstIsParent = false) {
             for (const tag of normalizeTags(item.tags))
                 artifactTags.add(tag);
             roles.add(item.role ?? roleOfType(item.contentType ?? contentTypeOf(item.name)));
-            const video = videoOf(item, by, where, effectiveScope(item.scope, ...scopes));
+            const video = videoOf(item, by, where, effectiveScope(item.scope, ...scopes), unstored);
             if (video)
                 videos.push(video);
         }
@@ -98,10 +110,12 @@ export function isVideoAttachment(attachment) {
 export function attachmentIdentity(item) {
     return item.sha256 ? `sha256:${item.sha256}` : item.bundleKey ?? item.uri ?? (item.text === undefined ? undefined : `inline:${item.name}`);
 }
-function videoOf(item, owner, where, scope) {
+function videoOf(item, owner, where, scope, unstored) {
     if (!isVideoAttachment(item))
         return undefined;
-    const recording = attachmentIdentity(item);
+    // A bundle key the layer's bundle does not hold names bytes that are not there: only an address or inline text can still carry them, and the digest of lost bytes identifies nothing playable.
+    const lost = item.bundleKey !== undefined && unstored?.has(item.bundleKey) === true;
+    const recording = lost ? attachmentIdentity({ name: item.name, uri: item.uri, text: item.text }) : attachmentIdentity(item);
     const tags = normalizeTags(item.tags);
     return { name: item.name, owner, ...where, ...(tags.length ? { tags } : {}), ...(recording === undefined ? {} : { recording }), available: recording !== undefined, ...(scope ? { scope } : {}) };
 }
@@ -181,3 +195,16 @@ export function bundleKeysOf(holder) {
     }
     return [...keys];
 }
+/**
+ * The bundle keys that attachments of a layer, its parts and their checks name, but that the layer's `bundle` index does
+ * not hold: files the producer referenced and never stored. Empty for a layer whose every file is there. This reads only
+ * the index, never a blob, so it is cheap enough to ask of every layer as it is read.
+ */
+export function unstoredKeys(layer) {
+    const missing = new Set();
+    for (const holder of [layer, ...layer.parts, ...layer.parts.flatMap(part => part.checks)])
+        for (const key of bundleKeysOf(holder))
+            if (!Object.hasOwn(layer.bundle, key))
+                missing.add(key);
+    return [...missing];
+}

package/dist/reports/src/evidence-types.d.ts

@@ -56,7 +56,7 @@ export type VideoRef = {
     readonly tags?: readonly string[];
     /** What identifies the bytes (digest, bundle key or address); references with the same one are the same recording. Absent when `available` is false. */
     readonly recording?: string;
-    /** False when the reference names no bytes at all (no bundle key, address or inline text), so it can never be played. */
+    /** False when the reference names no bytes at all (no bundle key, address or inline text), or a bundle key that the layer's own bundle does not hold, so it can never be played. */
     readonly available: boolean;
     /** What this recording shows, as declared for it: its own `scope`, else its attempt's, check's, part's or layer's. Absent when none declared one: its scope is unknown, not live, physical or audible. */
     readonly scope?: EvidenceScope;
@@ -67,7 +67,7 @@ export type VideoCounts = {
     readonly references: number;
     /** Distinct playable recordings: identical bytes referenced twice are one. */
     readonly recordings: number;
-    /** References that name no bytes. A report that predates availability metadata has none of these and none of the others. */
+    /** References that name no bytes, or bytes the layer does not hold. A report that predates availability metadata has none of these and none of the others. */
     readonly unavailable: number;
 };
 /**

package/dist/reports/src/layer-merge.js

@@ -1,4 +1,4 @@
-import { summarizeEvidenceIn, summarizeLayerEvidence } from './evidence-detail.js';
+import { summarizeStoredEvidence, summarizeStoredLayerEvidence } from './evidence-detail.js';
 import { effectiveScope } from './evidence-scope.js';
 import { layerPlace } from './layer-axes.js';
 import { UI_PLAN_KIND, uiStateOf } from './ui-state.js';
@@ -90,13 +90,14 @@ export function layerEvidence(summary, backend) {
             normalized.set(path, found = specPath(path));
         return found;
     };
+    const unstored = summary.unstored?.length ? new Set(summary.unstored) : undefined;
     const rows = [];
     const receipt = summary.recordings;
     const expectations = receipt ? new Map(receipt.items.map(item => [item.check, item])) : null;
     for (const part of summary.parts) {
         for (const check of part.checks) {
             const test = tests.get(check.id);
-            const evidence = summarizeEvidenceIn(effectiveScope(part.scope, summary.scope), check);
+            const evidence = summarizeStoredEvidence(effectiveScope(part.scope, summary.scope), unstored, check);
             const item = expectations?.get(check.id);
             const state = uiStateOf(part.kind, check);
             rows.push({
@@ -117,7 +118,7 @@ export function layerEvidence(summary, backend) {
         }
     }
     const recording = summary.recording;
-    const layerSummaryEvidence = summarizeLayerEvidence(summary, ...summary.parts);
+    const layerSummaryEvidence = summarizeStoredLayerEvidence(unstored, summary, ...summary.parts);
     const layer = {
         ...(summary.freshness ? { freshness: summary.freshness } : {}),
         ...(summary.run ? { run: summary.run } : {}),

package/dist/reports/src/layer-store.d.ts

@@ -48,5 +48,5 @@ export declare function publishLayerBundle(store: ObjectStore, summary: Omit<Lay
 export declare function readLayer(store: ObjectStore, id: string): Promise<ReportLayer | null>;
 /** Reads one captured file (a screenshot, a frame, a recorded report) of a layer, or `null` when the layer or the file is missing. Throws like `readLayer`. */
 export declare function readLayerFile(store: ObjectStore, id: string, file: string): Promise<BundleEntry | null>;
-/** Drops a layer's bundle index, leaving the summary composition reads. */
+/** Drops a layer's bundle index, leaving the summary composition reads, plus the bundle keys its attachments name that the bundle does not hold (`unstored`), so a video that was never stored is not counted as a recording. */
 export declare function layerSummary(layer: ReportLayer): LayerSummary;

package/dist/reports/src/layer-store.js

@@ -10,6 +10,7 @@
  */
 import { createBundleReader, writeBundle } from './bundle.js';
 import { readUtf8, writeUtf8 } from './object-io.js';
+import { unstoredKeys } from './evidence-detail.js';
 import { LAYER_SCHEMA_VERSION, parseLayerDocument } from '../../schema/src/layer.js';
 /** The key prefix under which every layer's catalog entry is written. */
 export const LAYER_INDEX = 'layers/index/';
@@ -87,8 +88,9 @@ export async function readLayerFile(store, id, file) {
     const layer = await readLayer(store, id);
     return layer ? createBundleReader(store, layer.bundle)(file) : null;
 }
-/** Drops a layer's bundle index, leaving the summary composition reads. */
+/** Drops a layer's bundle index, leaving the summary composition reads, plus the bundle keys its attachments name that the bundle does not hold (`unstored`), so a video that was never stored is not counted as a recording. */
 export function layerSummary(layer) {
     const { bundle: _, ...summary } = layer;
-    return summary;
+    const unstored = unstoredKeys(layer);
+    return unstored.length ? { ...summary, unstored } : summary;
 }

package/dist/reports/src/layer-types.d.ts

@@ -52,6 +52,8 @@ export type LayerSummary = LayerExtras & {
         readonly sha: string;
         readonly source: string;
     } | null;
+    /** Bundle keys that attachments name but the layer's bundle does not hold, read from the layer's index when its summary was made (see `unstoredKeys`). Absent when there are none. In memory only: never part of the layer document. */
+    readonly unstored?: readonly string[];
 };
 /** A layer with the index that locates its captured files (screenshots, frames) in the store. */
 export type ReportLayer = LayerSummary & {

package/dist/reports/src/run-compare.d.ts

@@ -0,0 +1,182 @@
+import { type LatencyChange, type LatencyEntry, type LatencySummary } from './latency.js';
+import type { ReliabilityDocument, ReliabilityRate, ReliabilityScope } from './reliability.js';
+import type { RunDetail, RunResult, RunSummary } from './test-catalog.js';
+/** Where a pair of results (or one result with no partner) stands, in the order a review reads them. */
+export declare const COMPARE_GROUPS: readonly ["regressed", "recovered", "still-failing", "not-comparable", "unknown", "removed", "added", "unchanged"];
+/** One of {@link COMPARE_GROUPS}. */
+export type CompareGroup = (typeof COMPARE_GROUPS)[number];
+/** What each group is called and what it does and does not say. */
+export declare const COMPARE_GROUP_TEXT: Readonly<Record<CompareGroup, {
+    readonly label: string;
+    readonly meaning: string;
+}>>;
+/**
+ * Whether the promise (what the behavior says should happen) is the same on both sides: `same` (same wording and the
+ * same recorded spec revision), `changed` (the wording differs), `revision-differs` (same wording, another spec-file
+ * revision), or `unknown` (a revision was not recorded on a side, so nothing can be said).
+ */
+export type PromiseState = 'same' | 'changed' | 'revision-differs' | 'unknown';
+/** What is known about whether the promise changed, and why. */
+export type PromiseReading = {
+    readonly state: PromiseState;
+    readonly baselineTitle: string | null;
+    readonly candidateTitle: string | null;
+    readonly baselineRevision: string | null;
+    readonly candidateRevision: string | null;
+    /** In words, for a person. */
+    readonly reasons: readonly string[];
+};
+/** The condition a result was recorded under: with the check, what makes it its own result. */
+export type Condition = {
+    readonly kind: string | null;
+    readonly producer: string;
+    readonly environment: string | null;
+    readonly parameters: Readonly<Record<string, string>>;
+};
+/** The condition of a result. A server that does not send `kind` leaves it null: unknown, equal only to another unknown. */
+export declare function conditionOf(result: Pick<RunResult, 'kind' | 'producer' | 'environment' | 'parameters'>): Condition;
+/** A stable text for a condition, equal exactly when two conditions are. */
+export declare function conditionId(condition: Condition): string;
+/** How two conditions differ, in words; empty when they are the same. */
+export declare function conditionDifferences(baseline: Condition, candidate: Condition): string[];
+/** One behavior's results on the two sides. */
+export type ComparedResult = {
+    /** Identifies the pair within one comparison: the check, and the condition of each side. Stable for the same two runs. */
+    readonly id: string;
+    /** The check's identity (`TestSummary.key`). */
+    readonly key: string;
+    /** The check's title: the candidate's when it has one, else the baseline's. */
+    readonly title: string;
+    readonly specPath: string | null;
+    readonly baseline: RunResult | null;
+    readonly candidate: RunResult | null;
+    readonly group: CompareGroup;
+    /** One sentence saying what happened, and what is not known. */
+    readonly summary: string;
+    readonly promise: PromiseReading;
+    /** How the two conditions differ, in words, when both sides have a result and they were recorded under different conditions. */
+    readonly conditions: readonly string[];
+    /** What a reader must not assume about this pair, in words. */
+    readonly caveats: readonly string[];
+};
+/** The whole comparison of two pinned runs. */
+export type RunComparison = {
+    readonly baseline: RunSummary;
+    readonly candidate: RunSummary;
+    /** Every pair and every unpaired result, in the order of {@link COMPARE_GROUPS}, then spec and title. */
+    readonly results: readonly ComparedResult[];
+    readonly counts: Readonly<Record<CompareGroup, number>>;
+    /** What a reader must not assume about the two runs as a whole: the same run on both sides, different branches, a candidate older than the baseline. */
+    readonly notes: readonly string[];
+};
+/** Short form of a spec fingerprint, to show. */
+export declare function shortRevision(revision: string | null | undefined): string;
+/** The wording a run recorded for a check: what that run's layer called it, else the check's newest title. */
+export declare function recordedTitle(result: Pick<RunResult, 'title' | 'recordedTitle'>): string;
+/**
+ * Pairs the results of two runs and reads what changed. Results are paired by check identity and then by condition (see
+ * the module comment); titles never pair two results. Each side's results are those the run recorded last for each
+ * check and condition (`RunDetail.results`).
+ *
+ * @param baseline - The run to compare against.
+ * @param candidate - The run under review. Both are used as given: a newer run never replaces either.
+ * @returns Every result of either run, grouped by what changed, with the reasons.
+ * @example
+ * ```ts
+ * const comparison = compareRuns(await runDetail('run:ci-100'), await runDetail('run:ci-101'));
+ * comparison.results.filter(item => item.group === 'regressed');
+ * ```
+ */
+export declare function compareRuns(baseline: RunDetail, candidate: RunDetail): RunComparison;
+/** What one product-latency metric shows on the two sides. */
+export type MetricComparison = {
+    readonly state: 'compared';
+    readonly name: string;
+    readonly baseline: LatencySummary;
+    readonly candidate: LatencySummary;
+    readonly change: LatencyChange;
+    readonly reasons: readonly string[];
+} | {
+    readonly state: 'incompatible';
+    readonly name: string;
+    readonly baseline: LatencySummary;
+    readonly candidate: LatencySummary;
+    readonly change: LatencyChange | null;
+    readonly reasons: readonly string[];
+} | {
+    readonly state: 'baseline-only' | 'candidate-only';
+    readonly name: string;
+    readonly summary: LatencySummary;
+    readonly reasons: readonly string[];
+};
+/**
+ * Compares the product latency recorded in two sets of results of one check, one per side. A metric is compared only
+ * when it is the same measurement (name, unit, boundaries) and both sides were measured on the same devices, in the
+ * same environments, with the same instrumentation and other conditions; otherwise it is `incompatible` and no change
+ * is computed, with the reasons. A metric only one side recorded is named as such. A side without any latency is unknown,
+ * not fast: it contributes no metric. `conditionsDiffer` names differences of the results' own conditions, which make
+ * every metric incompatible.
+ */
+export declare function compareProductLatency(baseline: readonly LatencyEntry[], candidate: readonly LatencyEntry[], conditionsDiffer?: readonly string[]): MetricComparison[];
+/** How reliable a check was on each side, and whether the two figures can be set against each other. */
+export type ReliabilityComparison = {
+    /** True only when both figures are established, were counted in the same scope and against the same recorded spec revision. */
+    readonly comparable: boolean;
+    readonly baseline: ReliabilityRate;
+    readonly candidate: ReliabilityRate;
+    /** Candidate minus baseline, in percentage points, only when comparable. */
+    readonly points?: number;
+    /** `higher` or `lower` only when the two intervals do not overlap; `indistinguishable` when they do. Absent when not comparable. */
+    readonly direction?: 'higher' | 'lower' | 'indistinguishable';
+    /** Why no change is stated, or what qualifies one. */
+    readonly reasons: readonly string[];
+};
+/**
+ * The scope that counts the independent executions of one result's check up to the end of its run: the same
+ * environment, producer and parameters, and the spec revision that result was recorded against (executions against
+ * another revision are other promises). A result that recorded no revision is counted across revisions, which
+ * {@link compareReliability} will not compare. Use it for each side, with that side's result and run.
+ */
+export declare function reliabilityScopeAt(result: Pick<RunResult, 'environment' | 'producer' | 'parameters' | 'specRevision'>, run: Pick<RunSummary, 'endedAtMs'>): ReliabilityScope;
+/**
+ * Sets the reliability of a check up to the baseline run against its reliability up to the candidate run. A change is
+ * stated only when both samples are established (at least `minimumExecutions` independent executions), both counted the
+ * same recorded spec revision (see {@link reliabilityScopeAt}) and the conditions match; whether the difference exceeds the uncertainty is
+ * said by `direction`. Otherwise each figure stands alone and the reasons say why.
+ */
+export declare function compareReliability(baseline: ReliabilityDocument, candidate: ReliabilityDocument, conditionsDiffer?: readonly string[]): ReliabilityComparison;
+/** The part of a recorded step this module reads. */
+export type StepLike = {
+    readonly name: string;
+    readonly offsetMs?: number | undefined;
+};
+/** One step, found on one or both sides. */
+export type AlignedStep = {
+    readonly name: string;
+    /** Milliseconds after the check began, on each side; absent when that side did not record it or its time. */
+    readonly baselineOffsetMs?: number;
+    readonly candidateOffsetMs?: number;
+    readonly onBaseline: boolean;
+    readonly onCandidate: boolean;
+};
+/** How the recorded steps of two results line up. */
+export type StepAlignment = {
+    /** `aligned`: some step is on both sides with a time on both (a recording can be sought to it); `unaligned`: otherwise. */
+    readonly state: 'aligned' | 'unaligned';
+    readonly steps: readonly AlignedStep[];
+    /** The steps with a time on both sides: where both recordings can be sought together. */
+    readonly seekable: readonly (AlignedStep & {
+        readonly baselineOffsetMs: number;
+        readonly candidateOffsetMs: number;
+    })[];
+    /** Whether the two lists of step names differ (the steps themselves are part of the promise). */
+    readonly differs: boolean;
+    /** In words, why the recordings are or are not aligned. */
+    readonly explanation: string;
+};
+/**
+ * Lines up the recorded steps of two results by name (the n-th step of a name on one side with the n-th of that name on
+ * the other), keeping each side's own time origin: an offset is milliseconds after that side's check began. Nothing is
+ * aligned by position or by fuzzy wording. When nothing lines up, `explanation` says why.
+ */
+export declare function alignSteps(baseline: readonly StepLike[], candidate: readonly StepLike[]): StepAlignment;