@crvouga/atlas 1.7.0 → 1.9.0

package/docs/reference/reports.md

@@ -2145,6 +2145,8 @@ export type EvidenceSummary = {
   readonly annotations: number;
   /** How many tries were recorded when the check was retried. */
   readonly attempts: number;
+  /** The status of each recorded try, in the order they were made (its length is `attempts`). Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed. */
+  readonly attemptStatuses?: readonly EvidenceStatus[];
   readonly links: number;
   readonly tags: readonly string[];
   /** The normalized tags of the attachments themselves (`#demo` is `demo`), sorted and distinct. Absent when no attachment is tagged. Unlike `tags` they are never inherited from a check or run. */
@@ -2608,6 +2610,22 @@ export type EvidenceMeasurement = {
   readonly unit: string;
   /** The allowance for `value`, in the same unit, when there is one. */
   readonly limit?: number;
+  /** What the number is, in an open vocabulary. `product-latency` is a user-visible response time between `from` and `to`; absent is just a number. */
+  readonly kind?: string;
+  /** The event the measured interval starts at. */
+  readonly from?: string;
+  /** The event the measured interval ends at. */
+  readonly to?: string;
+  /** The interval the name claims, when the measured one is only part of it. */
+  readonly intended?: { readonly from?: string; readonly to?: string };
+  /** How it was measured. */
+  readonly source?: string;
+  /** `observed` (default), `timeout`, `failed`; otherwise `value` is how long the observation waited, at least. */
+  readonly outcome?: string;
+  /** The version or method of the measurement. */
+  readonly instrumentation?: string;
+  /** What the observation is comparable with: `device`, `build`. */
+  readonly labels?: Readonly<Record<string, string>>;
 };
 ```
 
@@ -4073,6 +4091,335 @@ export function splitViewerTestId(
 
 The inverse of {@link viewerTestId}: splits at the first `/`; null when either side is not a safe key segment.
 
+### Product latency
+
+Source: `packages/reports/src/latency.ts`. Import: `@crvouga/atlas/reports/latency`
+
+Tells how quickly a product responded apart from how long a test took. A test's `durationMs` includes fixture setup,
+deliberate waits, demo pacing, retries and cleanup, so it says nothing about a user's wait. Product latency is only
+what a producer measured and named as such: a measurement of `kind` `product-latency` between two named events
+(`from`, `to`), optionally with a budget (`limit`), an outcome (`observed`, `timeout`, `failed`), the instrumentation,
+and labels (`device`, `build`) in its `labels`. Pure functions, safe in a browser.
+
+The rules, each of which keeps a number from looking healthier or more certain than the evidence is:
+
+- **Only named measurements are latency.** A check's duration, a video's length, a recorded step and a number with no
+  `kind` are never product latency. No latency recorded is unknown, not fast.
+- **Boundaries are part of the claim.** Without both `from` and `to`, nothing is claimed from the numbers. When
+  `intended` names a longer interval than the one measured (speech end to TTS dispatch, claimed as microphone to
+  audible response) the measurement is partial and says so.
+- **Percentiles need samples.** p50 needs {@link MIN_P50_SAMPLES} observations and p95 {@link MIN_P95_SAMPLES}; below
+  that they are withheld with the reason, never shown as if supported.
+- **A timeout or failed observation is a sample.** Its value is how long it waited, so a lower bound: it counts
+  in the sample, in the percentiles and as a budget breach, and the summary says the figures are lower bounds.
+- **Comparisons are of the same metric.** Different boundaries are not compared; a change of instrumentation,
+  environment, budget or other condition between the two sides is disclosed beside the change.
+
+It decides nothing: a latency, a budget breach and a comparison never change a status, a slot or a winner.
+
+See docs/rich-evidence.md#product-latency
+
+#### `PRODUCT_LATENCY`
+
+```ts
+export const PRODUCT_LATENCY = …
+```
+
+The `kind` of a measurement that is a user-visible product latency.
+
+#### `MIN_P50_SAMPLES`
+
+```ts
+export const MIN_P50_SAMPLES = …
+```
+
+Observations needed before a p50 is shown.
+
+#### `MIN_P95_SAMPLES`
+
+```ts
+export const MIN_P95_SAMPLES = …
+```
+
+Observations needed before a p95 is shown.
+
+#### `isProductLatency`
+
+```ts
+export function isProductLatency(measurement: Pick<EvidenceMeasurement, 'kind'>): boolean
+```
+
+Whether a measurement is a product latency: only the producer's `kind` makes it one. A name, a unit or a value never does.
+
+#### `productLatency`
+
+```ts
+export function productLatency(measurements: readonly EvidenceMeasurement[] | undefined): readonly EvidenceMeasurement[] | undefined
+```
+
+The product-latency measurements of a check, or undefined when it has none (so a result with no latency carries
+nothing). Measurements that are not well formed (no finite value, no name) are left out.
+
+#### `BoundaryState`
+
+```ts
+export type BoundaryState = 'declared' | 'partial' | 'undeclared';
+```
+
+How well the ends of a measurement are known: `declared` (both named), `partial` (it stops short of, or starts after, what it claims) or `undeclared`.
+
+#### `Boundary`
+
+```ts
+export type Boundary = {
+  readonly state: BoundaryState;
+  readonly from?: string;
+  readonly to?: string;
+  /** The interval the measurement's name claims, when it is longer than what was measured. */
+  readonly intended?: string;
+  /** `submit pressed to confirmation visible`, `to confirmation visible (start not declared)`. */
+  readonly label: string;
+};
+```
+
+The ends of a measurement, in words, and whether they support a claim about the user's wait.
+
+#### `boundaryOf`
+
+```ts
+export function boundaryOf(measurement: Pick<EvidenceMeasurement, 'from' | 'to' | 'intended'>): Boundary
+```
+
+What a measurement spans. `undeclared` when either end is missing: the numbers are shown and no product claim is made.
+`partial` when `intended` names a different end than the measured one: the interval is the measured one, never the intended.
+
+#### `ObservationOutcome`
+
+```ts
+export type ObservationOutcome = 'observed' | 'timeout' | 'failed' | 'unclear';
+```
+
+What happened to one observation: `observed`, or it `timeout`ed or `failed`, or the producer said something Atlas cannot read (`unclear`).
+
+#### `LatencyEntry`
+
+```ts
+export type LatencyEntry = {
+  /** When it was recorded, epoch milliseconds. */
+  readonly at: number;
+  readonly environment: string | null;
+  readonly parameters: Readonly<Record<string, string>>;
+  readonly sha: string | null;
+  readonly layer: string;
+  readonly run?: string;
+  readonly latency?: readonly EvidenceMeasurement[];
+  readonly evidence?: { readonly labels: Readonly<Record<string, string>> };
+};
+```
+
+One recorded result, as much of it as this module reads. A `TestResultEntry` is one.
+
+#### `LatencyObservation`
+
+```ts
+export type LatencyObservation = {
+  readonly name: string;
+  readonly unit: string;
+  readonly value: number;
+  readonly limit?: number;
+  readonly outcome: ObservationOutcome;
+  readonly boundary: Boundary;
+  readonly source?: string;
+  readonly instrumentation?: string;
+  readonly cohort: Readonly<Record<string, string>>;
+  readonly at: number;
+  readonly layer: string;
+  readonly environment: string | null;
+  readonly parameters: Readonly<Record<string, string>>;
+  readonly labels: Readonly<Record<string, string>>;
+  readonly sha: string | null;
+};
+```
+
+One observation of a latency. Time units are converted to milliseconds.
+
+#### `latencyObservations`
+
+```ts
+export function latencyObservations(entries: readonly LatencyEntry[]): LatencyObservation[]
+```
+
+The observations in recorded results, in the order given. Entries with no latency contribute none.
+
+#### `LatencyGrouping`
+
+```ts
+export type LatencyGrouping = 'none' | 'device' | 'build' | 'window';
+```
+
+How observations are grouped: not at all, by `device`, by `build`, or by the UTC `window` (day) they were recorded in.
+
+#### `LATENCY_GROUPINGS`
+
+```ts
+export const LATENCY_GROUPINGS: readonly LatencyGrouping[]
+```
+
+The groupings, in the order a control offers them.
+
+#### `groupOf`
+
+```ts
+export function groupOf(observation: LatencyObservation, by: LatencyGrouping): string
+```
+
+The group an observation falls in. `device` is the measurement's cohort, else the result's label, parameter or
+environment; `build` is the cohort or label, else the commit. One that names nothing is `unknown device` or
+`unknown build`: it is never merged into a named group.
+
+#### `LatencyMetric`
+
+```ts
+export type LatencyMetric = {
+  /** Equal for observations of the same name, unit and boundaries. */
+  readonly key: string;
+  readonly name: string;
+  readonly unit: string;
+  readonly boundary: Boundary;
+};
+```
+
+A measured interval: the name, the unit and the ends. Observations of one metric are comparable; of two metrics they are not.
+
+#### `LatencySummary`
+
+```ts
+export type LatencySummary = {
+  readonly metric: LatencyMetric;
+  /** The group (a device, a build, a day, or all). */
+  readonly group: string;
+  /** Every observation, including those that timed out or failed. */
+  readonly samples: number;
+  /** Observations that completed. */
+  readonly observed: number;
+  readonly timeouts: number;
+  readonly failures: number;
+  /** Observations over their budget, plus every timeout or failure when a budget is set. */
+  readonly breaches: number;
+  /** The budget, from the newest observation that has one. */
+  readonly budget?: number;
+  /** The median, or undefined with the reason in `withheld`. */
+  readonly p50?: number;
+  readonly p95?: number;
+  /** Why a figure is not shown, in words. */
+  readonly withheld: readonly string[];
+  /** True when a timeout or failure's waiting time is in the figures: they are lower bounds. */
+  readonly lowerBound: boolean;
+  readonly max: number;
+  /** The newest observation's value. */
+  readonly last: number;
+  readonly lastAt: number;
+  /** Whether the numbers support a statement about the user's wait: false when the boundaries are not declared. */
+  readonly claim: boolean;
+  /** What the reader must not assume, in words: partial interval, missing boundaries, a single observation. */
+  readonly gaps: readonly string[];
+  readonly instrumentation: readonly string[];
+  readonly environments: readonly string[];
+  /** Other facts the observations differ by: cohort entries and parameters, as `key=value`. */
+  readonly conditions: readonly string[];
+  readonly budgets: readonly number[];
+};
+```
+
+What the observations of one metric in one group show, and what they cannot.
+
+#### `LatencyReview`
+
+```ts
+export type LatencyReview = {
+  readonly by: LatencyGrouping;
+  readonly metrics: readonly { readonly metric: LatencyMetric; readonly groups: readonly LatencySummary[] }[];
+  /** Results that recorded no product latency. They are unknown, not fast. */
+  readonly unmeasured: number;
+  /** Results that did. */
+  readonly measured: number;
+};
+```
+
+The metrics of a result set, each with its summary in every group, groups newest first.
+
+#### `latencyReview`
+
+```ts
+export function latencyReview(entries: readonly LatencyEntry[], by: LatencyGrouping = 'none'): LatencyReview
+```
+
+Reads recorded results (any order) into a review: for every metric, a summary in every group. A result with no
+product latency counts only in `unmeasured`. Metrics are listed by name; groups newest first.
+
+- `by`: How observations are grouped. Default `none`.
+
+**Example**
+
+```ts
+const review = latencyReview(history.history, 'build');
+review.metrics[0]?.groups[0]?.p95; // undefined until 20 observations of that build
+```
+
+#### `LatencyDelta`
+
+```ts
+export type LatencyDelta = {
+  readonly baseline: number;
+  readonly candidate: number;
+  /** Candidate minus baseline, in the metric's unit. Positive is slower. */
+  readonly absolute: number;
+  /** The absolute change over the baseline, or null when the baseline is zero. */
+  readonly relative: number | null;
+};
+```
+
+A change of one figure from the baseline to the candidate.
+
+#### `LatencyChange`
+
+```ts
+export type LatencyChange = {
+  /** Whether a change is stated at all. */
+  readonly comparable: boolean;
+  /** `p50` or `p95` when both sides support it; `last` compares the newest observation of each, which is not a percentile. */
+  readonly basis?: 'p50' | 'p95' | 'last';
+  readonly p50?: LatencyDelta;
+  readonly p95?: LatencyDelta;
+  readonly last?: LatencyDelta;
+  /** Why no change is stated. */
+  readonly reasons: readonly string[];
+  /** Differences in how the two sides were measured or run. A change should be read with them. */
+  readonly disclosures: readonly string[];
+};
+```
+
+What changed between two summaries, or why that cannot be said.
+
+#### `compareLatency`
+
+```ts
+export function compareLatency(baseline: LatencySummary, candidate: LatencySummary): LatencyChange
+```
+
+The change from a baseline to a candidate of the same metric. Different metrics (names, units or boundaries) are not
+compared and the reason is given. When both sides support p50 (and p95) the change is of those; otherwise only the
+newest observation of each is compared, labelled as such. Changes of instrumentation, environments, other
+conditions, budget and partial or lower-bound figures are listed in `disclosures`, never hidden.
+
+#### `formatLatency`
+
+```ts
+export function formatLatency(value: number | undefined, unit: string): string
+```
+
+`500 ms`, `1.2 s`: a latency in the largest unit that keeps it readable. Non-time units print as given.
+
 ### Environment axis of a layer
 
 Source: `packages/reports/src/layer-axes.ts`. Import: `@crvouga/atlas/reports/layer-axes`
@@ -5992,6 +6339,273 @@ skipped or unresolved; otherwise `unknown` when a contributing layer has no rece
 report with no results at all is unknown too); otherwise `complete` when something was expected and `none-expected` when
 nothing was. Gaps are sorted so the output never depends on read order.
 
+### Reliability of a test
+
+Source: `packages/reports/src/reliability.ts`. Import: `@crvouga/atlas/reports/reliability`
+
+How often a test passed, counted so the number can be trusted or doubted. A pass rate over raw history is easy to
+overinterpret: one execution is published as several snapshots, a retry is not a second trial, desktop and phone are
+different questions, and an earlier version of a behavior says nothing about the current one. This module answers
+from the recorded results of one test and a {@link ReliabilityScope}, and states what it counted:
+
+- **Independent executions.** The snapshots one run recorded for one producer, environment, parameter set and kind
+  (running, finished, enriched) are one execution; its last word counts once, with the tries it recorded. Raw
+  snapshots stay inspectable (`snapshots`) and are never weighted as new trials.
+- **First try and eventual.** Where tries were recorded, `firstAttempt` counts executions whose first try passed and
+  `rate` counts those that passed in the end. An execution that recorded no tries is not assumed to have passed first
+  time: it is left out of `firstAttempt` and counted in `unrecordedAttempts`.
+- **Scope.** Environment, producer, parameters, branch, commit, spec revision and a time window select the executions;
+  the rest are shown as `outsideScope`, never mixed in. By default only the newest recorded revision of the spec
+  counts, and earlier ones are shown as `historical` context.
+- **What is not a trial.** Only `passed` and `failed` executions are in the denominator. `not-run` and unfinished
+  executions are excluded and counted by name; nothing missing becomes a pass.
+- **Limits, in words.** `notes` say when the sample is too small to establish anything, when environments, producers
+  or branches are pooled, when failed cases were published again, and when history is bounded. A rate is never a
+  production failure probability: the checks are a selection.
+
+It only reads results; it never changes a status, a slot or a winner.
+
+See docs/server.md#reading-the-report-as-an-agent
+
+#### `RELIABILITY_VERSION`
+
+```ts
+export const RELIABILITY_VERSION = …
+```
+
+Version of the reliability document.
+
+#### `MIN_EXECUTIONS`
+
+```ts
+export const MIN_EXECUTIONS = …
+```
+
+Completed executions below which a rate is shown but not treated as established.
+
+#### `ReliabilityScope`
+
+```ts
+export type ReliabilityScope = {
+  /** Environments or devices (`phone`, `desktop-web`). Executions in others never count towards these. */
+  readonly environment?: readonly string[];
+  readonly producer?: readonly string[];
+  /** `key=value` parameter choices (`device=small`). */
+  readonly parameter?: readonly string[];
+  readonly branch?: readonly string[];
+  /** Commits: a full id, or a prefix of at least seven characters. */
+  readonly sha?: readonly string[];
+  /** Only executions recorded against these spec fingerprints. Overrides `revision`. */
+  readonly specRevision?: readonly string[];
+  /** `current` (the default): only the newest spec revision recorded in the rest of the scope, the older ones shown as context. `any`: every revision, pooled. */
+  readonly revision?: 'current' | 'any';
+  /** Only executions at or after this time, epoch ms. */
+  readonly since?: number;
+  /** Only executions at or before this time, epoch ms. */
+  readonly until?: number;
+};
+```
+
+What a reliability figure is about. Every field is optional; nothing asked is everything the history holds (of the current spec revision).
+
+#### `ReliabilityExclusion`
+
+```ts
+export type ReliabilityExclusion = 'not-run' | 'unfinished' | 'unknown-status' | 'historical-revision' | 'revision-not-recorded';
+```
+
+Why an execution is not in the denominator.
+
+#### `ReliabilityExecution`
+
+```ts
+export type ReliabilityExecution = {
+  /** The key of the run it belongs to (see `runKey`). */
+  readonly run: string;
+  /** When its last word was recorded, epoch ms. */
+  readonly at: number;
+  /** Its last word. */
+  readonly status: string;
+  /** True when it is in the denominator. */
+  readonly counted: boolean;
+  /** Why it is not, when it is not. */
+  readonly excluded?: ReliabilityExclusion;
+  readonly environment: string | null;
+  readonly producer: string;
+  readonly kind: string;
+  readonly parameters: Readonly<Record<string, string>>;
+  readonly branch: string | null;
+  readonly sha: string | null;
+  readonly runUrl: string | null;
+  /** The layer holding its last word (open it with `/api/reports/evidence/detail/{backend}/{layer}?check=`). */
+  readonly layer: string;
+  readonly backend: string;
+  readonly durationMs: number | null;
+  readonly specRevision?: string;
+  /** How many raw snapshots (layers) recorded it. They are one execution, not that many trials. */
+  readonly snapshots: number;
+  /** The status of each recorded try, in order. Absent when no try was recorded. */
+  readonly attempts?: readonly string[];
+  /** The first try's status, when tries were recorded. */
+  readonly firstAttempt?: string;
+  /** True when a failed snapshot of this execution was followed by a later one: it was published again after a failure. */
+  readonly republished: boolean;
+};
+```
+
+One independent execution: the last word of one run on one producer, environment, parameter set and kind.
+
+#### `ReliabilityRate`
+
+```ts
+export type ReliabilityRate = {
+  /** Independent executions that passed or failed. */
+  readonly executions: number;
+  readonly passed: number;
+  readonly failed: number;
+  /** passed / executions, or null with none. This is the eventual success rate. */
+  readonly rate: number | null;
+};
+```
+
+Counts and rates over a set of executions.
+
+#### `ReliabilityInterval`
+
+```ts
+export type ReliabilityInterval = {
+  readonly method: 'wilson';
+  readonly level: number;
+  readonly lower: number;
+  readonly upper: number;
+  /** What must hold for the interval to mean anything. */
+  readonly assumptions: string;
+};
+```
+
+A confidence interval and what it assumes.
+
+#### `ReliabilityNote`
+
+```ts
+export type ReliabilityNote = {
+  readonly code: string;
+  /** `warning`: the figure should not be read as representative without this in mind. */
+  readonly severity: 'warning' | 'info';
+  readonly message: string;
+};
+```
+
+A reason to doubt or qualify the figure. An open set: treat a code you do not know as informational.
+
+#### `ReliabilityCohort`
+
+```ts
+export type ReliabilityCohort = ReliabilityRate & {
+  readonly environment: string | null;
+  readonly producer: string;
+  readonly parameters: Readonly<Record<string, string>>;
+};
+```
+
+One environment, producer and parameter set among the counted executions, with its own rate.
+
+#### `ReliabilityDocument`
+
+```ts
+export type ReliabilityDocument = {
+  readonly schemaVersion: typeof RELIABILITY_VERSION;
+  readonly key: string;
+  readonly scope: ReliabilityScope;
+  /** Completed executions below which the sample is `insufficient`. */
+  readonly minimumExecutions: number;
+  /** Count only: `none` without an independent execution, `insufficient` below the minimum, `sufficient` otherwise. Says nothing about representativeness; read `notes`. */
+  readonly sample: 'none' | 'insufficient' | 'sufficient';
+  /** Eventual success over the counted executions. */
+  readonly eventual: ReliabilityRate;
+  readonly interval: ReliabilityInterval | null;
+  /** Executions whose first try is known (tries were recorded), and how many of those passed first time. */
+  readonly firstAttempt: ReliabilityRate;
+  /** Counted executions that recorded no tries: unknown first try, left out of `firstAttempt`. */
+  readonly unrecordedAttempts: number;
+  readonly attempts: {
+    /** Tries recorded across the counted executions. */
+    readonly total: number;
+    /** Counted executions that needed more than one try. */
+    readonly retried: number;
+    /** Counted executions that failed first and passed in the end. */
+    readonly recovered: number;
+  };
+  /** The revision `revision: 'current'` counted, or null when none is recorded. */
+  readonly currentRevision: string | null;
+  /** Earlier spec revisions, shown as context and counted nowhere above. */
+  readonly historical: { readonly executions: number; readonly passed: number; readonly failed: number; readonly revisions: readonly { readonly specRevision: string; readonly executions: number }[] };
+  /** What was left out, by name. */
+  readonly excluded: Readonly<Record<ReliabilityExclusion, number>>;
+  /** Executions the scope did not select. */
+  readonly outsideScope: number;
+  /** Raw recorded snapshots behind the counted executions. */
+  readonly snapshots: number;
+  /** Counted executions per environment, producer and parameter set. */
+  readonly cohorts: readonly ReliabilityCohort[];
+  /** What the scope could choose among: the values the history holds, with their independent executions. */
+  readonly available: {
+    readonly environments: readonly { readonly value: string; readonly count: number }[];
+    readonly producers: readonly { readonly value: string; readonly count: number }[];
+    readonly branches: readonly { readonly value: string; readonly count: number }[];
+  …
+```
+
+`GET /api/reports/tests/reliability`: how reliable a test is within a scope, and how that figure was counted.
+
+#### `reliability`
+
+```ts
+export function reliability(key: string, history: readonly TestResultEntry[], scope: ReliabilityScope = {}): ReliabilityDocument
+```
+
+How reliable one test is inside a scope, from the recorded results of that test (`TestHistoryDocument.history`, any
+order). See the module comment for what is counted and what is left out.
+
+- `key`: The test's key, echoed in the document.
+
+- `history`: Every recorded result of the test the caller holds. When it holds as many as `HISTORY_PER_TEST` the
+document says the history is bounded.
+
+- `scope`: What to count; see {@link ReliabilityScope}.
+
+**Example**
+
+```ts
+const document = reliability(test.key, history.history, { environment: ['phone'], since: Date.now() - 30 * 86_400_000 });
+if (document.sample !== 'sufficient') console.log(`only ${document.eventual.executions} executions`);
+```
+
+#### `scopeText`
+
+```ts
+export function scopeText(scope: ReliabilityScope): string
+```
+
+The scope as a short phrase, or `all recorded executions`.
+
+#### `reliabilityMarkdown`
+
+```ts
+export function reliabilityMarkdown(document: ReliabilityDocument): string
+```
+
+The reliability document as Markdown, for an agent or a terminal.
+
+#### `parseReliabilityScope`
+
+```ts
+export function parseReliabilityScope(params: URLSearchParams): ReliabilityScope
+```
+
+The scope an address asks for: `environment`, `producer`, `parameter`, `branch`, `sha` and `specRevision` (comma-separated
+lists), `revision` (`any`, else current) and `since` and `until` (epoch milliseconds, or a date).
+
 ### Remote commit report
 
 Source: `packages/reports/src/remote-report.ts`. Import: `@crvouga/atlas/reports/remote-report`
@@ -8316,8 +8930,12 @@ export type TestResultEntry = {
   readonly backend: string;
   /** Why it has its status: the first 500 characters of the check's detail. */
   readonly detail: string;
+  /** The fingerprint of its spec text the layer recorded, when it recorded one: results with different revisions were recorded against different promises. Absent means not recorded, never "the same". */
+  readonly specRevision?: string;
   /** What it carried beyond its outcome (attachments, steps, retries, tags, labels), when anything. */
   readonly evidence?: EvidenceSummary;
+  /** Its product-latency measurements (`kind: product-latency`), when it recorded any. Separate from `durationMs`, which is how long the test took (setup, waits and pacing included). */
+  readonly latency?: readonly EvidenceMeasurement[];
 };
 ```
 
@@ -8832,7 +9450,7 @@ const catalog = testCatalog(catalogModel(layers, { history }));
 - `get specs(): RecordedSpecs { return this.#specs; }` — The specs the added layers recorded.
 - `tests(): readonly { readonly key: string; readonly id: string; readonly specPath: string | null }[]` — Every test the history holds a result for (keyed as `TestSummary.key`), including tests the report no longer has a current result for.
 - `get layers(): number { return this.#seen.size; }` — How many layers were added.
-- `add(layer: LayerHeader, rows: readonly LayerRow[], specs: readonly SpecEntry[] = [], backend = layer.backend): void` — Takes in one layer: its header, its rows, the specs it recorded and the backend it was read from. A layer id already added is ignored.
+- `add(layer: LayerHeader, rows: readonly LayerRow[], specs: readonly SpecEntry[] = [], backend = layer.backend, fingerprints: Readonly<Record<string, string>> = {}): void` — Takes in one layer: its header, its rows, the specs it recorded and the backend it was read from. A layer id already added is ignored. `fingerprints` (spec path to the fingerprint the layer recorded, as `LayerRows.fingerprints`) gives each result its `specRevision`; without it no result has one.
 - `of(key: string): readonly TestResultEntry[]` — The recorded results of one test (by `TestSummary.key`), newest first. Empty when none.
 - `get runCount(): number { return this.#runs.size; }` — How many runs it knows of.
 - `runs(): RunSummary[]` — Every run, newest first.