@crvouga/atlas 1.8.0 → 1.9.0

package/docs/reference/reports.md

@@ -2145,6 +2145,8 @@ export type EvidenceSummary = {
   readonly annotations: number;
   /** How many tries were recorded when the check was retried. */
   readonly attempts: number;
+  /** The status of each recorded try, in the order they were made (its length is `attempts`). Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed. */
+  readonly attemptStatuses?: readonly EvidenceStatus[];
   readonly links: number;
   readonly tags: readonly string[];
   /** The normalized tags of the attachments themselves (`#demo` is `demo`), sorted and distinct. Absent when no attachment is tagged. Unlike `tags` they are never inherited from a check or run. */
@@ -6337,6 +6339,273 @@ skipped or unresolved; otherwise `unknown` when a contributing layer has no rece
 report with no results at all is unknown too); otherwise `complete` when something was expected and `none-expected` when
 nothing was. Gaps are sorted so the output never depends on read order.
 
+### Reliability of a test
+
+Source: `packages/reports/src/reliability.ts`. Import: `@crvouga/atlas/reports/reliability`
+
+How often a test passed, counted so the number can be trusted or doubted. A pass rate over raw history is easy to
+overinterpret: one execution is published as several snapshots, a retry is not a second trial, desktop and phone are
+different questions, and an earlier version of a behavior says nothing about the current one. This module answers
+from the recorded results of one test and a {@link ReliabilityScope}, and states what it counted:
+
+- **Independent executions.** The snapshots one run recorded for one producer, environment, parameter set and kind
+  (running, finished, enriched) are one execution; its last word counts once, with the tries it recorded. Raw
+  snapshots stay inspectable (`snapshots`) and are never weighted as new trials.
+- **First try and eventual.** Where tries were recorded, `firstAttempt` counts executions whose first try passed and
+  `rate` counts those that passed in the end. An execution that recorded no tries is not assumed to have passed first
+  time: it is left out of `firstAttempt` and counted in `unrecordedAttempts`.
+- **Scope.** Environment, producer, parameters, branch, commit, spec revision and a time window select the executions;
+  the rest are shown as `outsideScope`, never mixed in. By default only the newest recorded revision of the spec
+  counts, and earlier ones are shown as `historical` context.
+- **What is not a trial.** Only `passed` and `failed` executions are in the denominator. `not-run` and unfinished
+  executions are excluded and counted by name; nothing missing becomes a pass.
+- **Limits, in words.** `notes` say when the sample is too small to establish anything, when environments, producers
+  or branches are pooled, when failed cases were published again, and when history is bounded. A rate is never a
+  production failure probability: the checks are a selection.
+
+It only reads results; it never changes a status, a slot or a winner.
+
+See docs/server.md#reading-the-report-as-an-agent
+
+#### `RELIABILITY_VERSION`
+
+```ts
+export const RELIABILITY_VERSION = …
+```
+
+Version of the reliability document.
+
+#### `MIN_EXECUTIONS`
+
+```ts
+export const MIN_EXECUTIONS = …
+```
+
+Completed executions below which a rate is shown but not treated as established.
+
+#### `ReliabilityScope`
+
+```ts
+export type ReliabilityScope = {
+  /** Environments or devices (`phone`, `desktop-web`). Executions in others never count towards these. */
+  readonly environment?: readonly string[];
+  readonly producer?: readonly string[];
+  /** `key=value` parameter choices (`device=small`). */
+  readonly parameter?: readonly string[];
+  readonly branch?: readonly string[];
+  /** Commits: a full id, or a prefix of at least seven characters. */
+  readonly sha?: readonly string[];
+  /** Only executions recorded against these spec fingerprints. Overrides `revision`. */
+  readonly specRevision?: readonly string[];
+  /** `current` (the default): only the newest spec revision recorded in the rest of the scope, the older ones shown as context. `any`: every revision, pooled. */
+  readonly revision?: 'current' | 'any';
+  /** Only executions at or after this time, epoch ms. */
+  readonly since?: number;
+  /** Only executions at or before this time, epoch ms. */
+  readonly until?: number;
+};
+```
+
+What a reliability figure is about. Every field is optional; nothing asked is everything the history holds (of the current spec revision).
+
+#### `ReliabilityExclusion`
+
+```ts
+export type ReliabilityExclusion = 'not-run' | 'unfinished' | 'unknown-status' | 'historical-revision' | 'revision-not-recorded';
+```
+
+Why an execution is not in the denominator.
+
+#### `ReliabilityExecution`
+
+```ts
+export type ReliabilityExecution = {
+  /** The key of the run it belongs to (see `runKey`). */
+  readonly run: string;
+  /** When its last word was recorded, epoch ms. */
+  readonly at: number;
+  /** Its last word. */
+  readonly status: string;
+  /** True when it is in the denominator. */
+  readonly counted: boolean;
+  /** Why it is not, when it is not. */
+  readonly excluded?: ReliabilityExclusion;
+  readonly environment: string | null;
+  readonly producer: string;
+  readonly kind: string;
+  readonly parameters: Readonly<Record<string, string>>;
+  readonly branch: string | null;
+  readonly sha: string | null;
+  readonly runUrl: string | null;
+  /** The layer holding its last word (open it with `/api/reports/evidence/detail/{backend}/{layer}?check=`). */
+  readonly layer: string;
+  readonly backend: string;
+  readonly durationMs: number | null;
+  readonly specRevision?: string;
+  /** How many raw snapshots (layers) recorded it. They are one execution, not that many trials. */
+  readonly snapshots: number;
+  /** The status of each recorded try, in order. Absent when no try was recorded. */
+  readonly attempts?: readonly string[];
+  /** The first try's status, when tries were recorded. */
+  readonly firstAttempt?: string;
+  /** True when a failed snapshot of this execution was followed by a later one: it was published again after a failure. */
+  readonly republished: boolean;
+};
+```
+
+One independent execution: the last word of one run on one producer, environment, parameter set and kind.
+
+#### `ReliabilityRate`
+
+```ts
+export type ReliabilityRate = {
+  /** Independent executions that passed or failed. */
+  readonly executions: number;
+  readonly passed: number;
+  readonly failed: number;
+  /** passed / executions, or null with none. This is the eventual success rate. */
+  readonly rate: number | null;
+};
+```
+
+Counts and rates over a set of executions.
+
+#### `ReliabilityInterval`
+
+```ts
+export type ReliabilityInterval = {
+  readonly method: 'wilson';
+  readonly level: number;
+  readonly lower: number;
+  readonly upper: number;
+  /** What must hold for the interval to mean anything. */
+  readonly assumptions: string;
+};
+```
+
+A confidence interval and what it assumes.
+
+#### `ReliabilityNote`
+
+```ts
+export type ReliabilityNote = {
+  readonly code: string;
+  /** `warning`: the figure should not be read as representative without this in mind. */
+  readonly severity: 'warning' | 'info';
+  readonly message: string;
+};
+```
+
+A reason to doubt or qualify the figure. An open set: treat a code you do not know as informational.
+
+#### `ReliabilityCohort`
+
+```ts
+export type ReliabilityCohort = ReliabilityRate & {
+  readonly environment: string | null;
+  readonly producer: string;
+  readonly parameters: Readonly<Record<string, string>>;
+};
+```
+
+One environment, producer and parameter set among the counted executions, with its own rate.
+
+#### `ReliabilityDocument`
+
+```ts
+export type ReliabilityDocument = {
+  readonly schemaVersion: typeof RELIABILITY_VERSION;
+  readonly key: string;
+  readonly scope: ReliabilityScope;
+  /** Completed executions below which the sample is `insufficient`. */
+  readonly minimumExecutions: number;
+  /** Count only: `none` without an independent execution, `insufficient` below the minimum, `sufficient` otherwise. Says nothing about representativeness; read `notes`. */
+  readonly sample: 'none' | 'insufficient' | 'sufficient';
+  /** Eventual success over the counted executions. */
+  readonly eventual: ReliabilityRate;
+  readonly interval: ReliabilityInterval | null;
+  /** Executions whose first try is known (tries were recorded), and how many of those passed first time. */
+  readonly firstAttempt: ReliabilityRate;
+  /** Counted executions that recorded no tries: unknown first try, left out of `firstAttempt`. */
+  readonly unrecordedAttempts: number;
+  readonly attempts: {
+    /** Tries recorded across the counted executions. */
+    readonly total: number;
+    /** Counted executions that needed more than one try. */
+    readonly retried: number;
+    /** Counted executions that failed first and passed in the end. */
+    readonly recovered: number;
+  };
+  /** The revision `revision: 'current'` counted, or null when none is recorded. */
+  readonly currentRevision: string | null;
+  /** Earlier spec revisions, shown as context and counted nowhere above. */
+  readonly historical: { readonly executions: number; readonly passed: number; readonly failed: number; readonly revisions: readonly { readonly specRevision: string; readonly executions: number }[] };
+  /** What was left out, by name. */
+  readonly excluded: Readonly<Record<ReliabilityExclusion, number>>;
+  /** Executions the scope did not select. */
+  readonly outsideScope: number;
+  /** Raw recorded snapshots behind the counted executions. */
+  readonly snapshots: number;
+  /** Counted executions per environment, producer and parameter set. */
+  readonly cohorts: readonly ReliabilityCohort[];
+  /** What the scope could choose among: the values the history holds, with their independent executions. */
+  readonly available: {
+    readonly environments: readonly { readonly value: string; readonly count: number }[];
+    readonly producers: readonly { readonly value: string; readonly count: number }[];
+    readonly branches: readonly { readonly value: string; readonly count: number }[];
+  …
+```
+
+`GET /api/reports/tests/reliability`: how reliable a test is within a scope, and how that figure was counted.
+
+#### `reliability`
+
+```ts
+export function reliability(key: string, history: readonly TestResultEntry[], scope: ReliabilityScope = {}): ReliabilityDocument
+```
+
+How reliable one test is inside a scope, from the recorded results of that test (`TestHistoryDocument.history`, any
+order). See the module comment for what is counted and what is left out.
+
+- `key`: The test's key, echoed in the document.
+
+- `history`: Every recorded result of the test the caller holds. When it holds as many as `HISTORY_PER_TEST` the
+document says the history is bounded.
+
+- `scope`: What to count; see {@link ReliabilityScope}.
+
+**Example**
+
+```ts
+const document = reliability(test.key, history.history, { environment: ['phone'], since: Date.now() - 30 * 86_400_000 });
+if (document.sample !== 'sufficient') console.log(`only ${document.eventual.executions} executions`);
+```
+
+#### `scopeText`
+
+```ts
+export function scopeText(scope: ReliabilityScope): string
+```
+
+The scope as a short phrase, or `all recorded executions`.
+
+#### `reliabilityMarkdown`
+
+```ts
+export function reliabilityMarkdown(document: ReliabilityDocument): string
+```
+
+The reliability document as Markdown, for an agent or a terminal.
+
+#### `parseReliabilityScope`
+
+```ts
+export function parseReliabilityScope(params: URLSearchParams): ReliabilityScope
+```
+
+The scope an address asks for: `environment`, `producer`, `parameter`, `branch`, `sha` and `specRevision` (comma-separated
+lists), `revision` (`any`, else current) and `since` and `until` (epoch milliseconds, or a date).
+
 ### Remote commit report
 
 Source: `packages/reports/src/remote-report.ts`. Import: `@crvouga/atlas/reports/remote-report`
@@ -8661,6 +8930,8 @@ export type TestResultEntry = {
   readonly backend: string;
   /** Why it has its status: the first 500 characters of the check's detail. */
   readonly detail: string;
+  /** The fingerprint of its spec text the layer recorded, when it recorded one: results with different revisions were recorded against different promises. Absent means not recorded, never "the same". */
+  readonly specRevision?: string;
   /** What it carried beyond its outcome (attachments, steps, retries, tags, labels), when anything. */
   readonly evidence?: EvidenceSummary;
   /** Its product-latency measurements (`kind: product-latency`), when it recorded any. Separate from `durationMs`, which is how long the test took (setup, waits and pacing included). */
@@ -9179,7 +9450,7 @@ const catalog = testCatalog(catalogModel(layers, { history }));
 - `get specs(): RecordedSpecs { return this.#specs; }` — The specs the added layers recorded.
 - `tests(): readonly { readonly key: string; readonly id: string; readonly specPath: string | null }[]` — Every test the history holds a result for (keyed as `TestSummary.key`), including tests the report no longer has a current result for.
 - `get layers(): number { return this.#seen.size; }` — How many layers were added.
-- `add(layer: LayerHeader, rows: readonly LayerRow[], specs: readonly SpecEntry[] = [], backend = layer.backend): void` — Takes in one layer: its header, its rows, the specs it recorded and the backend it was read from. A layer id already added is ignored.
+- `add(layer: LayerHeader, rows: readonly LayerRow[], specs: readonly SpecEntry[] = [], backend = layer.backend, fingerprints: Readonly<Record<string, string>> = {}): void` — Takes in one layer: its header, its rows, the specs it recorded and the backend it was read from. A layer id already added is ignored. `fingerprints` (spec path to the fingerprint the layer recorded, as `LayerRows.fingerprints`) gives each result its `specRevision`; without it no result has one.
 - `of(key: string): readonly TestResultEntry[]` — The recorded results of one test (by `TestSummary.key`), newest first. Empty when none.
 - `get runCount(): number { return this.#runs.size; }` — How many runs it knows of.
 - `runs(): RunSummary[]` — Every run, newest first.

package/docs/reference/schema.md

@@ -300,6 +300,17 @@ export const TestHistorySchema = …
 
 `GET /api/reports/tests/history`: one test and every result recorded for it (at most 100), newest first.
 
+#### `TestReliabilitySchema`
+
+```ts
+export const TestReliabilitySchema = …
+```
+
+`GET /api/reports/tests/reliability`: how reliable one test is inside a scope, and exactly how that figure was counted.
+The snapshots one run recorded for one producer, environment and parameter set are one independent execution; only
+`passed` and `failed` executions are trials; `notes` say when the sample is too small, pooled, selectively rerun or
+bounded. It reads results and never changes a status. It is not a production failure probability.
+
 #### `ReportRunSchema`
 
 ```ts

package/docs/server.md

@@ -374,6 +374,7 @@ with `format=markdown` (or `Accept: text/markdown`), and takes the project prefi
 | Every test, a summary (by status, environment, producer, domain), insights (failing, flaky, slowest) and facets | `GET /api/reports/tests` |
 | Only some: `q` (all words), `status` (or `flaky`), `environment`, `producer`, `domain`, `spec`, `tag`, `run`; `sort`, `offset`, `limit` | `GET /api/reports/tests?status=failed&environment=ios` |
 | One test: every recorded result (status, duration, environment, branch, commit, CI run), newest first | `GET /api/reports/tests/history?key=<key>` |
+| How reliable one test is in a scope (environment, producer, parameters, branch, commit, spec revision, time window): independent executions, first-try and eventual success, what was left out and why | `GET /api/reports/tests/reliability?key=<key>&environment=phone` |
 | Why Specs, domains and counts disagree: ambiguous spec paths, shared domains, gaps, a reconciliation of the totals | `GET /api/reports/tests/health` |
 | The runs that recorded the report (by CI run URL, else branch and commit); `?video=true` keeps those with a playable video; `artifactTag`, `type`, `label`, `environment`, `branch`, `outcome`, `since`, `until` narrow it ([filtering](rich-evidence.md#finding-runs-and-artifacts)) | `GET /api/reports/runs` |
 | Recorded results that carry evidence, across the whole history, found by tag, media type, label, feature, environment, branch, revision, outcome and time | `GET /api/reports/artifacts?artifactTag=demo&type=video` |
@@ -388,6 +389,21 @@ curl "$ATLAS_SERVER_URL/api/projects/$ATLAS_PROJECT/reports/tests?format=markdow
   `stats` describes its recorded history: pass rate, flips, whether it is flaky and why (it passed and failed on the
   same commit, passed only after a retry, or its producer said so), durations (avg, p50, p95, min, max, last), runs
   and branches.
+- The **reliability** document qualifies `stats.passRate` instead of repeating it. The snapshots one run recorded for one
+  producer, environment and parameter set (running, finished, enriched) are **one independent execution**, counted by its last
+  word with the tries it recorded; `firstAttempt` counts executions whose first try passed, `eventual` those that passed in the
+  end, and an execution that recorded no tries (`attemptStatuses` on its evidence) is never assumed to have passed first time.
+  Choose the scope with `environment`, `producer`, `parameter=key=value`, `branch`, `sha`, `specRevision`, `since` and `until`:
+  other environments never add to a phone's sample. By default only the newest recorded spec revision (the `fingerprint` of its
+  spec text, `specRevision` on each result) counts and earlier ones appear as `historical` context; `revision=any` pools them
+  and says so. Only `passed` and `failed` executions are trials; `not-run`, unfinished and unknown ones are named under
+  `excluded` and never become a pass. `sample` is `none`, `insufficient` (under `minimumExecutions`) or `sufficient`, a count
+  only: `notes` say when the figure pools environments, producers or branches, when failed cases were published again
+  (`selected-reruns`), and when history is bounded to the newest 100 results. `executions` is the denominator, one row per
+  independent execution with its tries and snapshots, so the same query can be inspected and shared. The Wilson interval states
+  its assumptions. It describes the checks Atlas holds; it is not a production failure probability. The viewer shows the same
+  figures under a test (scope in the address as `relEnv`, `relProducer`, `relBranch`, `relParam`, `relSince`, `relUntil`,
+  `relRevision=any`, `relOpen=true`).
 - The **health** document explains the numbers instead of adding one. It keeps spec files, defined specs, specs with an
   executed test, tests, outlines, examples and per-environment observations apart, and walks from the Specs total to
   the Tests total. Two spec paths are one logical spec only when a mapping says so
@@ -453,6 +469,8 @@ curl "$ATLAS_SERVER_URL/api/projects/$ATLAS_PROJECT/reports/tests?format=markdow
 | `GET /api/projects/{project}/reports/tests/health` | read | Why specs, domains and counts disagree. |
 | `GET /api/reports/tests/history` | read | One test and every result recorded for it. |
 | `GET /api/projects/{project}/reports/tests/history` | read | One test and every result recorded for it. |
+| `GET /api/reports/tests/reliability` | read | How reliable one test is in a scope, and how that was counted. |
+| `GET /api/projects/{project}/reports/tests/reliability` | read | How reliable one test is in a scope, and how that was counted. |
 | `GET /api/reports/artifact-tags` | read | Reviewer tags of attachments: every recorded edit. |
 | `GET /api/projects/{project}/reports/artifact-tags` | read | Reviewer tags of attachments: every recorded edit. |
 | `POST /api/reports/artifact-tags` | publish | Tag or untag one attachment after upload. |

package/docs/ui.md

@@ -87,6 +87,16 @@ when Watch is on screen.
 - The spec index searches titles, paths and declared domains, and can group by the domain from spec metadata (specs with
   none are listed under "No domain declared"; the Tests view reports domain problems).
 
+## Reliability
+
+An open test shows its reliability (`explorer/Reliability.tsx`, worked out by `reports/reliability` from the history the page
+already holds). Never show a bare percentage: it always comes with the number of independent executions it rests on, the
+"Insufficient evidence" badge below the minimum (one pass is not established reliability), the scope (environment,
+producer, branch, parameters, time window, spec revision) and the warning notes (pooled environments, failed cases
+published again, bounded history). First-try success is over executions that recorded their tries only; a result that
+did not run or did not finish is excluded by name and never counted as a pass. The scope and the open executions table
+are in the address (`rel*`), so a link reproduces the figure and the executions behind it.
+
 ## Product latency
 
 A test's duration and its product latency are different things and the viewer never mixes them (`explorer/ProductLatency.tsx`,

package/llms-full.txt

@@ -1763,6 +1763,7 @@ with `format=markdown` (or `Accept: text/markdown`), and takes the project prefi
 | Every test, a summary (by status, environment, producer, domain), insights (failing, flaky, slowest) and facets | `GET /api/reports/tests` |
 | Only some: `q` (all words), `status` (or `flaky`), `environment`, `producer`, `domain`, `spec`, `tag`, `run`; `sort`, `offset`, `limit` | `GET /api/reports/tests?status=failed&environment=ios` |
 | One test: every recorded result (status, duration, environment, branch, commit, CI run), newest first | `GET /api/reports/tests/history?key=<key>` |
+| How reliable one test is in a scope (environment, producer, parameters, branch, commit, spec revision, time window): independent executions, first-try and eventual success, what was left out and why | `GET /api/reports/tests/reliability?key=<key>&environment=phone` |
 | Why Specs, domains and counts disagree: ambiguous spec paths, shared domains, gaps, a reconciliation of the totals | `GET /api/reports/tests/health` |
 | The runs that recorded the report (by CI run URL, else branch and commit); `?video=true` keeps those with a playable video; `artifactTag`, `type`, `label`, `environment`, `branch`, `outcome`, `since`, `until` narrow it ([filtering](rich-evidence.md#finding-runs-and-artifacts)) | `GET /api/reports/runs` |
 | Recorded results that carry evidence, across the whole history, found by tag, media type, label, feature, environment, branch, revision, outcome and time | `GET /api/reports/artifacts?artifactTag=demo&type=video` |
@@ -1777,6 +1778,21 @@ curl "$ATLAS_SERVER_URL/api/projects/$ATLAS_PROJECT/reports/tests?format=markdow
   `stats` describes its recorded history: pass rate, flips, whether it is flaky and why (it passed and failed on the
   same commit, passed only after a retry, or its producer said so), durations (avg, p50, p95, min, max, last), runs
   and branches.
+- The **reliability** document qualifies `stats.passRate` instead of repeating it. The snapshots one run recorded for one
+  producer, environment and parameter set (running, finished, enriched) are **one independent execution**, counted by its last
+  word with the tries it recorded; `firstAttempt` counts executions whose first try passed, `eventual` those that passed in the
+  end, and an execution that recorded no tries (`attemptStatuses` on its evidence) is never assumed to have passed first time.
+  Choose the scope with `environment`, `producer`, `parameter=key=value`, `branch`, `sha`, `specRevision`, `since` and `until`:
+  other environments never add to a phone's sample. By default only the newest recorded spec revision (the `fingerprint` of its
+  spec text, `specRevision` on each result) counts and earlier ones appear as `historical` context; `revision=any` pools them
+  and says so. Only `passed` and `failed` executions are trials; `not-run`, unfinished and unknown ones are named under
+  `excluded` and never become a pass. `sample` is `none`, `insufficient` (under `minimumExecutions`) or `sufficient`, a count
+  only: `notes` say when the figure pools environments, producers or branches, when failed cases were published again
+  (`selected-reruns`), and when history is bounded to the newest 100 results. `executions` is the denominator, one row per
+  independent execution with its tries and snapshots, so the same query can be inspected and shared. The Wilson interval states
+  its assumptions. It describes the checks Atlas holds; it is not a production failure probability. The viewer shows the same
+  figures under a test (scope in the address as `relEnv`, `relProducer`, `relBranch`, `relParam`, `relSince`, `relUntil`,
+  `relRevision=any`, `relOpen=true`).
 - The **health** document explains the numbers instead of adding one. It keeps spec files, defined specs, specs with an
   executed test, tests, outlines, examples and per-environment observations apart, and walks from the Specs total to
   the Tests total. Two spec paths are one logical spec only when a mapping says so
@@ -1842,6 +1858,8 @@ curl "$ATLAS_SERVER_URL/api/projects/$ATLAS_PROJECT/reports/tests?format=markdow
 | `GET /api/projects/{project}/reports/tests/health` | read | Why specs, domains and counts disagree. |
 | `GET /api/reports/tests/history` | read | One test and every result recorded for it. |
 | `GET /api/projects/{project}/reports/tests/history` | read | One test and every result recorded for it. |
+| `GET /api/reports/tests/reliability` | read | How reliable one test is in a scope, and how that was counted. |
+| `GET /api/projects/{project}/reports/tests/reliability` | read | How reliable one test is in a scope, and how that was counted. |
 | `GET /api/reports/artifact-tags` | read | Reviewer tags of attachments: every recorded edit. |
 | `GET /api/projects/{project}/reports/artifact-tags` | read | Reviewer tags of attachments: every recorded edit. |
 | `POST /api/reports/artifact-tags` | publish | Tag or untag one attachment after upload. |
@@ -3979,6 +3997,16 @@ when Watch is on screen.
 - The spec index searches titles, paths and declared domains, and can group by the domain from spec metadata (specs with
   none are listed under "No domain declared"; the Tests view reports domain problems).
 
+## Reliability
+
+An open test shows its reliability (`explorer/Reliability.tsx`, worked out by `reports/reliability` from the history the page
+already holds). Never show a bare percentage: it always comes with the number of independent executions it rests on, the
+"Insufficient evidence" badge below the minimum (one pass is not established reliability), the scope (environment,
+producer, branch, parameters, time window, spec revision) and the warning notes (pooled environments, failed cases
+published again, bounded history). First-try success is over executions that recorded their tries only; a result that
+did not run or did not finish is excluded by name and never counted as a pass. The scope and the open executions table
+are in the address (`rel*`), so a link reproduces the figure and the executions behind it.
+
 ## Product latency
 
 A test's duration and its product latency are different things and the viewer never mixes them (`explorer/ProductLatency.tsx`,
@@ -5208,6 +5236,54 @@ Scoped to one project. The same project may be selected on the unprefixed API us
 
 Responses: `200` The test and its history.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
 
+### How reliable one test is in a scope, and how that was counted
+
+`GET /api/reports/tests/reliability` · role: read
+
+A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).
+
+Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.
+
+- `key` (query, required): The test's `key` from `/api/reports/tests`.
+- `environment` (query): Environments or devices to count. Comma-separated.
+- `producer` (query): Producers to count. Comma-separated.
+- `parameter` (query): Parameter choices `key=value`; all must match. Comma-separated.
+- `branch` (query): Branches to count. Comma-separated.
+- `sha` (query): Commits: a full id or a prefix of at least seven characters. Comma-separated.
+- `specRevision` (query): Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.
+- `revision` (query): `any` pools every spec revision; otherwise only the newest recorded one counts.
+- `since` (query): Only executions at or after this time: epoch milliseconds or a date.
+- `until` (query): Only executions at or before this time: epoch milliseconds or a date.
+- `format` (query): `markdown` for Markdown instead of JSON.
+- `project` (query): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
+
+Responses: `200` The reliability of the test in the scope.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
+
+### How reliable one test is in a scope, and how that was counted
+
+`GET /api/projects/{project}/reports/tests/reliability` · role: read
+
+A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).
+
+Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.
+
+Scoped to one project. The same project may be selected on the unprefixed API using ?project= or X-Atlas-Project. Unknown projects return 404 on reads and are created by an authenticated upload.
+
+- `project` (path, required): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
+- `key` (query, required): The test's `key` from `/api/reports/tests`.
+- `environment` (query): Environments or devices to count. Comma-separated.
+- `producer` (query): Producers to count. Comma-separated.
+- `parameter` (query): Parameter choices `key=value`; all must match. Comma-separated.
+- `branch` (query): Branches to count. Comma-separated.
+- `sha` (query): Commits: a full id or a prefix of at least seven characters. Comma-separated.
+- `specRevision` (query): Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.
+- `revision` (query): `any` pools every spec revision; otherwise only the newest recorded one counts.
+- `since` (query): Only executions at or after this time: epoch milliseconds or a date.
+- `until` (query): Only executions at or before this time: epoch milliseconds or a date.
+- `format` (query): `markdown` for Markdown instead of JSON.
+
+Responses: `200` The reliability of the test in the scope.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
+
 ### Reviewer tags of attachments: every recorded edit
 
 `GET /api/reports/artifact-tags` · role: read

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crvouga/atlas",
-  "version": "1.8.0",
+  "version": "1.9.0",
   "description": "Atlas: specs of any kind, reported as layered evidence. Publish checks from any runner (JUnit, CTRF, a script), combine them across nodes, branches, environments and backends, gate CI on the result, and run statecharts against the real app.",
   "license": "MIT",
   "type": "module",

package/packages/schema/schemas/effective-delta.schema.json

@@ -179,6 +179,19 @@
                       "minimum": 0,
                       "maximum": 9007199254740991
                     },
+                    "attemptStatuses": {
+                      "description": "The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed",
+                      "type": "array",
+                      "items": {
+                        "type": "string",
+                        "enum": [
+                          "passed",
+                          "failed",
+                          "running",
+                          "not-run"
+                        ]
+                      }
+                    },
                     "links": {
                       "type": "integer",
                       "minimum": 0,
@@ -572,6 +585,19 @@
                     "minimum": 0,
                     "maximum": 9007199254740991
                   },
+                  "attemptStatuses": {
+                    "description": "The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed",
+                    "type": "array",
+                    "items": {
+                      "type": "string",
+                      "enum": [
+                        "passed",
+                        "failed",
+                        "running",
+                        "not-run"
+                      ]
+                    }
+                  },
                   "links": {
                     "type": "integer",
                     "minimum": 0,

package/packages/schema/schemas/effective-evidence.schema.json

@@ -179,6 +179,19 @@
                       "minimum": 0,
                       "maximum": 9007199254740991
                     },
+                    "attemptStatuses": {
+                      "description": "The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed",
+                      "type": "array",
+                      "items": {
+                        "type": "string",
+                        "enum": [
+                          "passed",
+                          "failed",
+                          "running",
+                          "not-run"
+                        ]
+                      }
+                    },
                     "links": {
                       "type": "integer",
                       "minimum": 0,
@@ -572,6 +585,19 @@
                     "minimum": 0,
                     "maximum": 9007199254740991
                   },
+                  "attemptStatuses": {
+                    "description": "The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed",
+                    "type": "array",
+                    "items": {
+                      "type": "string",
+                      "enum": [
+                        "passed",
+                        "failed",
+                        "running",
+                        "not-run"
+                      ]
+                    }
+                  },
                   "links": {
                     "type": "integer",
                     "minimum": 0,

package/packages/schema/schemas/evidence-page.schema.json

@@ -166,6 +166,19 @@
                 "minimum": 0,
                 "maximum": 9007199254740991
               },
+              "attemptStatuses": {
+                "description": "The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed",
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "enum": [
+                    "passed",
+                    "failed",
+                    "running",
+                    "not-run"
+                  ]
+                }
+              },
               "links": {
                 "type": "integer",
                 "minimum": 0,

package/packages/schema/schemas/layer-rows.schema.json

@@ -170,6 +170,19 @@
                 "minimum": 0,
                 "maximum": 9007199254740991
               },
+              "attemptStatuses": {
+                "description": "The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed",
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "enum": [
+                    "passed",
+                    "failed",
+                    "running",
+                    "not-run"
+                  ]
+                }
+              },
               "links": {
                 "type": "integer",
                 "minimum": 0,