@crvouga/atlas 1.7.0 → 1.9.0

package/dist/schema/src/api.js

@@ -14,7 +14,7 @@
  * @see docs/server.md
  */
 import { z } from 'zod/v4';
-import { CheckSchema, EvidenceScopeSchema, LayerRunSchema } from './layer.js';
+import { CheckSchema, EvidenceScopeSchema, LayerRunSchema, MeasurementSchema } from './layer.js';
 /** One of the statuses a check has in a report: `passed`, `failed`, `running` (not finished) or `not-run`. Closed. */
 export const EvidenceStatusSchema = z.enum(['passed', 'failed', 'running', 'not-run']);
 /**
@@ -85,6 +85,7 @@ export const EvidenceSummarySchema = z.looseObject({
     steps: z.number().int().nonnegative(),
     annotations: z.number().int().nonnegative(),
     attempts: z.number().int().nonnegative(),
+    attemptStatuses: z.array(EvidenceStatusSchema).optional().describe('The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed'),
     links: z.number().int().nonnegative(),
     tags: z.array(z.string()),
     artifactTags: z.array(z.string()).optional().describe('Normalized tags of the attachments themselves, sorted and distinct. Absent when no attachment is tagged; never inherited from a check or run'),
@@ -124,7 +125,7 @@ export const LayerRowSchema = z.looseObject({
     status: EvidenceStatusSchema,
     specPath: z.string().nullable().describe('The spec the check belongs to, `specs/<path>`, or null'),
     detail: z.string().describe('Why it has this status; empty when there is nothing to add'),
-    measurements: z.array(z.looseObject({ name: z.string(), value: z.number(), unit: z.string(), limit: z.number().optional() })),
+    measurements: z.array(MeasurementSchema),
     testId: z.string().nullable(),
     variant: z.record(z.string(), z.string()),
     durationMs: z.number().nullable(),
@@ -270,7 +271,9 @@ export const TestResultEntrySchema = z.looseObject({
     layer: z.string().describe('The id of the layer that recorded it'),
     backend: z.string(),
     detail: z.string().describe('Why it has its status, at most 500 characters'),
-    evidence: EvidenceSummarySchema.optional()
+    specRevision: z.string().optional().describe('The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match'),
+    evidence: EvidenceSummarySchema.optional(),
+    latency: z.array(MeasurementSchema).optional().describe('Its product-latency measurements (`kind: product-latency`), when it recorded any. Separate from `durationMs`, which is how long the test took')
 });
 const DurationStatsSchema = z.looseObject({
     count: z.number().int().nonnegative(), totalMs: z.number(), avgMs: z.number(), p50Ms: z.number(), p95Ms: z.number(), minMs: z.number(), maxMs: z.number(), lastMs: z.number()
@@ -389,6 +392,55 @@ export const TestHistorySchema = z.looseObject({
     test: TestSummarySchema,
     history: z.array(TestResultEntrySchema)
 });
+const ReliabilityRateSchema = z.looseObject({
+    executions: z.number().int().nonnegative().describe('Independent executions that passed or failed'),
+    passed: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative(),
+    rate: z.number().nullable().describe('passed / executions, or null with none')
+});
+/**
+ * `GET /api/reports/tests/reliability`: how reliable one test is inside a scope, and exactly how that figure was counted.
+ * The snapshots one run recorded for one producer, environment and parameter set are one independent execution; only
+ * `passed` and `failed` executions are trials; `notes` say when the sample is too small, pooled, selectively rerun or
+ * bounded. It reads results and never changes a status. It is not a production failure probability.
+ */
+export const TestReliabilitySchema = z.looseObject({
+    schemaVersion: z.literal(1),
+    key: z.string().describe('The test\'s key'),
+    scope: z.looseObject({
+        environment: z.array(z.string()).optional(), producer: z.array(z.string()).optional(), parameter: z.array(z.string()).optional(),
+        branch: z.array(z.string()).optional(), sha: z.array(z.string()).optional(), specRevision: z.array(z.string()).optional(),
+        revision: z.enum(['current', 'any']).optional(), since: z.number().optional(), until: z.number().optional()
+    }).describe('What was asked: the selection the figures are about'),
+    minimumExecutions: z.number().int().positive().describe('Completed executions below which the sample is `insufficient`'),
+    sample: z.enum(['none', 'insufficient', 'sufficient']).describe('A count only: it says nothing about whether the executions are representative; read `notes`'),
+    eventual: ReliabilityRateSchema.describe('Executions that passed in the end, over those that passed or failed'),
+    interval: z.looseObject({ method: z.literal('wilson'), level: z.number(), lower: z.number(), upper: z.number(), assumptions: z.string() }).nullable().describe('An interval estimate with the assumptions it needs; null without an execution'),
+    firstAttempt: ReliabilityRateSchema.describe('Over the executions that recorded their tries: those whose first try passed'),
+    unrecordedAttempts: z.number().int().nonnegative().describe('Counted executions that recorded no tries: their first try is unknown, never assumed to have passed'),
+    attempts: z.looseObject({ total: z.number().int().nonnegative(), retried: z.number().int().nonnegative(), recovered: z.number().int().nonnegative() }),
+    currentRevision: z.string().nullable().describe('The spec revision `revision=current` counted, or null'),
+    historical: z.looseObject({
+        executions: z.number().int().nonnegative(), passed: z.number().int().nonnegative(), failed: z.number().int().nonnegative(),
+        revisions: z.array(z.looseObject({ specRevision: z.string(), executions: z.number().int().nonnegative() }))
+    }).describe('Earlier spec revisions: context only, counted nowhere above'),
+    excluded: z.record(z.string(), z.number().int().nonnegative()).describe('Executions in the scope that are not trials, by reason: `not-run`, `unfinished`, `unknown-status`, `historical-revision`, `revision-not-recorded`'),
+    outsideScope: z.number().int().nonnegative().describe('Executions the scope did not select'),
+    snapshots: z.number().int().nonnegative().describe('Raw recorded snapshots behind the counted executions'),
+    cohorts: z.array(ReliabilityRateSchema.extend({ environment: z.string().nullable(), producer: z.string(), parameters: z.record(z.string(), z.string()) })),
+    available: z.looseObject({
+        environments: z.array(FacetValueSchema), producers: z.array(FacetValueSchema), branches: z.array(FacetValueSchema), specRevisions: z.array(FacetValueSchema)
+    }).describe('What a scope could choose among, with independent executions'),
+    notes: z.array(z.looseObject({ code: z.string().describe('An open set: treat a code you do not know as informational'), severity: z.enum(['warning', 'info']), message: z.string() })),
+    executions: z.array(z.looseObject({
+        run: z.string(), at: z.number(), status: z.string(), counted: z.boolean(), excluded: z.string().optional(),
+        environment: z.string().nullable(), producer: z.string(), kind: z.string(), parameters: z.record(z.string(), z.string()),
+        branch: z.string().nullable(), sha: z.string().nullable(), runUrl: z.string().nullable(), layer: z.string(), backend: z.string(),
+        durationMs: z.number().nullable(), specRevision: z.string().optional(), snapshots: z.number().int().positive(),
+        attempts: z.array(z.string()).optional().describe('The status of each recorded try, in order'), firstAttempt: z.string().optional(),
+        republished: z.boolean().describe('A failed snapshot of this execution was followed by a later one')
+    })).describe('Every independent execution of the selection, newest first: the denominator, open to inspection')
+});
 /** A run: the layers that name the same execution (else share a run URL, else a branch and commit), with what they recorded. */
 export const ReportRunSchema = z.looseObject({
     key: z.string().describe('`run:<id>` for an execution a producer named, else the run URL, else `branch@sha`. Pass it to `/api/reports/runs/detail?key=` or `/api/reports/tests?run=`'),

package/dist/schema/src/layer.d.ts

@@ -74,16 +74,40 @@ export declare const LayerRunSchema: z.ZodObject<{
 }, z.core.$loose>;
 /** The execution of a layer, parsed from `LayerRunSchema`. */
 export type LayerRun = z.output<typeof LayerRunSchema>;
+/** The `kind` of a measurement that is a named, user-visible product latency (see docs/rich-evidence.md#product-latency). */
+export declare const PRODUCT_LATENCY_KIND = "product-latency";
+/** The two ends of a measured interval, named in the producer's words: `{ from: 'submit pressed', to: 'confirmation visible' }`. */
+export declare const MeasurementBoundarySchema: z.ZodObject<{
+    from: z.ZodOptional<z.ZodString>;
+    to: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
 /**
  * A number a check measured, such as a latency or a size, with its unit and an optional allowance. A measurement over
  * its `limit` is reported and never changes the check's status.
+ *
+ * A measurement of `kind` `product-latency` is how quickly the product responded to a user, between two named events.
+ * It is not the check's duration (which includes setup, waits and pacing) and a number with no `kind` is never read as
+ * product latency. All the fields after `limit` are optional and describe the measurement; none decides a status.
  */
 export declare const MeasurementSchema: z.ZodObject<{
     name: z.ZodString;
     value: z.ZodNumber;
     unit: z.ZodString;
     limit: z.ZodOptional<z.ZodNumber>;
+    kind: z.ZodOptional<z.ZodString>;
+    from: z.ZodOptional<z.ZodString>;
+    to: z.ZodOptional<z.ZodString>;
+    intended: z.ZodOptional<z.ZodObject<{
+        from: z.ZodOptional<z.ZodString>;
+        to: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>;
+    source: z.ZodOptional<z.ZodString>;
+    outcome: z.ZodOptional<z.ZodString>;
+    instrumentation: z.ZodOptional<z.ZodString>;
+    labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
 }, z.core.$loose>;
+/** A measurement as stored in a layer, parsed from `MeasurementSchema`. */
+export type Measurement = z.output<typeof MeasurementSchema>;
 /**
  * Structured data of a producer's or tool's own, by namespace: `{ "com.example.coverage": { … } }`.
  * Use a name you own (a reverse domain, or your tool's name). Readers preserve it and never fail on it.
@@ -867,6 +891,17 @@ export declare const CheckSchema: z.ZodObject<{
         value: z.ZodNumber;
         unit: z.ZodString;
         limit: z.ZodOptional<z.ZodNumber>;
+        kind: z.ZodOptional<z.ZodString>;
+        from: z.ZodOptional<z.ZodString>;
+        to: z.ZodOptional<z.ZodString>;
+        intended: z.ZodOptional<z.ZodObject<{
+            from: z.ZodOptional<z.ZodString>;
+            to: z.ZodOptional<z.ZodString>;
+        }, z.core.$loose>>;
+        source: z.ZodOptional<z.ZodString>;
+        outcome: z.ZodOptional<z.ZodString>;
+        instrumentation: z.ZodOptional<z.ZodString>;
+        labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     }, z.core.$loose>>>;
     durationMs: z.ZodOptional<z.ZodNumber>;
     recordingExpectation: z.ZodOptional<z.ZodString>;
@@ -1338,6 +1373,17 @@ export declare const ReportPartSchema: z.ZodObject<{
             value: z.ZodNumber;
             unit: z.ZodString;
             limit: z.ZodOptional<z.ZodNumber>;
+            kind: z.ZodOptional<z.ZodString>;
+            from: z.ZodOptional<z.ZodString>;
+            to: z.ZodOptional<z.ZodString>;
+            intended: z.ZodOptional<z.ZodObject<{
+                from: z.ZodOptional<z.ZodString>;
+                to: z.ZodOptional<z.ZodString>;
+            }, z.core.$loose>>;
+            source: z.ZodOptional<z.ZodString>;
+            outcome: z.ZodOptional<z.ZodString>;
+            instrumentation: z.ZodOptional<z.ZodString>;
+            labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
         }, z.core.$loose>>>;
         durationMs: z.ZodOptional<z.ZodNumber>;
         recordingExpectation: z.ZodOptional<z.ZodString>;
@@ -1708,6 +1754,17 @@ export declare const LayerSourceSchema: z.ZodObject<{
                 value: z.ZodNumber;
                 unit: z.ZodString;
                 limit: z.ZodOptional<z.ZodNumber>;
+                kind: z.ZodOptional<z.ZodString>;
+                from: z.ZodOptional<z.ZodString>;
+                to: z.ZodOptional<z.ZodString>;
+                intended: z.ZodOptional<z.ZodObject<{
+                    from: z.ZodOptional<z.ZodString>;
+                    to: z.ZodOptional<z.ZodString>;
+                }, z.core.$loose>>;
+                source: z.ZodOptional<z.ZodString>;
+                outcome: z.ZodOptional<z.ZodString>;
+                instrumentation: z.ZodOptional<z.ZodString>;
+                labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
             }, z.core.$loose>>>;
             durationMs: z.ZodOptional<z.ZodNumber>;
             recordingExpectation: z.ZodOptional<z.ZodString>;
@@ -2324,6 +2381,17 @@ export declare const LayerDocumentSchema: z.ZodObject<{
                 value: z.ZodNumber;
                 unit: z.ZodString;
                 limit: z.ZodOptional<z.ZodNumber>;
+                kind: z.ZodOptional<z.ZodString>;
+                from: z.ZodOptional<z.ZodString>;
+                to: z.ZodOptional<z.ZodString>;
+                intended: z.ZodOptional<z.ZodObject<{
+                    from: z.ZodOptional<z.ZodString>;
+                    to: z.ZodOptional<z.ZodString>;
+                }, z.core.$loose>>;
+                source: z.ZodOptional<z.ZodString>;
+                outcome: z.ZodOptional<z.ZodString>;
+                instrumentation: z.ZodOptional<z.ZodString>;
+                labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
             }, z.core.$loose>>>;
             durationMs: z.ZodOptional<z.ZodNumber>;
             recordingExpectation: z.ZodOptional<z.ZodString>;
@@ -2693,6 +2761,17 @@ export declare const LayerDocumentSchema: z.ZodObject<{
                     value: z.ZodNumber;
                     unit: z.ZodString;
                     limit: z.ZodOptional<z.ZodNumber>;
+                    kind: z.ZodOptional<z.ZodString>;
+                    from: z.ZodOptional<z.ZodString>;
+                    to: z.ZodOptional<z.ZodString>;
+                    intended: z.ZodOptional<z.ZodObject<{
+                        from: z.ZodOptional<z.ZodString>;
+                        to: z.ZodOptional<z.ZodString>;
+                    }, z.core.$loose>>;
+                    source: z.ZodOptional<z.ZodString>;
+                    outcome: z.ZodOptional<z.ZodString>;
+                    instrumentation: z.ZodOptional<z.ZodString>;
+                    labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 }, z.core.$loose>>>;
                 durationMs: z.ZodOptional<z.ZodNumber>;
                 recordingExpectation: z.ZodOptional<z.ZodString>;
@@ -2901,6 +2980,17 @@ export declare const PublishChecksSchema: z.ZodObject<{
             value: z.ZodNumber;
             unit: z.ZodString;
             limit: z.ZodOptional<z.ZodNumber>;
+            kind: z.ZodOptional<z.ZodString>;
+            from: z.ZodOptional<z.ZodString>;
+            to: z.ZodOptional<z.ZodString>;
+            intended: z.ZodOptional<z.ZodObject<{
+                from: z.ZodOptional<z.ZodString>;
+                to: z.ZodOptional<z.ZodString>;
+            }, z.core.$loose>>;
+            source: z.ZodOptional<z.ZodString>;
+            outcome: z.ZodOptional<z.ZodString>;
+            instrumentation: z.ZodOptional<z.ZodString>;
+            labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
         }, z.core.$loose>>>;
         durationMs: z.ZodOptional<z.ZodNumber>;
         recordingExpectation: z.ZodOptional<z.ZodString>;

package/dist/schema/src/layer.js

@@ -62,15 +62,34 @@ export const LayerRunSchema = z.looseObject({
     startedAtMs: z.number().nonnegative().optional().describe('When the execution started, epoch milliseconds'),
     endedAtMs: z.number().nonnegative().optional().describe('When the execution ended, epoch milliseconds')
 });
+/** The `kind` of a measurement that is a named, user-visible product latency (see docs/rich-evidence.md#product-latency). */
+export const PRODUCT_LATENCY_KIND = 'product-latency';
+/** The two ends of a measured interval, named in the producer's words: `{ from: 'submit pressed', to: 'confirmation visible' }`. */
+export const MeasurementBoundarySchema = z.looseObject({
+    from: z.string().min(1).optional().describe('The event the interval starts at: `end of speech`, `submit pressed`'),
+    to: z.string().min(1).optional().describe('The event the interval ends at: `audible response`, `confirmation visible`')
+});
 /**
  * A number a check measured, such as a latency or a size, with its unit and an optional allowance. A measurement over
  * its `limit` is reported and never changes the check's status.
+ *
+ * A measurement of `kind` `product-latency` is how quickly the product responded to a user, between two named events.
+ * It is not the check's duration (which includes setup, waits and pacing) and a number with no `kind` is never read as
+ * product latency. All the fields after `limit` are optional and describe the measurement; none decides a status.
  */
 export const MeasurementSchema = z.looseObject({
     name: z.string().min(1),
     value: z.number().describe('A finite number'),
     unit: z.string(),
-    limit: z.number().optional().describe('The allowance. A measurement over it is reported, and never fails the check by itself')
+    limit: z.number().optional().describe('The allowance, or budget. A measurement over it is reported, and never fails the check by itself'),
+    kind: z.string().min(1).optional().describe('What the number is, in an open vocabulary. `product-latency`: a user-visible response time between `from` and `to`. Absent: just a number, never read as product latency'),
+    from: z.string().min(1).optional().describe('The event the interval starts at. Without both `from` and `to` a latency is shown but no product claim is made from it'),
+    to: z.string().min(1).optional().describe('The event the interval ends at'),
+    intended: MeasurementBoundarySchema.optional().describe('The interval the name of the measurement claims, when what was measured is only part of it. If `intended.to` differs from `to` (or `intended.from` from `from`) the measurement is shown as partial, never as the complete interval'),
+    source: z.string().optional().describe('How it was measured: `app-instrumentation`, `trace`, `recording-analysis`'),
+    outcome: z.string().min(1).optional().describe('`observed` (the default), `timeout` or `failed`. For anything but `observed` the value is how long the observation waited: at least that much, so it counts against a budget and is never dropped'),
+    instrumentation: z.string().optional().describe('The version or method of the measurement. A change of it between two sets of observations is disclosed when they are compared'),
+    labels: z.record(z.string(), z.string()).optional().describe('What the observation is comparable with: `device`, `build`, `network`. `device` and `build` group the observations and, when two groups differ in the others, the comparison says so')
 });
 /**
  * Structured data of a producer's or tool's own, by namespace: `{ "com.example.coverage": { … } }`.

package/dist/server/src/app.js

@@ -28,6 +28,7 @@ import { listRunSummaries, readRunFile } from '../../reports/src/run-bundle.js';
 import { latestSpecSet, readSpecBody } from '../../reports/src/spec-set.js';
 import { catalogHealth, healthMarkdown, parseRootMappings } from '../../reports/src/catalog-health.js';
 import { parseArtifactFilter } from '../../reports/src/artifact-query.js';
+import { parseReliabilityScope, reliability, reliabilityMarkdown } from '../../reports/src/reliability.js';
 import { artifactList, artifactsMarkdown, catalogMarkdown, catalogModel, historyMarkdown, parseTestQuery, runDetail, runList, runMarkdown, runsMarkdown, testCatalog, testHistory } from '../../reports/src/test-catalog.js';
 import { llmsTxt } from './llms.js';
 import { openApiDocument } from './openapi.js';
@@ -467,6 +468,20 @@ function createProjectApplication(options) {
             return problemResponse(404, 'No test in the report has that key.');
         return answer(c, found.version, () => document, () => historyMarkdown(document), `history|${key}|${wantsMarkdown(c)}`);
     });
+    app.get('/api/reports/tests/reliability', async (c) => {
+        if (!index.ready)
+            return notReady(c);
+        const key = c.req.query('key');
+        if (!key)
+            return problemResponse(400, 'Name the test with ?key=, a key from GET /api/reports/tests.');
+        const found = await catalog();
+        const recorded = testHistory(found, key);
+        if (!recorded)
+            return problemResponse(404, 'No test in the report has that key.');
+        const search = new URL(c.req.url).searchParams;
+        const document = () => reliability(key, recorded.history, parseReliabilityScope(search));
+        return answer(c, found.version, document, () => reliabilityMarkdown(document()), `reliability|${search.toString()}|${wantsMarkdown(c)}`);
+    });
     app.get('/api/reports/runs', async (c) => {
         if (!index.ready)
             return notReady(c);

package/dist/server/src/llms.js

@@ -68,6 +68,7 @@ Every endpoint below answers JSON, or Markdown with \`format=markdown\`. Prefix
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests?format=markdown&limit=50"                 # summary, environments, domains, failing, flaky, slowest, then the tests
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests?status=failed,flaky&environment=desktop-web" # filter: q, status, environment, producer, domain, spec, tag, run; sort, offset, limit
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/history?key=<url-encoded test key>"         # one test: every recorded result, durations, branches, CI runs
+curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/reliability?key=<url-encoded test key>&environment=phone&format=markdown" # how reliable one test is in a scope: independent executions, first-try vs eventual success, what was excluded and why
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/health?format=markdown"                      # why Specs, domains and counts disagree; add root=<from>><to> to declare a path-root mapping
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/runs?format=markdown"                             # CI runs, newest first, with their results by status and video counts
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/runs?video=true&format=markdown"                 # only runs with a playable video recording

package/dist/server/src/openapi.js

@@ -14,7 +14,7 @@
  */
 import { z } from 'zod/v4';
 import { ProjectsSchema } from '../../schema/src/project.js';
-import { EffectiveDeltaSchema, EffectiveEvidenceSchema, EvidencePageSchema, HealthSchema, LayerRowsSchema, CheckDetailSchema, OutlineSchema, ProblemSchema, ReadySchema, ServerInfoSchema, StoreListingSchema, StoreWriteResultSchema, ChecksResultSchema, HelloEventSchema, ReportEventSchema, PingEventSchema, TestCatalogSchema, CatalogHealthSchema, TestHistorySchema, RunListSchema, RunDetailSchema, ArtifactListSchema, ArtifactTagEditSchema, ArtifactTagEntrySchema, ArtifactTagsSchema } from '../../schema/src/api.js';
+import { EffectiveDeltaSchema, EffectiveEvidenceSchema, EvidencePageSchema, HealthSchema, LayerRowsSchema, CheckDetailSchema, OutlineSchema, ProblemSchema, ReadySchema, ServerInfoSchema, StoreListingSchema, StoreWriteResultSchema, ChecksResultSchema, HelloEventSchema, ReportEventSchema, PingEventSchema, TestCatalogSchema, CatalogHealthSchema, TestHistorySchema, TestReliabilitySchema, RunListSchema, RunDetailSchema, ArtifactListSchema, ArtifactTagEditSchema, ArtifactTagEntrySchema, ArtifactTagsSchema } from '../../schema/src/api.js';
 import { LiveRunSchema, LiveRunStartSchema, LiveRunUpdateSchema, ServerEditableSettingsSchema, ServerSettingsSnapshotSchema, GcPolicySchema, GcSnapshotSchema, GcRunResultSchema, AtlasChangeSchema, PublishChecksSchema, ReportSourcesSchema, ReportStatusSchema, RunsIndexSchema, SpecIndexSchema } from '../../schema/src/index.js';
 /** The version of the API described, `major.minor.patch`: the minor grows when an operation or field is added, the major never within `/api`. */
 export const API_VERSION = '1.5.0';
@@ -37,7 +37,7 @@ const COMPONENTS = {
     StoreListing: [StoreListingSchema, 'output'], StoreWriteResult: [StoreWriteResultSchema, 'output'],
     Outline: [OutlineSchema, 'output'], EvidencePage: [EvidencePageSchema, 'output'], LayerRows: [LayerRowsSchema, 'output'], CheckDetail: [CheckDetailSchema, 'output'],
     ArtifactTagEdit: [ArtifactTagEditSchema, 'input'], ArtifactTagEntry: [ArtifactTagEntrySchema, 'output'], ArtifactTags: [ArtifactTagsSchema, 'output'],
-    TestCatalog: [TestCatalogSchema, 'output'], CatalogHealth: [CatalogHealthSchema, 'output'], TestHistory: [TestHistorySchema, 'output'], RunList: [RunListSchema, 'output'], ArtifactList: [ArtifactListSchema, 'output'], RunDetail: [RunDetailSchema, 'output'],
+    TestCatalog: [TestCatalogSchema, 'output'], CatalogHealth: [CatalogHealthSchema, 'output'], TestHistory: [TestHistorySchema, 'output'], TestReliability: [TestReliabilitySchema, 'output'], RunList: [RunListSchema, 'output'], ArtifactList: [ArtifactListSchema, 'output'], RunDetail: [RunDetailSchema, 'output'],
     SpecIndex: [SpecIndexSchema, 'output'], ReportSources: [ReportSourcesSchema, 'output'], RunsIndex: [RunsIndexSchema, 'output'],
     RunEvent: [LiveRunSchema, 'output'], HelloEvent: [HelloEventSchema, 'output'], ReportEvent: [ReportEventSchema, 'output'], AtlasChangeEvent: [AtlasChangeSchema, 'output'], PingEvent: [PingEventSchema, 'output']
 };
@@ -328,6 +328,29 @@ const BASE_OPERATIONS = [
         ],
         responses: { '200': json('The test and its history.', 'TestHistory'), '304': { description: 'Unchanged since the `If-None-Match` ETag.' }, '400': problem('No `key`.'), '401': UNAUTHORIZED, '404': problem('No test in the report has that key.'), '503': NOT_READY }
     },
+    {
+        route: 'GET /api/reports/tests/reliability', path: '/api/reports/tests/reliability', operationId: 'getTestReliability', tag: 'Report', role: 'read',
+        summary: 'How reliable one test is in a scope, and how that was counted',
+        description: [
+            'A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).',
+            '',
+            'Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.'
+        ].join('\n'),
+        parameters: [
+            { name: 'key', in: 'query', required: true, description: 'The test\'s `key` from `/api/reports/tests`.', schema: str, example: '["specs/checkout/pay.feature","Pay with a card"]' },
+            { name: 'environment', in: 'query', description: 'Environments or devices to count. Comma-separated.', schema: str, example: 'phone' },
+            { name: 'producer', in: 'query', description: 'Producers to count. Comma-separated.', schema: str },
+            { name: 'parameter', in: 'query', description: 'Parameter choices `key=value`; all must match. Comma-separated.', schema: str, example: 'device=small' },
+            { name: 'branch', in: 'query', description: 'Branches to count. Comma-separated.', schema: str },
+            { name: 'sha', in: 'query', description: 'Commits: a full id or a prefix of at least seven characters. Comma-separated.', schema: str },
+            { name: 'specRevision', in: 'query', description: 'Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.', schema: str },
+            { name: 'revision', in: 'query', description: '`any` pools every spec revision; otherwise only the newest recorded one counts.', schema: { type: 'string', enum: ['current', 'any'] } },
+            { name: 'since', in: 'query', description: 'Only executions at or after this time: epoch milliseconds or a date.', schema: str, example: '2026-01-01' },
+            { name: 'until', in: 'query', description: 'Only executions at or before this time: epoch milliseconds or a date.', schema: str },
+            { name: 'format', in: 'query', description: '`markdown` for Markdown instead of JSON.', schema: { type: 'string', enum: ['json', 'markdown'] } }
+        ],
+        responses: { '200': json('The reliability of the test in the scope.', 'TestReliability'), '304': { description: 'Unchanged since the `If-None-Match` ETag.' }, '400': problem('No `key`.'), '401': UNAUTHORIZED, '404': problem('No test in the report has that key.'), '503': NOT_READY }
+    },
     {
         route: 'GET /api/reports/artifact-tags', path: '/api/reports/artifact-tags', operationId: 'getArtifactTags', tag: 'Report', role: 'read',
         summary: 'Reviewer tags of attachments: every recorded edit',

package/docs/extending.md

@@ -34,7 +34,7 @@ carry any of these. Every one is optional.
 | `scope{}` | What the evidence does and does not show: surface, device kind, synthetic data, scripted dependencies, audio, pacing, what was asserted and its limits. Unknown when absent. See [Evidence scope](rich-evidence.md#evidence-scope). |
 | `links[]` | Where to read more: the CI run, the issue, the trace client. |
 | `extensions{}` | Structured data of your own or a tool's, under a name you own. |
-| `measurements[]` | Numbers, with units and an allowance (already part of the contract). |
+| `measurements[]` | Numbers, with units and an allowance (already part of the contract). A measurement of `kind` `product-latency` names a user-visible response time between two events; see [Product latency](rich-evidence.md#product-latency). |
 
 An attachment may carry its own `tags[]` (`demo`): shown as `#demo` and searchable; see
 [Tagging artifacts and demos](rich-evidence.md#tagging-artifacts-and-demos).

package/docs/reference/http-api.md

@@ -598,6 +598,54 @@ Scoped to one project. The same project may be selected on the unprefixed API us
 
 Responses: `200` The test and its history.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
 
+### How reliable one test is in a scope, and how that was counted
+
+`GET /api/reports/tests/reliability` · role: read
+
+A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).
+
+Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.
+
+- `key` (query, required): The test's `key` from `/api/reports/tests`.
+- `environment` (query): Environments or devices to count. Comma-separated.
+- `producer` (query): Producers to count. Comma-separated.
+- `parameter` (query): Parameter choices `key=value`; all must match. Comma-separated.
+- `branch` (query): Branches to count. Comma-separated.
+- `sha` (query): Commits: a full id or a prefix of at least seven characters. Comma-separated.
+- `specRevision` (query): Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.
+- `revision` (query): `any` pools every spec revision; otherwise only the newest recorded one counts.
+- `since` (query): Only executions at or after this time: epoch milliseconds or a date.
+- `until` (query): Only executions at or before this time: epoch milliseconds or a date.
+- `format` (query): `markdown` for Markdown instead of JSON.
+- `project` (query): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
+
+Responses: `200` The reliability of the test in the scope.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
+
+### How reliable one test is in a scope, and how that was counted
+
+`GET /api/projects/{project}/reports/tests/reliability` · role: read
+
+A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).
+
+Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.
+
+Scoped to one project. The same project may be selected on the unprefixed API using ?project= or X-Atlas-Project. Unknown projects return 404 on reads and are created by an authenticated upload.
+
+- `project` (path, required): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
+- `key` (query, required): The test's `key` from `/api/reports/tests`.
+- `environment` (query): Environments or devices to count. Comma-separated.
+- `producer` (query): Producers to count. Comma-separated.
+- `parameter` (query): Parameter choices `key=value`; all must match. Comma-separated.
+- `branch` (query): Branches to count. Comma-separated.
+- `sha` (query): Commits: a full id or a prefix of at least seven characters. Comma-separated.
+- `specRevision` (query): Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.
+- `revision` (query): `any` pools every spec revision; otherwise only the newest recorded one counts.
+- `since` (query): Only executions at or after this time: epoch milliseconds or a date.
+- `until` (query): Only executions at or before this time: epoch milliseconds or a date.
+- `format` (query): `markdown` for Markdown instead of JSON.
+
+Responses: `200` The reliability of the test in the scope.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
+
 ### Reviewer tags of attachments: every recorded edit
 
 `GET /api/reports/artifact-tags` · role: read