@crvouga/atlas 1.8.0 → 1.9.0

package/dist/schema/src/api.d.ts

@@ -121,6 +121,12 @@ export declare const EvidenceSummarySchema: z.ZodObject<{
     steps: z.ZodNumber;
     annotations: z.ZodNumber;
     attempts: z.ZodNumber;
+    attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        passed: "passed";
+        failed: "failed";
+        running: "running";
+        "not-run": "not-run";
+    }>>>;
     links: z.ZodNumber;
     tags: z.ZodArray<z.ZodString>;
     artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -191,6 +197,12 @@ export declare const LayerHeaderSchema: z.ZodObject<{
         steps: z.ZodNumber;
         annotations: z.ZodNumber;
         attempts: z.ZodNumber;
+        attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            passed: "passed";
+            failed: "failed";
+            running: "running";
+            "not-run": "not-run";
+        }>>>;
         links: z.ZodNumber;
         tags: z.ZodArray<z.ZodString>;
         artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -272,6 +284,12 @@ export declare const LayerRowSchema: z.ZodObject<{
         steps: z.ZodNumber;
         annotations: z.ZodNumber;
         attempts: z.ZodNumber;
+        attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            passed: "passed";
+            failed: "failed";
+            running: "running";
+            "not-run": "not-run";
+        }>>>;
         links: z.ZodNumber;
         tags: z.ZodArray<z.ZodString>;
         artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -351,6 +369,12 @@ export declare const LayerRowsSchema: z.ZodObject<{
             steps: z.ZodNumber;
             annotations: z.ZodNumber;
             attempts: z.ZodNumber;
+            attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                passed: "passed";
+                failed: "failed";
+                running: "running";
+                "not-run": "not-run";
+            }>>>;
             links: z.ZodNumber;
             tags: z.ZodArray<z.ZodString>;
             artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -432,6 +456,12 @@ export declare const LayerEvidenceSchema: z.ZodObject<{
             steps: z.ZodNumber;
             annotations: z.ZodNumber;
             attempts: z.ZodNumber;
+            attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                passed: "passed";
+                failed: "failed";
+                running: "running";
+                "not-run": "not-run";
+            }>>>;
             links: z.ZodNumber;
             tags: z.ZodArray<z.ZodString>;
             artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -507,6 +537,12 @@ export declare const LayerEvidenceSchema: z.ZodObject<{
             steps: z.ZodNumber;
             annotations: z.ZodNumber;
             attempts: z.ZodNumber;
+            attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                passed: "passed";
+                failed: "failed";
+                running: "running";
+                "not-run": "not-run";
+            }>>>;
             links: z.ZodNumber;
             tags: z.ZodArray<z.ZodString>;
             artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -597,6 +633,12 @@ export declare const EffectiveEvidenceSchema: z.ZodObject<{
                 steps: z.ZodNumber;
                 annotations: z.ZodNumber;
                 attempts: z.ZodNumber;
+                attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                    passed: "passed";
+                    failed: "failed";
+                    running: "running";
+                    "not-run": "not-run";
+                }>>>;
                 links: z.ZodNumber;
                 tags: z.ZodArray<z.ZodString>;
                 artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -672,6 +714,12 @@ export declare const EffectiveEvidenceSchema: z.ZodObject<{
                 steps: z.ZodNumber;
                 annotations: z.ZodNumber;
                 attempts: z.ZodNumber;
+                attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                    passed: "passed";
+                    failed: "failed";
+                    running: "running";
+                    "not-run": "not-run";
+                }>>>;
                 links: z.ZodNumber;
                 tags: z.ZodArray<z.ZodString>;
                 artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -783,6 +831,12 @@ export declare const EffectiveDeltaSchema: z.ZodObject<{
                 steps: z.ZodNumber;
                 annotations: z.ZodNumber;
                 attempts: z.ZodNumber;
+                attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                    passed: "passed";
+                    failed: "failed";
+                    running: "running";
+                    "not-run": "not-run";
+                }>>>;
                 links: z.ZodNumber;
                 tags: z.ZodArray<z.ZodString>;
                 artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -858,6 +912,12 @@ export declare const EffectiveDeltaSchema: z.ZodObject<{
                 steps: z.ZodNumber;
                 annotations: z.ZodNumber;
                 attempts: z.ZodNumber;
+                attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                    passed: "passed";
+                    failed: "failed";
+                    running: "running";
+                    "not-run": "not-run";
+                }>>>;
                 links: z.ZodNumber;
                 tags: z.ZodArray<z.ZodString>;
                 artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -961,6 +1021,12 @@ export declare const EvidencePageSchema: z.ZodObject<{
             steps: z.ZodNumber;
             annotations: z.ZodNumber;
             attempts: z.ZodNumber;
+            attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                passed: "passed";
+                failed: "failed";
+                running: "running";
+                "not-run": "not-run";
+            }>>>;
             links: z.ZodNumber;
             tags: z.ZodArray<z.ZodString>;
             artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -1525,11 +1591,18 @@ export declare const TestResultEntrySchema: z.ZodObject<{
     layer: z.ZodString;
     backend: z.ZodString;
     detail: z.ZodString;
+    specRevision: z.ZodOptional<z.ZodString>;
     evidence: z.ZodOptional<z.ZodObject<{
         attachments: z.ZodNumber;
         steps: z.ZodNumber;
         annotations: z.ZodNumber;
         attempts: z.ZodNumber;
+        attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            passed: "passed";
+            failed: "failed";
+            running: "running";
+            "not-run": "not-run";
+        }>>>;
         links: z.ZodNumber;
         tags: z.ZodArray<z.ZodString>;
         artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -1622,11 +1695,18 @@ export declare const TestSummarySchema: z.ZodObject<{
         layer: z.ZodString;
         backend: z.ZodString;
         detail: z.ZodString;
+        specRevision: z.ZodOptional<z.ZodString>;
         evidence: z.ZodOptional<z.ZodObject<{
             attachments: z.ZodNumber;
             steps: z.ZodNumber;
             annotations: z.ZodNumber;
             attempts: z.ZodNumber;
+            attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                passed: "passed";
+                failed: "failed";
+                running: "running";
+                "not-run": "not-run";
+            }>>>;
             links: z.ZodNumber;
             tags: z.ZodArray<z.ZodString>;
             artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -1914,11 +1994,18 @@ export declare const TestCatalogSchema: z.ZodObject<{
             layer: z.ZodString;
             backend: z.ZodString;
             detail: z.ZodString;
+            specRevision: z.ZodOptional<z.ZodString>;
             evidence: z.ZodOptional<z.ZodObject<{
                 attachments: z.ZodNumber;
                 steps: z.ZodNumber;
                 annotations: z.ZodNumber;
                 attempts: z.ZodNumber;
+                attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                    passed: "passed";
+                    failed: "failed";
+                    running: "running";
+                    "not-run": "not-run";
+                }>>>;
                 links: z.ZodNumber;
                 tags: z.ZodArray<z.ZodString>;
                 artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -2133,11 +2220,18 @@ export declare const TestHistorySchema: z.ZodObject<{
             layer: z.ZodString;
             backend: z.ZodString;
             detail: z.ZodString;
+            specRevision: z.ZodOptional<z.ZodString>;
             evidence: z.ZodOptional<z.ZodObject<{
                 attachments: z.ZodNumber;
                 steps: z.ZodNumber;
                 annotations: z.ZodNumber;
                 attempts: z.ZodNumber;
+                attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                    passed: "passed";
+                    failed: "failed";
+                    running: "running";
+                    "not-run": "not-run";
+                }>>>;
                 links: z.ZodNumber;
                 tags: z.ZodArray<z.ZodString>;
                 artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -2249,11 +2343,18 @@ export declare const TestHistorySchema: z.ZodObject<{
         layer: z.ZodString;
         backend: z.ZodString;
         detail: z.ZodString;
+        specRevision: z.ZodOptional<z.ZodString>;
         evidence: z.ZodOptional<z.ZodObject<{
             attachments: z.ZodNumber;
             steps: z.ZodNumber;
             annotations: z.ZodNumber;
             attempts: z.ZodNumber;
+            attemptStatuses: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+                passed: "passed";
+                failed: "failed";
+                running: "running";
+                "not-run": "not-run";
+            }>>>;
             links: z.ZodNumber;
             tags: z.ZodArray<z.ZodString>;
             artifactTags: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -2307,6 +2408,131 @@ export declare const TestHistorySchema: z.ZodObject<{
         }, z.core.$loose>>>;
     }, z.core.$loose>>;
 }, z.core.$loose>;
+/**
+ * `GET /api/reports/tests/reliability`: how reliable one test is inside a scope, and exactly how that figure was counted.
+ * The snapshots one run recorded for one producer, environment and parameter set are one independent execution; only
+ * `passed` and `failed` executions are trials; `notes` say when the sample is too small, pooled, selectively rerun or
+ * bounded. It reads results and never changes a status. It is not a production failure probability.
+ */
+export declare const TestReliabilitySchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<1>;
+    key: z.ZodString;
+    scope: z.ZodObject<{
+        environment: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        producer: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        parameter: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        branch: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        sha: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        specRevision: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        revision: z.ZodOptional<z.ZodEnum<{
+            any: "any";
+            current: "current";
+        }>>;
+        since: z.ZodOptional<z.ZodNumber>;
+        until: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$loose>;
+    minimumExecutions: z.ZodNumber;
+    sample: z.ZodEnum<{
+        none: "none";
+        insufficient: "insufficient";
+        sufficient: "sufficient";
+    }>;
+    eventual: z.ZodObject<{
+        executions: z.ZodNumber;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        rate: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$loose>;
+    interval: z.ZodNullable<z.ZodObject<{
+        method: z.ZodLiteral<"wilson">;
+        level: z.ZodNumber;
+        lower: z.ZodNumber;
+        upper: z.ZodNumber;
+        assumptions: z.ZodString;
+    }, z.core.$loose>>;
+    firstAttempt: z.ZodObject<{
+        executions: z.ZodNumber;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        rate: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$loose>;
+    unrecordedAttempts: z.ZodNumber;
+    attempts: z.ZodObject<{
+        total: z.ZodNumber;
+        retried: z.ZodNumber;
+        recovered: z.ZodNumber;
+    }, z.core.$loose>;
+    currentRevision: z.ZodNullable<z.ZodString>;
+    historical: z.ZodObject<{
+        executions: z.ZodNumber;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        revisions: z.ZodArray<z.ZodObject<{
+            specRevision: z.ZodString;
+            executions: z.ZodNumber;
+        }, z.core.$loose>>;
+    }, z.core.$loose>;
+    excluded: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    outsideScope: z.ZodNumber;
+    snapshots: z.ZodNumber;
+    cohorts: z.ZodArray<z.ZodObject<{
+        executions: z.ZodNumber;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        rate: z.ZodNullable<z.ZodNumber>;
+        environment: z.ZodNullable<z.ZodString>;
+        producer: z.ZodString;
+        parameters: z.ZodRecord<z.ZodString, z.ZodString>;
+    }, z.core.$loose>>;
+    available: z.ZodObject<{
+        environments: z.ZodArray<z.ZodObject<{
+            value: z.ZodString;
+            count: z.ZodNumber;
+        }, z.core.$loose>>;
+        producers: z.ZodArray<z.ZodObject<{
+            value: z.ZodString;
+            count: z.ZodNumber;
+        }, z.core.$loose>>;
+        branches: z.ZodArray<z.ZodObject<{
+            value: z.ZodString;
+            count: z.ZodNumber;
+        }, z.core.$loose>>;
+        specRevisions: z.ZodArray<z.ZodObject<{
+            value: z.ZodString;
+            count: z.ZodNumber;
+        }, z.core.$loose>>;
+    }, z.core.$loose>;
+    notes: z.ZodArray<z.ZodObject<{
+        code: z.ZodString;
+        severity: z.ZodEnum<{
+            info: "info";
+            warning: "warning";
+        }>;
+        message: z.ZodString;
+    }, z.core.$loose>>;
+    executions: z.ZodArray<z.ZodObject<{
+        run: z.ZodString;
+        at: z.ZodNumber;
+        status: z.ZodString;
+        counted: z.ZodBoolean;
+        excluded: z.ZodOptional<z.ZodString>;
+        environment: z.ZodNullable<z.ZodString>;
+        producer: z.ZodString;
+        kind: z.ZodString;
+        parameters: z.ZodRecord<z.ZodString, z.ZodString>;
+        branch: z.ZodNullable<z.ZodString>;
+        sha: z.ZodNullable<z.ZodString>;
+        runUrl: z.ZodNullable<z.ZodString>;
+        layer: z.ZodString;
+        backend: z.ZodString;
+        durationMs: z.ZodNullable<z.ZodNumber>;
+        specRevision: z.ZodOptional<z.ZodString>;
+        snapshots: z.ZodNumber;
+        attempts: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        firstAttempt: z.ZodOptional<z.ZodString>;
+        republished: z.ZodBoolean;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
 /** A run: the layers that name the same execution (else share a run URL, else a branch and commit), with what they recorded. */
 export declare const ReportRunSchema: z.ZodObject<{
     key: z.ZodString;

package/dist/schema/src/api.js

@@ -85,6 +85,7 @@ export const EvidenceSummarySchema = z.looseObject({
     steps: z.number().int().nonnegative(),
     annotations: z.number().int().nonnegative(),
     attempts: z.number().int().nonnegative(),
+    attemptStatuses: z.array(EvidenceStatusSchema).optional().describe('The status of each recorded try, in the order they were made. Absent when no try was recorded, and in reports written before this was recorded: a first try is then unknown, never assumed to have passed'),
     links: z.number().int().nonnegative(),
     tags: z.array(z.string()),
     artifactTags: z.array(z.string()).optional().describe('Normalized tags of the attachments themselves, sorted and distinct. Absent when no attachment is tagged; never inherited from a check or run'),
@@ -270,6 +271,7 @@ export const TestResultEntrySchema = z.looseObject({
     layer: z.string().describe('The id of the layer that recorded it'),
     backend: z.string(),
     detail: z.string().describe('Why it has its status, at most 500 characters'),
+    specRevision: z.string().optional().describe('The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match'),
     evidence: EvidenceSummarySchema.optional(),
     latency: z.array(MeasurementSchema).optional().describe('Its product-latency measurements (`kind: product-latency`), when it recorded any. Separate from `durationMs`, which is how long the test took')
 });
@@ -390,6 +392,55 @@ export const TestHistorySchema = z.looseObject({
     test: TestSummarySchema,
     history: z.array(TestResultEntrySchema)
 });
+const ReliabilityRateSchema = z.looseObject({
+    executions: z.number().int().nonnegative().describe('Independent executions that passed or failed'),
+    passed: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative(),
+    rate: z.number().nullable().describe('passed / executions, or null with none')
+});
+/**
+ * `GET /api/reports/tests/reliability`: how reliable one test is inside a scope, and exactly how that figure was counted.
+ * The snapshots one run recorded for one producer, environment and parameter set are one independent execution; only
+ * `passed` and `failed` executions are trials; `notes` say when the sample is too small, pooled, selectively rerun or
+ * bounded. It reads results and never changes a status. It is not a production failure probability.
+ */
+export const TestReliabilitySchema = z.looseObject({
+    schemaVersion: z.literal(1),
+    key: z.string().describe('The test\'s key'),
+    scope: z.looseObject({
+        environment: z.array(z.string()).optional(), producer: z.array(z.string()).optional(), parameter: z.array(z.string()).optional(),
+        branch: z.array(z.string()).optional(), sha: z.array(z.string()).optional(), specRevision: z.array(z.string()).optional(),
+        revision: z.enum(['current', 'any']).optional(), since: z.number().optional(), until: z.number().optional()
+    }).describe('What was asked: the selection the figures are about'),
+    minimumExecutions: z.number().int().positive().describe('Completed executions below which the sample is `insufficient`'),
+    sample: z.enum(['none', 'insufficient', 'sufficient']).describe('A count only: it says nothing about whether the executions are representative; read `notes`'),
+    eventual: ReliabilityRateSchema.describe('Executions that passed in the end, over those that passed or failed'),
+    interval: z.looseObject({ method: z.literal('wilson'), level: z.number(), lower: z.number(), upper: z.number(), assumptions: z.string() }).nullable().describe('An interval estimate with the assumptions it needs; null without an execution'),
+    firstAttempt: ReliabilityRateSchema.describe('Over the executions that recorded their tries: those whose first try passed'),
+    unrecordedAttempts: z.number().int().nonnegative().describe('Counted executions that recorded no tries: their first try is unknown, never assumed to have passed'),
+    attempts: z.looseObject({ total: z.number().int().nonnegative(), retried: z.number().int().nonnegative(), recovered: z.number().int().nonnegative() }),
+    currentRevision: z.string().nullable().describe('The spec revision `revision=current` counted, or null'),
+    historical: z.looseObject({
+        executions: z.number().int().nonnegative(), passed: z.number().int().nonnegative(), failed: z.number().int().nonnegative(),
+        revisions: z.array(z.looseObject({ specRevision: z.string(), executions: z.number().int().nonnegative() }))
+    }).describe('Earlier spec revisions: context only, counted nowhere above'),
+    excluded: z.record(z.string(), z.number().int().nonnegative()).describe('Executions in the scope that are not trials, by reason: `not-run`, `unfinished`, `unknown-status`, `historical-revision`, `revision-not-recorded`'),
+    outsideScope: z.number().int().nonnegative().describe('Executions the scope did not select'),
+    snapshots: z.number().int().nonnegative().describe('Raw recorded snapshots behind the counted executions'),
+    cohorts: z.array(ReliabilityRateSchema.extend({ environment: z.string().nullable(), producer: z.string(), parameters: z.record(z.string(), z.string()) })),
+    available: z.looseObject({
+        environments: z.array(FacetValueSchema), producers: z.array(FacetValueSchema), branches: z.array(FacetValueSchema), specRevisions: z.array(FacetValueSchema)
+    }).describe('What a scope could choose among, with independent executions'),
+    notes: z.array(z.looseObject({ code: z.string().describe('An open set: treat a code you do not know as informational'), severity: z.enum(['warning', 'info']), message: z.string() })),
+    executions: z.array(z.looseObject({
+        run: z.string(), at: z.number(), status: z.string(), counted: z.boolean(), excluded: z.string().optional(),
+        environment: z.string().nullable(), producer: z.string(), kind: z.string(), parameters: z.record(z.string(), z.string()),
+        branch: z.string().nullable(), sha: z.string().nullable(), runUrl: z.string().nullable(), layer: z.string(), backend: z.string(),
+        durationMs: z.number().nullable(), specRevision: z.string().optional(), snapshots: z.number().int().positive(),
+        attempts: z.array(z.string()).optional().describe('The status of each recorded try, in order'), firstAttempt: z.string().optional(),
+        republished: z.boolean().describe('A failed snapshot of this execution was followed by a later one')
+    })).describe('Every independent execution of the selection, newest first: the denominator, open to inspection')
+});
 /** A run: the layers that name the same execution (else share a run URL, else a branch and commit), with what they recorded. */
 export const ReportRunSchema = z.looseObject({
     key: z.string().describe('`run:<id>` for an execution a producer named, else the run URL, else `branch@sha`. Pass it to `/api/reports/runs/detail?key=` or `/api/reports/tests?run=`'),

package/dist/server/src/app.js

@@ -28,6 +28,7 @@ import { listRunSummaries, readRunFile } from '../../reports/src/run-bundle.js';
 import { latestSpecSet, readSpecBody } from '../../reports/src/spec-set.js';
 import { catalogHealth, healthMarkdown, parseRootMappings } from '../../reports/src/catalog-health.js';
 import { parseArtifactFilter } from '../../reports/src/artifact-query.js';
+import { parseReliabilityScope, reliability, reliabilityMarkdown } from '../../reports/src/reliability.js';
 import { artifactList, artifactsMarkdown, catalogMarkdown, catalogModel, historyMarkdown, parseTestQuery, runDetail, runList, runMarkdown, runsMarkdown, testCatalog, testHistory } from '../../reports/src/test-catalog.js';
 import { llmsTxt } from './llms.js';
 import { openApiDocument } from './openapi.js';
@@ -467,6 +468,20 @@ function createProjectApplication(options) {
             return problemResponse(404, 'No test in the report has that key.');
         return answer(c, found.version, () => document, () => historyMarkdown(document), `history|${key}|${wantsMarkdown(c)}`);
     });
+    app.get('/api/reports/tests/reliability', async (c) => {
+        if (!index.ready)
+            return notReady(c);
+        const key = c.req.query('key');
+        if (!key)
+            return problemResponse(400, 'Name the test with ?key=, a key from GET /api/reports/tests.');
+        const found = await catalog();
+        const recorded = testHistory(found, key);
+        if (!recorded)
+            return problemResponse(404, 'No test in the report has that key.');
+        const search = new URL(c.req.url).searchParams;
+        const document = () => reliability(key, recorded.history, parseReliabilityScope(search));
+        return answer(c, found.version, document, () => reliabilityMarkdown(document()), `reliability|${search.toString()}|${wantsMarkdown(c)}`);
+    });
     app.get('/api/reports/runs', async (c) => {
         if (!index.ready)
             return notReady(c);

package/dist/server/src/llms.js

@@ -68,6 +68,7 @@ Every endpoint below answers JSON, or Markdown with \`format=markdown\`. Prefix
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests?format=markdown&limit=50"                 # summary, environments, domains, failing, flaky, slowest, then the tests
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests?status=failed,flaky&environment=desktop-web" # filter: q, status, environment, producer, domain, spec, tag, run; sort, offset, limit
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/history?key=<url-encoded test key>"         # one test: every recorded result, durations, branches, CI runs
+curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/reliability?key=<url-encoded test key>&environment=phone&format=markdown" # how reliable one test is in a scope: independent executions, first-try vs eventual success, what was excluded and why
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/health?format=markdown"                      # why Specs, domains and counts disagree; add root=<from>><to> to declare a path-root mapping
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/runs?format=markdown"                             # CI runs, newest first, with their results by status and video counts
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/runs?video=true&format=markdown"                 # only runs with a playable video recording

package/dist/server/src/openapi.js

@@ -14,7 +14,7 @@
  */
 import { z } from 'zod/v4';
 import { ProjectsSchema } from '../../schema/src/project.js';
-import { EffectiveDeltaSchema, EffectiveEvidenceSchema, EvidencePageSchema, HealthSchema, LayerRowsSchema, CheckDetailSchema, OutlineSchema, ProblemSchema, ReadySchema, ServerInfoSchema, StoreListingSchema, StoreWriteResultSchema, ChecksResultSchema, HelloEventSchema, ReportEventSchema, PingEventSchema, TestCatalogSchema, CatalogHealthSchema, TestHistorySchema, RunListSchema, RunDetailSchema, ArtifactListSchema, ArtifactTagEditSchema, ArtifactTagEntrySchema, ArtifactTagsSchema } from '../../schema/src/api.js';
+import { EffectiveDeltaSchema, EffectiveEvidenceSchema, EvidencePageSchema, HealthSchema, LayerRowsSchema, CheckDetailSchema, OutlineSchema, ProblemSchema, ReadySchema, ServerInfoSchema, StoreListingSchema, StoreWriteResultSchema, ChecksResultSchema, HelloEventSchema, ReportEventSchema, PingEventSchema, TestCatalogSchema, CatalogHealthSchema, TestHistorySchema, TestReliabilitySchema, RunListSchema, RunDetailSchema, ArtifactListSchema, ArtifactTagEditSchema, ArtifactTagEntrySchema, ArtifactTagsSchema } from '../../schema/src/api.js';
 import { LiveRunSchema, LiveRunStartSchema, LiveRunUpdateSchema, ServerEditableSettingsSchema, ServerSettingsSnapshotSchema, GcPolicySchema, GcSnapshotSchema, GcRunResultSchema, AtlasChangeSchema, PublishChecksSchema, ReportSourcesSchema, ReportStatusSchema, RunsIndexSchema, SpecIndexSchema } from '../../schema/src/index.js';
 /** The version of the API described, `major.minor.patch`: the minor grows when an operation or field is added, the major never within `/api`. */
 export const API_VERSION = '1.5.0';
@@ -37,7 +37,7 @@ const COMPONENTS = {
     StoreListing: [StoreListingSchema, 'output'], StoreWriteResult: [StoreWriteResultSchema, 'output'],
     Outline: [OutlineSchema, 'output'], EvidencePage: [EvidencePageSchema, 'output'], LayerRows: [LayerRowsSchema, 'output'], CheckDetail: [CheckDetailSchema, 'output'],
     ArtifactTagEdit: [ArtifactTagEditSchema, 'input'], ArtifactTagEntry: [ArtifactTagEntrySchema, 'output'], ArtifactTags: [ArtifactTagsSchema, 'output'],
-    TestCatalog: [TestCatalogSchema, 'output'], CatalogHealth: [CatalogHealthSchema, 'output'], TestHistory: [TestHistorySchema, 'output'], RunList: [RunListSchema, 'output'], ArtifactList: [ArtifactListSchema, 'output'], RunDetail: [RunDetailSchema, 'output'],
+    TestCatalog: [TestCatalogSchema, 'output'], CatalogHealth: [CatalogHealthSchema, 'output'], TestHistory: [TestHistorySchema, 'output'], TestReliability: [TestReliabilitySchema, 'output'], RunList: [RunListSchema, 'output'], ArtifactList: [ArtifactListSchema, 'output'], RunDetail: [RunDetailSchema, 'output'],
     SpecIndex: [SpecIndexSchema, 'output'], ReportSources: [ReportSourcesSchema, 'output'], RunsIndex: [RunsIndexSchema, 'output'],
     RunEvent: [LiveRunSchema, 'output'], HelloEvent: [HelloEventSchema, 'output'], ReportEvent: [ReportEventSchema, 'output'], AtlasChangeEvent: [AtlasChangeSchema, 'output'], PingEvent: [PingEventSchema, 'output']
 };
@@ -328,6 +328,29 @@ const BASE_OPERATIONS = [
         ],
         responses: { '200': json('The test and its history.', 'TestHistory'), '304': { description: 'Unchanged since the `If-None-Match` ETag.' }, '400': problem('No `key`.'), '401': UNAUTHORIZED, '404': problem('No test in the report has that key.'), '503': NOT_READY }
     },
+    {
+        route: 'GET /api/reports/tests/reliability', path: '/api/reports/tests/reliability', operationId: 'getTestReliability', tag: 'Report', role: 'read',
+        summary: 'How reliable one test is in a scope, and how that was counted',
+        description: [
+            'A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).',
+            '',
+            'Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.'
+        ].join('\n'),
+        parameters: [
+            { name: 'key', in: 'query', required: true, description: 'The test\'s `key` from `/api/reports/tests`.', schema: str, example: '["specs/checkout/pay.feature","Pay with a card"]' },
+            { name: 'environment', in: 'query', description: 'Environments or devices to count. Comma-separated.', schema: str, example: 'phone' },
+            { name: 'producer', in: 'query', description: 'Producers to count. Comma-separated.', schema: str },
+            { name: 'parameter', in: 'query', description: 'Parameter choices `key=value`; all must match. Comma-separated.', schema: str, example: 'device=small' },
+            { name: 'branch', in: 'query', description: 'Branches to count. Comma-separated.', schema: str },
+            { name: 'sha', in: 'query', description: 'Commits: a full id or a prefix of at least seven characters. Comma-separated.', schema: str },
+            { name: 'specRevision', in: 'query', description: 'Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.', schema: str },
+            { name: 'revision', in: 'query', description: '`any` pools every spec revision; otherwise only the newest recorded one counts.', schema: { type: 'string', enum: ['current', 'any'] } },
+            { name: 'since', in: 'query', description: 'Only executions at or after this time: epoch milliseconds or a date.', schema: str, example: '2026-01-01' },
+            { name: 'until', in: 'query', description: 'Only executions at or before this time: epoch milliseconds or a date.', schema: str },
+            { name: 'format', in: 'query', description: '`markdown` for Markdown instead of JSON.', schema: { type: 'string', enum: ['json', 'markdown'] } }
+        ],
+        responses: { '200': json('The reliability of the test in the scope.', 'TestReliability'), '304': { description: 'Unchanged since the `If-None-Match` ETag.' }, '400': problem('No `key`.'), '401': UNAUTHORIZED, '404': problem('No test in the report has that key.'), '503': NOT_READY }
+    },
     {
         route: 'GET /api/reports/artifact-tags', path: '/api/reports/artifact-tags', operationId: 'getArtifactTags', tag: 'Report', role: 'read',
         summary: 'Reviewer tags of attachments: every recorded edit',

package/docs/reference/http-api.md

@@ -598,6 +598,54 @@ Scoped to one project. The same project may be selected on the unprefixed API us
 
 Responses: `200` The test and its history.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
 
+### How reliable one test is in a scope, and how that was counted
+
+`GET /api/reports/tests/reliability` · role: read
+
+A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).
+
+Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.
+
+- `key` (query, required): The test's `key` from `/api/reports/tests`.
+- `environment` (query): Environments or devices to count. Comma-separated.
+- `producer` (query): Producers to count. Comma-separated.
+- `parameter` (query): Parameter choices `key=value`; all must match. Comma-separated.
+- `branch` (query): Branches to count. Comma-separated.
+- `sha` (query): Commits: a full id or a prefix of at least seven characters. Comma-separated.
+- `specRevision` (query): Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.
+- `revision` (query): `any` pools every spec revision; otherwise only the newest recorded one counts.
+- `since` (query): Only executions at or after this time: epoch milliseconds or a date.
+- `until` (query): Only executions at or before this time: epoch milliseconds or a date.
+- `format` (query): `markdown` for Markdown instead of JSON.
+- `project` (query): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
+
+Responses: `200` The reliability of the test in the scope.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
+
+### How reliable one test is in a scope, and how that was counted
+
+`GET /api/projects/{project}/reports/tests/reliability` · role: read
+
+A success rate for one test that says what it counted. The snapshots one run recorded for one producer, environment and parameter set (running, finished, enriched) are one independent execution; its last word counts once with the tries it recorded. `eventual` is the share of executions that passed in the end, `firstAttempt` the share whose first try passed (over executions that recorded their tries; the rest are `unrecordedAttempts`, never assumed to have passed first time).
+
+Choose the scope with `environment`, `producer`, `parameter` (`key=value`), `branch`, `sha`, `specRevision`, `since` and `until`; executions elsewhere never count. By default only the newest recorded spec revision counts and earlier ones appear under `historical`; send `revision=any` to pool them. Only `passed` and `failed` executions are trials: `not-run` and unfinished ones are listed under `excluded` and never counted as a pass. `sample` is `none`, `insufficient` (fewer than `minimumExecutions`) or `sufficient`, a count only: `notes` say when environments, producers or branches are pooled, failed cases were published again, or history is bounded to the newest 100 results. `executions` is the denominator, open to inspection. The interval states its assumptions. This is not a production failure probability. Send `format=markdown` for Markdown.
+
+Scoped to one project. The same project may be selected on the unprefixed API using ?project= or X-Atlas-Project. Unknown projects return 404 on reads and are created by an authenticated upload.
+
+- `project` (path, required): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
+- `key` (query, required): The test's `key` from `/api/reports/tests`.
+- `environment` (query): Environments or devices to count. Comma-separated.
+- `producer` (query): Producers to count. Comma-separated.
+- `parameter` (query): Parameter choices `key=value`; all must match. Comma-separated.
+- `branch` (query): Branches to count. Comma-separated.
+- `sha` (query): Commits: a full id or a prefix of at least seven characters. Comma-separated.
+- `specRevision` (query): Only executions recorded against these spec fingerprints (overrides `revision`). Comma-separated.
+- `revision` (query): `any` pools every spec revision; otherwise only the newest recorded one counts.
+- `since` (query): Only executions at or after this time: epoch milliseconds or a date.
+- `until` (query): Only executions at or before this time: epoch milliseconds or a date.
+- `format` (query): `markdown` for Markdown instead of JSON.
+
+Responses: `200` The reliability of the test in the scope.; `304` Unchanged since the `If-None-Match` ETag.; `400` No `key`.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `404` No test in the report has that key.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.
+
 ### Reviewer tags of attachments: every recorded edit
 
 `GET /api/reports/artifact-tags` · role: read