@crvouga/atlas 0.9.0 → 0.10.0

package/dist/reports/src/test-catalog.d.ts

@@ -0,0 +1,386 @@
+/**
+ * @module Test catalog, history and runs
+ *
+ * What a person or an agent asks of a report once it holds more than a handful of checks: every test with where it ran,
+ * how it went over time and how long it takes; the specs, domains, environments and producers they fall under; the CI
+ * runs that produced them and which tests each one ran.
+ *
+ * - `TestHistory` keeps every recorded result of every test (bounded per test) and every run's results, folded in one
+ *   layer at a time, in any order, with the same outcome. It never decides a status: the current status of a test is
+ *   always the report's winner (`combineEvidence`), and history only describes how it got there.
+ * - `testCatalog` turns the report's winners (plus a history, when there is one) into a `TestCatalog`: a summary with
+ *   insights (slowest, flaky, failing), facets to filter by, and the tests that match a `TestQuery`.
+ * - `testHistory`, `runList` and `runDetail` answer for one test, every run, and one run.
+ * - `catalogMarkdown`, `historyMarkdown`, `runsMarkdown` and `runMarkdown` say the same in Markdown, for an agent or a
+ *   terminal.
+ *
+ * A test is a check identity (its spec path and id) regardless of producer, environment and parameters; each of those
+ * combinations is one of its results. A result that is not `passed` is never counted as a pass, and a test with no
+ * completed result has no pass rate.
+ *
+ * @see docs/server.md#reading-the-report-as-an-agent
+ */
+import type { EvidenceStatus, SpecEntry } from './explorer-types.js';
+import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts } from './evidence-types.js';
+import { RecordedSpecs } from './recorded-specs.js';
+/** The version of the documents this module writes (`TestCatalog`, `TestHistoryDocument`, `RunList`, `RunDetail`). Only optional fields are added within it. */
+export declare const TEST_CATALOG_VERSION = 1;
+/** Recorded results kept per test, newest first. Older ones are counted in no statistic. */
+export declare const HISTORY_PER_TEST = 100;
+/** One recorded result of a test: when, where, how it went and how long it took, and the layer and run it came from. */
+export type TestResultEntry = {
+    /** When it was recorded, epoch ms. */
+    readonly at: number;
+    readonly status: EvidenceStatus;
+    /** How long it took, ms, or null when the producer did not measure it. */
+    readonly durationMs: number | null;
+    /** The client it checked (`desktop-web`, `api`, `ios`…), or null. */
+    readonly environment: string | null;
+    readonly producer: string;
+    /** The kind of the part that held it (`checks`, `behavior`, `performance`…). */
+    readonly kind: string;
+    /** Other choices that made it its own result (a device size, a locale…). */
+    readonly parameters: Readonly<Record<string, string>>;
+    readonly branch: string | null;
+    readonly sha: string | null;
+    readonly runUrl: string | null;
+    /** The key of the run it belongs to (see `runKey`). */
+    readonly run: string;
+    /** The id of the layer that recorded it. */
+    readonly layer: string;
+    readonly backend: string;
+    /** Why it has its status: the first 500 characters of the check's detail. */
+    readonly detail: string;
+    /** What it carried beyond its outcome (attachments, steps, retries, tags, labels), when anything. */
+    readonly evidence?: EvidenceSummary;
+};
+/** The current result of a test in one producer, environment and parameter set: the report's winner for that slot. */
+export type TestOutcome = TestResultEntry;
+/** Durations of a set of results, ms. Only results that recorded one are counted. */
+export type DurationStats = {
+    readonly count: number;
+    readonly totalMs: number;
+    readonly avgMs: number;
+    readonly p50Ms: number;
+    readonly p95Ms: number;
+    readonly minMs: number;
+    readonly maxMs: number;
+    /** The newest recorded duration. */
+    readonly lastMs: number;
+};
+/** How a test has gone across its recorded history (at most `HISTORY_PER_TEST` results). */
+export type TestStats = {
+    /** Recorded results: passed, failed and not-run (progress reported as `running` is not history). */
+    readonly results: number;
+    readonly passed: number;
+    readonly failed: number;
+    /** passed / (passed + failed), or null with no completed result. */
+    readonly passRate: number | null;
+    /** True when the same code both passed and failed, a retry was needed to pass, or the producer said so. */
+    readonly flaky: boolean;
+    /** Why it is flaky, for a person; empty when it is not. */
+    readonly flakyReason: string;
+    /** How many times the completed outcome changed, per result, newest history first. */
+    readonly flips: number;
+    readonly durations: DurationStats | null;
+    readonly firstSeenMs: number | null;
+    readonly lastSeenMs: number | null;
+    readonly lastPassedAtMs: number | null;
+    readonly lastFailedAtMs: number | null;
+    /** Distinct runs it was recorded in. */
+    readonly runs: number;
+    /** Branches it was recorded on, most recent first, at most ten. */
+    readonly branches: readonly string[];
+};
+/** A spec as the catalog shows it: recorded by a layer or published in a spec set, or derived from the path alone. */
+export type CatalogSpec = {
+    readonly path: string;
+    readonly title: string;
+    /** The group it belongs to: the spec's own domain, else the folder under `specs/`. */
+    readonly domain: string;
+    readonly kind: string;
+};
+/** One test: what it is, its current status per environment, and how it has gone. */
+export type TestSummary = {
+    /** The check identity, `JSON.stringify([specPath, id])`. Stable; use it to ask for the test's history. */
+    readonly key: string;
+    readonly id: string;
+    readonly title: string;
+    readonly specPath: string | null;
+    readonly spec: CatalogSpec | null;
+    /** The worst current outcome across its results (failed, then running, then not-run, then passed). */
+    readonly status: EvidenceStatus;
+    readonly environments: readonly string[];
+    readonly producers: readonly string[];
+    /** The current result per producer, environment and parameter set, newest first. */
+    readonly outcomes: readonly TestOutcome[];
+    readonly stats: TestStats;
+    readonly tags: readonly string[];
+    readonly labels: Readonly<Record<string, string>>;
+    /** What its current results carry: attachments, steps, retries, and the kinds of attachment. */
+    readonly evidence: {
+        readonly attachments: number;
+        readonly steps: number;
+        readonly attempts: number;
+        readonly roles: readonly string[];
+    };
+    /** The newest time any of its results was recorded. */
+    readonly lastRecordedAtMs: number;
+    /** True while one of its results comes from a run still in progress. */
+    readonly live: boolean;
+};
+/** A test named in an insight list, with the number that put it there. */
+export type TestRef = {
+    readonly key: string;
+    readonly title: string;
+    readonly specPath: string | null;
+    readonly status: EvidenceStatus;
+    /** Average duration (ms) for `slowest`, flips for `flaky`, failures for `failing`. */
+    readonly value: number;
+};
+/** A group in the summary (an environment, a producer, a domain, a spec): how many tests, by status (for an environment, by how they went on it), and their timing. */
+export type CatalogGroup = {
+    readonly name: string;
+    readonly tests: number;
+    readonly counts: StatusCounts;
+    /** Average of its tests' average durations, ms, or null when none was measured. */
+    readonly avgMs: number | null;
+};
+/** The report at a glance: counts, timing and insights over every test (not only those a query matched). */
+export type CatalogSummary = {
+    readonly tests: number;
+    /** Tests by their current status. */
+    readonly counts: StatusCounts;
+    /** Current results (a test checked on two environments has two). */
+    readonly outcomes: number;
+    readonly flaky: number;
+    /** Over each test's average duration. */
+    readonly durations: DurationStats | null;
+    /** Tests with at least one measured duration. */
+    readonly measured: number;
+    readonly withAttachments: number;
+    readonly environments: readonly CatalogGroup[];
+    readonly producers: readonly CatalogGroup[];
+    readonly domains: readonly CatalogGroup[];
+    readonly slowest: readonly TestRef[];
+    readonly flakiest: readonly TestRef[];
+    readonly failing: readonly TestRef[];
+    /** Runs the history knows of. */
+    readonly runs: number;
+    readonly lastRecordedAtMs: number | null;
+    /** True when statistics come from recorded history, false when only the current results were available. */
+    readonly history: boolean;
+};
+/** A value to filter by, and how many tests have it. */
+export type FacetValue = {
+    readonly value: string;
+    readonly count: number;
+};
+/** Everything the tests can be filtered and grouped by, with counts over every test. */
+export type CatalogFacets = {
+    readonly statuses: readonly FacetValue[];
+    readonly environments: readonly FacetValue[];
+    readonly producers: readonly FacetValue[];
+    readonly domains: readonly FacetValue[];
+    readonly specs: readonly (CatalogSpec & {
+        readonly tests: number;
+        readonly counts: StatusCounts;
+    })[];
+    readonly tags: readonly FacetValue[];
+    readonly branches: readonly FacetValue[];
+    readonly labels: Readonly<Record<string, readonly FacetValue[]>>;
+};
+/** How tests are ordered. */
+export declare const TEST_SORTS: readonly ["spec", "title", "recent", "slowest", "failures", "flaky", "runs"];
+/** One of `TEST_SORTS`. `spec` (the default) orders by domain, spec and title. */
+export type TestSort = (typeof TEST_SORTS)[number];
+/** Which tests to list. Every field is optional; several values of one field match any of them, and fields combine. */
+export type TestQuery = {
+    /** Words that must all appear in the title, id, spec, domain, tags, labels, environments or producers. Case does not matter. */
+    readonly q?: string;
+    /** Current status, or `flaky`. */
+    readonly status?: readonly string[];
+    readonly environment?: readonly string[];
+    readonly producer?: readonly string[];
+    readonly domain?: readonly string[];
+    /** Spec paths (`specs/...`). */
+    readonly spec?: readonly string[];
+    readonly tag?: readonly string[];
+    /** Only tests recorded in this run (a key from `runList`). */
+    readonly run?: string;
+    readonly sort?: TestSort;
+    readonly offset?: number;
+    /** At most this many tests (default 500, at most 20000). */
+    readonly limit?: number;
+};
+/** `GET /api/reports/tests`: the summary, the facets and the tests that match a query. */
+export type TestCatalog = {
+    readonly schemaVersion: typeof TEST_CATALOG_VERSION;
+    /** Changes whenever the report or its history does. */
+    readonly version: string;
+    readonly generatedAtMs: number;
+    readonly summary: CatalogSummary;
+    readonly facets: CatalogFacets;
+    /** Tests that matched, before `offset` and `limit`. */
+    readonly total: number;
+    readonly offset: number;
+    readonly limit: number;
+    readonly tests: readonly TestSummary[];
+};
+/** `GET /api/reports/tests/history`: one test and every result recorded for it, newest first. */
+export type TestHistoryDocument = {
+    readonly schemaVersion: typeof TEST_CATALOG_VERSION;
+    readonly version: string;
+    readonly test: TestSummary;
+    readonly history: readonly TestResultEntry[];
+};
+/** A CI run (or any group of layers sharing a run URL, else a branch and commit): when, where, and how its tests went. */
+export type RunSummary = {
+    /** Stable key: the run URL, or `branch@sha`. */
+    readonly key: string;
+    readonly runUrl: string | null;
+    readonly branch: string | null;
+    readonly sha: string | null;
+    readonly startedAtMs: number;
+    readonly endedAtMs: number;
+    /** From its first to its last recorded result, ms. */
+    readonly spanMs: number;
+    readonly producers: readonly string[];
+    readonly environments: readonly string[];
+    readonly layers: number;
+    /** Distinct tests it recorded. */
+    readonly tests: number;
+    /** Its results by status (its last word on each test, environment and producer). */
+    readonly counts: StatusCounts;
+    /** Sum of its results' durations, ms, when any was measured. */
+    readonly totalDurationMs: number | null;
+    /** Up to twenty tests that failed in it. */
+    readonly failing: readonly {
+        readonly key: string;
+        readonly title: string;
+        readonly environment: string | null;
+        readonly detail: string;
+    }[];
+};
+/** `GET /api/reports/runs`: runs, newest first. */
+export type RunList = {
+    readonly schemaVersion: typeof TEST_CATALOG_VERSION;
+    readonly version: string;
+    readonly total: number;
+    readonly runs: readonly RunSummary[];
+};
+/** One test's last result within one run. */
+export type RunResult = {
+    readonly key: string;
+    readonly id: string;
+    readonly title: string;
+    readonly specPath: string | null;
+    readonly environment: string | null;
+    readonly producer: string;
+    readonly status: EvidenceStatus;
+    readonly durationMs: number | null;
+    readonly at: number;
+    readonly layer: string;
+    readonly detail: string;
+};
+/** `GET /api/reports/runs/detail`: one run and every result it recorded. */
+export type RunDetail = {
+    readonly schemaVersion: typeof TEST_CATALOG_VERSION;
+    readonly version: string;
+    readonly run: RunSummary;
+    readonly results: readonly RunResult[];
+};
+/** The run a layer belongs to: its run URL, else `branch@sha`, else `local`. Layers with the same key are one run. */
+export declare function runKey(layer: Pick<LayerHeader, 'runUrl' | 'branch' | 'sha'>): string;
+/**
+ * Every recorded result of every test (the newest `HISTORY_PER_TEST` per test, leaving out `running` progress) and the
+ * runs they came from. A run's results are read from that history, so memory follows the number of tests and not the age
+ * of the project: a run older than a test's last hundred results no longer lists that test. Add each layer once with
+ * `add`; the order layers arrive in does not change what it holds. It never decides a status.
+ *
+ * @example
+ * ```ts
+ * const history = new TestHistory();
+ * for (const { layer, rows } of layers) history.add(layer, rows, specs, layer.backend);
+ * const catalog = testCatalog(catalogModel(layers, { history }));
+ * ```
+ */
+export declare class TestHistory {
+    #private;
+    /** Changes each time a layer is added. */
+    get rev(): number;
+    /** The specs the added layers recorded. */
+    get specs(): RecordedSpecs;
+    /** How many layers were added. */
+    get layers(): number;
+    /** Takes in one layer: its header, its rows, the specs it recorded and the backend it was read from. A layer id already added is ignored. */
+    add(layer: LayerHeader, rows: readonly LayerRow[], specs?: readonly SpecEntry[], backend?: string): void;
+    /** The recorded results of one test (by `TestSummary.key`), newest first. Empty when none. */
+    of(key: string): readonly TestResultEntry[];
+    /** How many runs it knows of. */
+    get runCount(): number;
+    /** Every run, newest first. */
+    runs(): RunSummary[];
+    /** One run and its results (failed first, then by spec and title), or null. */
+    run(key: string): {
+        readonly run: RunSummary;
+        readonly results: RunResult[];
+    } | null;
+    /** The keys of the tests one run recorded. */
+    testsIn(key: string): ReadonlySet<string>;
+}
+/** Durations of a list of numbers (ms), or null when empty. `lastMs` is the first value, so give them newest first. */
+export declare function durationStats(values: readonly number[]): DurationStats | null;
+/** The spec a path names: the recorded one, or one derived from the path (`specs/<domain>/<name>.<ext>`). */
+export declare function catalogSpec(path: string | null, known: (path: string) => SpecEntry | undefined): CatalogSpec | null;
+/** Where the catalog's statistics and specs come from. */
+export type CatalogSources = {
+    /** Recorded results; without it, statistics describe only the current results. */
+    readonly history?: TestHistory | null;
+    /** Specs by path, published or recorded. Defaults to those the history recorded. */
+    readonly specs?: readonly SpecEntry[];
+    /** The report's version, echoed in the documents. */
+    readonly version?: string;
+    readonly now?: number;
+};
+/** The tests of a report, built once and queried many times: what `testCatalog` and `testHistory` read from. */
+export type CatalogModel = {
+    readonly version: string;
+    readonly generatedAtMs: number;
+    readonly tests: readonly TestSummary[];
+    readonly byKey: ReadonlyMap<string, TestSummary>;
+    readonly summary: CatalogSummary;
+    readonly facets: CatalogFacets;
+    readonly haystacks: ReadonlyMap<string, string>;
+    readonly history: TestHistory | null;
+};
+/** The lowercase text a search matches a test against: title, id, spec, domain, tags, labels, environments and producers. */
+export declare function haystackOf(test: TestSummary): string;
+/**
+ * Builds the tests of a report from its winners (`layers`, as the effective report holds them: each layer with the rows
+ * that win) and, when given, a history. The model is what `testCatalog` and `testHistory` query; build it once per
+ * version and reuse it.
+ */
+export declare function catalogModel(layers: readonly LayerEvidence[], sources?: CatalogSources): CatalogModel;
+/** Whether one test matches a query (everything but sorting and paging). */
+export declare function matchesQuery(model: CatalogModel, test: TestSummary, query: TestQuery, inRun?: ReadonlySet<string> | null): boolean;
+/** The catalog document for a query: the model's summary and facets, and the matching tests, sorted and paged. */
+export declare function testCatalog(model: CatalogModel, query?: TestQuery): TestCatalog;
+/** One test and its recorded results, or null when the report has no test with that key. */
+export declare function testHistory(model: CatalogModel, key: string): TestHistoryDocument | null;
+/** Every run the history knows of, newest first, `limit` at most (default 100). */
+export declare function runList(history: TestHistory, version: string, limit?: number): RunList;
+/** One run with every result it recorded, or null when no run has that key. */
+export declare function runDetail(history: TestHistory, version: string, key: string): RunDetail | null;
+/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`. */
+export declare function parseTestQuery(params: URLSearchParams): TestQuery;
+/** A duration for a person: `850 ms`, `12.3 s`, `4m 05s`. */
+export declare function formatMs(ms: number | null | undefined): string;
+/** The catalog as Markdown: the summary, insights, groups and the listed tests as a table. */
+export declare function catalogMarkdown(catalog: TestCatalog): string;
+/** One test's history as Markdown. */
+export declare function historyMarkdown(document: TestHistoryDocument): string;
+/** The run list as Markdown. */
+export declare function runsMarkdown(list: RunList): string;
+/** One run as Markdown. */
+export declare function runMarkdown(detail: RunDetail): string;