@crvouga/atlas 0.14.0 → 0.16.0

package/dist/reports/src/test-catalog.d.ts

@@ -1,27 +1,5 @@
-/**
- * @module Test catalog, history and runs
- *
- * What a person or an agent asks of a report once it holds more than a handful of checks: every test with where it ran,
- * how it went over time and how long it takes; the specs, domains, environments and producers they fall under; the CI
- * runs that produced them and which tests each one ran.
- *
- * - `TestHistory` keeps every recorded result of every test (bounded per test) and every run's results, folded in one
- *   layer at a time, in any order, with the same outcome. It never decides a status: the current status of a test is
- *   always the report's winner (`combineEvidence`), and history only describes how it got there.
- * - `testCatalog` turns the report's winners (plus a history, when there is one) into a `TestCatalog`: a summary with
- *   insights (slowest, flaky, failing), facets to filter by, and the tests that match a `TestQuery`.
- * - `testHistory`, `runList` and `runDetail` answer for one test, every run, and one run.
- * - `catalogMarkdown`, `historyMarkdown`, `runsMarkdown` and `runMarkdown` say the same in Markdown, for an agent or a
- *   terminal.
- *
- * A test is a check identity (its spec path and id) regardless of producer, environment and parameters; each of those
- * combinations is one of its results. A result that is not `passed` is never counted as a pass, and a test with no
- * completed result has no pass rate.
- *
- * @see docs/server.md#reading-the-report-as-an-agent
- */
 import type { EvidenceStatus, SpecEntry } from './explorer-types.js';
-import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts } from './evidence-types.js';
+import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts, VideoCounts, VideoRef } from './evidence-types.js';
 import { RecordedSpecs } from './recorded-specs.js';
 /** The version of the documents this module writes (`TestCatalog`, `TestHistoryDocument`, `RunList`, `RunDetail`). Only optional fields are added within it. */
 export declare const TEST_CATALOG_VERSION = 1;
@@ -116,6 +94,10 @@ export type TestSummary = {
     readonly outcomes: readonly TestOutcome[];
     readonly stats: TestStats;
     readonly tags: readonly string[];
+    /** The normalized tags of the attachments of its current results (`demo`, shown as `#demo`). Never inherited from a check or run; the `tags` above are. Absent in documents of older servers. */
+    readonly artifactTags?: readonly string[];
+    /** The normalized tags its attachments inherit from their check and run (those of the results that have attachments), apart from `artifactTags`. What an inherited artifact query also matches. */
+    readonly inheritedArtifactTags?: readonly string[];
     readonly labels: Readonly<Record<string, string>>;
     /** What its current results carry: attachments, steps, retries, and the kinds of attachment. */
     readonly evidence: {
@@ -187,6 +169,8 @@ export type CatalogFacets = {
         readonly counts: StatusCounts;
     })[];
     readonly tags: readonly FacetValue[];
+    /** The tags of attachments, normalized. Absent in documents of older servers. */
+    readonly artifactTags?: readonly FacetValue[];
     readonly branches: readonly FacetValue[];
     readonly labels: Readonly<Record<string, readonly FacetValue[]>>;
 };
@@ -206,6 +190,10 @@ export type TestQuery = {
     /** Spec paths (`specs/...`). */
     readonly spec?: readonly string[];
     readonly tag?: readonly string[];
+    /** Only tests with an attachment carrying this tag (`demo` and `#demo` are the same), not counting tags inherited from a check or run: the exact artifact query. `tag` is the inherited one. */
+    readonly artifactTag?: readonly string[];
+    /** With `artifactTag`: also match the tags an attachment inherits from its check and run (a recording without a tag of its own under a run tagged `demo`). Default false: the exact query. */
+    readonly inherited?: boolean;
     /** Only tests recorded in this run (a key from `runList`). */
     readonly run?: string;
     readonly sort?: TestSort;
@@ -270,13 +258,26 @@ export type RunSummary = {
         readonly environment: string | null;
         readonly detail: string;
     }[];
+    /**
+     * Its videos, from the run's layers and from the last result of each test: references, distinct playable recordings
+     * and references that cannot be played. Absent when it has none, and in a report that carries no video information.
+     */
+    readonly videos?: VideoCounts;
+};
+/** What `runList` can be asked. */
+export type RunQuery = {
+    /** Only runs that have at least one playable video recording. `total` then counts only those. */
+    readonly video?: boolean;
 };
 /** `GET /api/reports/runs`: runs, newest first. */
 export type RunList = {
     readonly schemaVersion: typeof TEST_CATALOG_VERSION;
     readonly version: string;
+    /** Runs that matched, before `limit`. */
     readonly total: number;
     readonly runs: readonly RunSummary[];
+    /** How many of all runs (whatever was asked, before `limit`) have a playable video recording. */
+    readonly withVideo?: number;
 };
 /** One test's last result within one run. */
 export type RunResult = {
@@ -291,6 +292,8 @@ export type RunResult = {
     readonly at: number;
     readonly layer: string;
     readonly detail: string;
+    /** Every video its evidence references, with who holds it (check, retry attempt or step). Absent when there is none. */
+    readonly videos?: readonly VideoRef[];
 };
 /** `GET /api/reports/runs/detail`: one run and every result it recorded. */
 export type RunDetail = {
@@ -392,11 +395,17 @@ export declare function matchesQuery(model: CatalogModel, test: TestSummary, que
 export declare function testCatalog(model: CatalogModel, query?: TestQuery): TestCatalog;
 /** One test and its recorded results, or null when the report has no test with that key. */
 export declare function testHistory(model: CatalogModel, key: string): TestHistoryDocument | null;
-/** Every run the history knows of, newest first, `limit` at most (default 100). */
-export declare function runList(history: TestHistory, version: string, limit?: number): RunList;
+/** Whether a run has at least one playable video recording. */
+export declare function hasVideo(run: Pick<RunSummary, 'videos'>): boolean;
+/**
+ * Every run the history knows of, newest first, `limit` at most (default 100). With `query.video` only the runs that
+ * have a playable video recording: the filter and `total` apply to every run before `limit`, and nothing is read from
+ * any recording's bytes. `withVideo` always counts every run that has one.
+ */
+export declare function runList(history: TestHistory, version: string, limit?: number, query?: RunQuery): RunList;
 /** One run with every result it recorded, or null when no run has that key. */
 export declare function runDetail(history: TestHistory, version: string, key: string): RunDetail | null;
-/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`. */
+/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, and `inherited=1`. */
 export declare function parseTestQuery(params: URLSearchParams): TestQuery;
 /** A duration for a person: `850 ms`, `12.3 s`, `4m 05s`. */
 export declare function formatMs(ms: number | null | undefined): string;

package/dist/reports/src/test-catalog.js

@@ -1,3 +1,27 @@
+/**
+ * @module Test catalog, history and runs
+ *
+ * What a person or an agent asks of a report once it holds more than a handful of checks: every test with where it ran,
+ * how it went over time and how long it takes; the specs, domains, environments and producers they fall under; the CI
+ * runs that produced them and which tests each one ran.
+ *
+ * - `TestHistory` keeps every recorded result of every test (bounded per test) and every run's results, folded in one
+ *   layer at a time, in any order, with the same outcome. It never decides a status: the current status of a test is
+ *   always the report's winner (`combineEvidence`), and history only describes how it got there.
+ * - `testCatalog` turns the report's winners (plus a history, when there is one) into a `TestCatalog`: a summary with
+ *   insights (slowest, flaky, failing), facets to filter by, and the tests that match a `TestQuery`.
+ * - `testHistory`, `runList` and `runDetail` answer for one test, every run, and one run.
+ * - `catalogMarkdown`, `historyMarkdown`, `runsMarkdown` and `runMarkdown` say the same in Markdown, for an agent or a
+ *   terminal.
+ *
+ * A test is a check identity (its spec path and id) regardless of producer, environment and parameters; each of those
+ * combinations is one of its results. A result that is not `passed` is never counted as a pass, and a test with no
+ * completed result has no pass rate.
+ *
+ * @see docs/server.md#reading-the-report-as-an-agent
+ */
+import { displayTag, normalizeTag, normalizeTags } from './artifact-tags.js';
+import { videoCounts } from './evidence-detail.js';
 import { checkSlots } from './evidence-view.js';
 import { combineEvidence, emptyCounts, slotKey, specPath as normalSpecPath } from './layer-merge.js';
 import { RecordedSpecs } from './recorded-specs.js';
@@ -103,7 +127,7 @@ export class TestHistory {
         const key = runKey(layer);
         let run = this.#runs.get(key);
         if (!run) {
-            run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined };
+            run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined, videos: [] };
             this.#runs.set(key, run);
         }
         // A run is told by its first layer (by time, then id), whatever order its layers arrive in.
@@ -130,6 +154,8 @@ export class TestHistory {
         if (layer.environment)
             run.environments.add(layer.environment);
         run.layers += 1;
+        if (layer.evidence?.videos)
+            run.videos.push(...layer.evidence.videos);
         const origin = originOf(layer, backend);
         for (const row of rows) {
             // "Still running" is progress, not history: a test running now shows it as its current outcome.
@@ -217,6 +243,7 @@ export class TestHistory {
         const results = cells.map(({ i, r }) => ({
             key: i.slot, id: i.id, title: i.title, specPath: i.specPath, environment: r.o.environment, producer: r.o.producer,
             status: r.s, durationMs: r.d, at: r.o.at, layer: r.o.layer, detail: shortDetail(r.t),
+            ...(r.e?.videos ? { videos: r.e.videos } : {}),
         }));
         results.sort((a, b) => FAILURE_RANK[b.status] - FAILURE_RANK[a.status] || compareText(a.specPath ?? '', b.specPath ?? '') || compareText(a.title, b.title) || compareText(a.environment ?? '', b.environment ?? '') || compareText(a.producer, b.producer) || compareText(a.key, b.key));
         return { run: runSummary(run, cells), results };
@@ -231,7 +258,10 @@ function runSummary(run, cells) {
     const tests = new Set();
     let total = 0, measured = false;
     const failed = [];
+    const videos = [...run.videos];
     for (const cell of cells) {
+        if (cell.r.e?.videos)
+            videos.push(...cell.r.e.videos);
         counts[cell.r.s] += 1;
         tests.add(cell.i.slot);
         if (cell.r.d !== null) {
@@ -242,6 +272,7 @@ function runSummary(run, cells) {
             failed.push(cell);
     }
     failed.sort((a, b) => compareText(a.i.slot, b.i.slot) || compareText(a.r.o.place, b.r.o.place) || compareText(a.r.k, b.r.k));
+    const video = videoCounts(videos);
     const failing = failed.slice(0, 20).map(cell => ({ key: cell.i.slot, title: cell.i.title, environment: cell.r.o.environment, detail: (cell.r.t.split('\n')[0] ?? '').slice(0, 500) }));
     return {
         key: run.key, ...(run.runId ? { runId: run.runId } : {}), ...(run.title ? { title: run.title } : {}), ...(run.freshness ? { freshness: run.freshness } : {}),
@@ -249,6 +280,7 @@ function runSummary(run, cells) {
         startedAtMs: run.start, endedAtMs: run.end, spanMs: run.end - run.start,
         producers: [...run.producers].sort(), environments: [...run.environments].sort(),
         layers: run.layers, tests: tests.size, counts, totalDurationMs: measured ? total : null, failing,
+        ...(video ? { videos: video } : {}),
     };
 }
 function compareText(a, b) {
@@ -337,15 +369,22 @@ function outcomeOf(result) {
 function summaryOf(slot, history, known) {
     const outcomes = slot.outcomes.map(outcomeOf);
     const tags = new Set();
+    const artifactTags = new Set();
+    const inheritedTags = new Set();
     const labels = {};
     let attachments = 0, steps = 0, attempts = 0;
     const roles = new Set();
-    for (const outcome of outcomes) {
+    for (const [at, outcome] of outcomes.entries()) {
         const evidence = outcome.evidence;
         if (!evidence)
             continue;
+        if (evidence.attachments > 0)
+            for (const tag of normalizeTags([...evidence.tags, ...(slot.outcomes[at]?.layer.evidence?.tags ?? [])]))
+                inheritedTags.add(tag);
         for (const tag of evidence.tags)
             tags.add(tag);
+        for (const tag of evidence.artifactTags ?? [])
+            artifactTags.add(tag);
         Object.assign(labels, evidence.labels);
         attachments += evidence.attachments;
         steps += evidence.steps;
@@ -366,6 +405,8 @@ function summaryOf(slot, history, known) {
         outcomes,
         stats: statsOf(history?.of(slot.key) ?? [], outcomes, tagList, labels),
         tags: tagList,
+        ...(artifactTags.size ? { artifactTags: [...artifactTags].sort() } : {}),
+        ...([...inheritedTags].some(tag => !artifactTags.has(tag)) ? { inheritedArtifactTags: [...inheritedTags].filter(tag => !artifactTags.has(tag)).sort() } : {}),
         labels,
         evidence: { attachments, steps, attempts, roles: [...roles].sort() },
         lastRecordedAtMs: slot.latestAtMs,
@@ -375,7 +416,7 @@ function summaryOf(slot, history, known) {
 /** The lowercase text a search matches a test against: title, id, spec, domain, tags, labels, environments and producers. */
 export function haystackOf(test) {
     return [
-        test.title, test.id, test.specPath ?? '', test.spec?.title ?? '', test.spec?.domain ?? '', ...test.tags,
+        test.title, test.id, test.specPath ?? '', test.spec?.title ?? '', test.spec?.domain ?? '', ...test.tags, ...(test.artifactTags ?? []).map(displayTag),
         ...Object.entries(test.labels).flatMap(([key, value]) => [key, value]), ...test.environments, ...test.producers,
     ].join('\n').toLowerCase();
 }
@@ -481,6 +522,7 @@ export function catalogModel(layers, sources = {}) {
         domains: facet(tests, test => [test.spec?.domain || '(no spec)']),
         specs: [...specGroups].map(([, list]) => { const g = group(list[0].spec.path, list); return { ...list[0].spec, tests: g.tests, counts: g.counts }; }).sort((a, b) => compareText(a.domain, b.domain) || compareText(a.title, b.title)),
         tags: facet(tests, test => test.tags),
+        artifactTags: facet(tests, test => test.artifactTags ?? []),
         branches: facet(tests, test => test.stats.branches),
         labels: Object.fromEntries([...labelKeys].sort().map(key => [key, facet(tests, test => (test.labels[key] === undefined ? [] : [test.labels[key]]))])),
     };
@@ -515,6 +557,8 @@ export function matchesQuery(model, test, query, inRun) {
         return false;
     if (query.tag?.length && !query.tag.some(tag => test.tags.includes(tag)))
         return false;
+    if (query.artifactTag?.length && !query.artifactTag.some(tag => { const wanted = normalizeTag(tag); return wanted !== null && ((test.artifactTags?.includes(wanted) ?? false) || (query.inherited === true && (test.inheritedArtifactTags?.includes(wanted) ?? false))); }))
+        return false;
     if (inRun && !inRun.has(test.key))
         return false;
     return true;
@@ -536,17 +580,27 @@ export function testHistory(model, key) {
         return null;
     return { schemaVersion: TEST_CATALOG_VERSION, version: model.version, test, history: model.history?.of(key) ?? test.outcomes };
 }
-/** Every run the history knows of, newest first, `limit` at most (default 100). */
-export function runList(history, version, limit = 100) {
-    const runs = history.runs();
-    return { schemaVersion: TEST_CATALOG_VERSION, version, total: runs.length, runs: runs.slice(0, Math.max(0, limit)) };
+/** Whether a run has at least one playable video recording. */
+export function hasVideo(run) {
+    return (run.videos?.recordings ?? 0) > 0;
+}
+/**
+ * Every run the history knows of, newest first, `limit` at most (default 100). With `query.video` only the runs that
+ * have a playable video recording: the filter and `total` apply to every run before `limit`, and nothing is read from
+ * any recording's bytes. `withVideo` always counts every run that has one.
+ */
+export function runList(history, version, limit = 100, query = {}) {
+    const all = history.runs();
+    const withVideo = all.filter(hasVideo).length;
+    const runs = query.video ? all.filter(hasVideo) : all;
+    return { schemaVersion: TEST_CATALOG_VERSION, version, total: runs.length, runs: runs.slice(0, Math.max(0, limit)), withVideo };
 }
 /** One run with every result it recorded, or null when no run has that key. */
 export function runDetail(history, version, key) {
     const found = history.run(key);
     return found ? { schemaVersion: TEST_CATALOG_VERSION, version, ...found } : null;
 }
-/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`. */
+/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, and `inherited=1`. */
 export function parseTestQuery(params) {
     const many = (name) => params.getAll(name).flatMap(value => value.split(',')).map(value => value.trim()).filter(Boolean);
     const number = (name) => { const value = params.get(name); return value !== null && /^\d+$/.test(value) ? Number(value) : undefined; };
@@ -555,11 +609,13 @@ export function parseTestQuery(params) {
     const q = params.get('q');
     if (q)
         query.q = q;
-    for (const name of ['status', 'environment', 'producer', 'domain', 'spec', 'tag']) {
+    for (const name of ['status', 'environment', 'producer', 'domain', 'spec', 'tag', 'artifactTag']) {
         const values = many(name);
         if (values.length)
             query[name] = values;
     }
+    if (['1', 'true'].includes(params.get('inherited') ?? ''))
+        query.inherited = true;
     const run = params.get('run');
     if (run)
         query.run = run;
@@ -633,6 +689,7 @@ export function historyMarkdown(document) {
         `- Results: ${test.stats.results} recorded, ${test.stats.passed} passed, ${test.stats.failed} failed${test.stats.passRate === null ? '' : `, pass rate ${Math.round(test.stats.passRate * 100)}%`}, in ${test.stats.runs} runs`,
         `- Duration: ${d ? `avg ${formatMs(d.avgMs)}, p50 ${formatMs(d.p50Ms)}, p95 ${formatMs(d.p95Ms)}, min ${formatMs(d.minMs)}, max ${formatMs(d.maxMs)}, last ${formatMs(d.lastMs)}` : 'not measured'}`,
         ...(test.tags.length ? [`- Tags: ${test.tags.join(', ')}`] : []),
+        ...(test.artifactTags?.length ? [`- Artifact tags: ${test.artifactTags.map(displayTag).join(', ')}`] : []),
         ...(Object.keys(test.labels).length ? [`- Labels: ${Object.entries(test.labels).map(([k, v]) => `${k}=${v}`).join(', ')}`] : []),
         `- Evidence: ${test.evidence.attachments} attachments${test.evidence.roles.length ? ` (${test.evidence.roles.join(', ')})` : ''}, ${test.evidence.steps} steps`,
         '',
@@ -649,6 +706,12 @@ export function historyMarkdown(document) {
         ...document.history.map(e => `| ${iso(e.at)} | ${e.environment ?? '—'} | ${e.status} | ${formatMs(e.durationMs)} | ${cell(e.branch ?? '—')} | ${e.sha?.slice(0, 10) ?? '—'} | ${cell(e.detail.split('\n')[0] ?? '')} |`),
     ].join('\n')}\n`;
 }
+/** Video counts in a few words: `2 (3 references)`, with how many cannot be played, or `—` when there are none. */
+function videosText(videos) {
+    if (!videos)
+        return '—';
+    return `${videos.recordings}${videos.references !== videos.recordings ? ` (${videos.references} references)` : ''}${videos.unavailable ? `, ${videos.unavailable} unavailable` : ''}`;
+}
 /** What to say beside a commit that was not recorded from a clean checkout: nothing, ` (modified)` or ` (origin unknown)`. */
 function freshnessNote(freshness) {
     return freshness === 'modified' ? ' (modified: local changes)' : freshness === 'unknown' ? ' (origin unknown)' : '';
@@ -658,9 +721,9 @@ export function runsMarkdown(list) {
     return `${[
         `# Runs (${list.runs.length} of ${list.total})`,
         '',
-        '| Ended | Branch | Commit | Environments | Tests | Passed | Failed | Not run | Span | Run |',
-        '|---|---|---|---|---:|---:|---:|---:|---:|---|',
-        ...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
+        '| Ended | Branch | Commit | Environments | Tests | Passed | Failed | Not run | Span | Videos | Run |',
+        '|---|---|---|---|---:|---:|---:|---:|---:|---|---|',
+        ...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${videosText(r.videos)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
     ].join('\n')}\n`;
 }
 /** One run as Markdown. */
@@ -674,9 +737,10 @@ export function runMarkdown(detail) {
         `- ${iso(run.startedAtMs)} to ${iso(run.endedAtMs)} (${formatMs(run.spanMs)}); ${run.layers} layers from ${run.producers.join(', ')}`,
         `- Environments: ${run.environments.join(', ') || '—'}`,
         `- ${run.tests} tests: ${counted(run.counts)}${run.totalDurationMs === null ? '' : `; ${formatMs(run.totalDurationMs)} of test time`}`,
+        ...(run.videos ? [`- Videos: ${videosText(run.videos)}`] : []),
         '',
-        '| Status | Test | Spec | Environment | Duration | Detail |',
-        '|---|---|---|---|---:|---|',
-        ...detail.results.map(r => `| ${r.status} | ${cell(r.title)} | ${cell(r.specPath ?? '—')} | ${r.environment ?? '—'} | ${formatMs(r.durationMs)} | ${cell(r.detail.split('\n')[0] ?? '')} |`),
+        '| Status | Test | Spec | Environment | Duration | Videos | Detail |',
+        '|---|---|---|---|---:|---|---|',
+        ...detail.results.map(r => `| ${r.status} | ${cell(r.title)} | ${cell(r.specPath ?? '—')} | ${r.environment ?? '—'} | ${formatMs(r.durationMs)} | ${videosText(videoCounts(r.videos))} | ${cell(r.detail.split('\n')[0] ?? '')} |`),
     ].join('\n')}\n`;
 }