@crvouga/atlas 1.1.0 → 1.3.0

package/dist/reports/src/test-catalog.js

@@ -10,7 +10,9 @@
  *   always the report's winner (`combineEvidence`), and history only describes how it got there.
  * - `testCatalog` turns the report's winners (plus a history, when there is one) into a `TestCatalog`: a summary with
  *   insights (slowest, flaky, failing), facets to filter by, and the tests that match a `TestQuery`.
- * - `testHistory`, `runList` and `runDetail` answer for one test, every run, and one run.
+ * - `testHistory`, `runList` and `runDetail` answer for one test, every run, and one run. `runList` and `artifactList`
+ *   narrow by tag, label, media type, environment, branch, outcome and time (see `artifact-query.ts`); `artifactList` finds
+ *   the recorded results that carry evidence across the whole history, not only the latest winners.
  * - `catalogMarkdown`, `historyMarkdown`, `runsMarkdown` and `runMarkdown` say the same in Markdown, for an agent or a
  *   terminal.
  *
@@ -21,7 +23,9 @@
  * @see docs/server.md#reading-the-report-as-an-agent
  */
 import { displayTag, normalizeTag, normalizeTags } from './artifact-tags.js';
+import { facetsOf, forRuns, labelList, matchesFilter, outcomeOfCounts, splitList, usedFilters } from './artifact-query.js';
 import { videoCounts } from './evidence-detail.js';
+import { matchesScope, scopeTokens, UNKNOWN_SCOPE } from './evidence-scope.js';
 import { checkSlots } from './evidence-view.js';
 import { combineEvidence, emptyCounts, slotKey, specPath as normalSpecPath } from './layer-merge.js';
 import { RecordedSpecs } from './recorded-specs.js';
@@ -101,6 +105,7 @@ export class TestHistory {
     #seen = new Set();
     #rev = 0;
     #cells = null;
+    #evidence = null;
     /** Changes each time a layer is added. */
     get rev() {
         return this.#rev;
@@ -127,7 +132,7 @@ export class TestHistory {
         const key = runKey(layer);
         let run = this.#runs.get(key);
         if (!run) {
-            run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined, videos: [] };
+            run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined, videos: [], evidence: { tags: new Set(), artifactTags: new Set(), roles: new Set(), labels: new Set() } };
             this.#runs.set(key, run);
         }
         // A run is told by its first layer (by time, then id), whatever order its layers arrive in.
@@ -156,6 +161,8 @@ export class TestHistory {
         run.layers += 1;
         if (layer.evidence?.videos)
             run.videos.push(...layer.evidence.videos);
+        if (layer.evidence)
+            takeEvidence(run.evidence, layer.evidence);
         const origin = originOf(layer, backend);
         for (const row of rows) {
             // "Still running" is progress, not history: a test running now shows it as its current outcome.
@@ -248,13 +255,75 @@ export class TestHistory {
         results.sort((a, b) => FAILURE_RANK[b.status] - FAILURE_RANK[a.status] || compareText(a.specPath ?? '', b.specPath ?? '') || compareText(a.title, b.title) || compareText(a.environment ?? '', b.environment ?? '') || compareText(a.producer, b.producer) || compareText(a.key, b.key));
         return { run: runSummary(run, cells), results };
     }
+    /**
+     * Every recorded result that carries an attachment, newest first, with the values a query looks at. Built once per
+     * layer added. The domain is left to the caller (`artifactList`), which may know more specs than the layers recorded.
+     */
+    evidenceResults() {
+        if (this.#evidence?.rev === this.#rev)
+            return this.#evidence.items;
+        const items = [];
+        for (const [slot, list] of this.#bySlot) {
+            const i = this.#idents.get(slot);
+            for (const held of list) {
+                const e = held.e;
+                if (!e || e.attachments < 1)
+                    continue;
+                const playable = (videoCounts(e.videos)?.recordings ?? 0) > 0;
+                const types = typesOf(e.roles, playable);
+                const artifactTags = [...(e.artifactTags ?? [])];
+                const o = held.o;
+                const result = {
+                    key: i.slot, id: i.id, title: i.title, specPath: i.specPath, domain: null, at: o.at, status: held.s, environment: o.environment, producer: o.producer, kind: held.k,
+                    parameters: o.parameters, branch: o.branch, sha: o.sha, run: o.run, runUrl: o.runUrl, layer: o.layer, backend: o.backend, attachments: e.attachments,
+                    artifactTags, tags: [...e.tags], types, labels: e.labels, ...(e.videos ? { videos: e.videos } : {}),
+                };
+                const media = [
+                    ...types.filter(type => type !== 'video').map(type => ({ type, tags: artifactTags })),
+                    ...(e.videos ?? []).filter(video => video.available).map(video => ({ type: 'video', tags: video.tags ?? [] })),
+                ];
+                const labels = labelList(e.labels);
+                items.push({
+                    result,
+                    fields: {
+                        text: [i.title, i.id, i.specPath ?? '', o.branch ?? '', o.producer, o.environment ?? '', ...artifactTags, ...e.tags, ...labels].join('\n').toLowerCase(),
+                        artifactTags, tags: [...new Set([...e.tags, ...artifactTags])], types, media, labels,
+                        environments: o.environment ? [o.environment] : [], producers: [o.producer], branches: o.branch ? [o.branch] : [], shas: o.sha ? [o.sha.toLowerCase()] : [],
+                        outcomes: [held.s], at: o.at,
+                    },
+                });
+            }
+        }
+        items.sort((a, b) => b.result.at - a.result.at || compareText(a.result.layer, b.result.layer) || compareText(a.result.key, b.result.key) || compareText(a.result.kind, b.result.kind) || compareText(a.result.producer, b.result.producer));
+        this.#evidence = { rev: this.#rev, items };
+        return items;
+    }
     /** The keys of the tests one run recorded. */
     testsIn(key) {
         return new Set((this.#runCells().get(key) ?? []).map(cell => cell.i.slot));
     }
 }
+function takeEvidence(into, evidence) {
+    for (const tag of evidence.tags)
+        into.tags.add(tag);
+    for (const tag of evidence.artifactTags ?? [])
+        into.artifactTags.add(tag);
+    for (const role of evidence.roles)
+        into.roles.add(role);
+    for (const label of labelList(evidence.labels))
+        into.labels.add(label);
+}
+/** The media types of an evidence summary: its roles, with `video` only when a recording can be played. */
+function typesOf(roles, playable) {
+    const types = new Set(roles);
+    types.delete('video');
+    if (playable)
+        types.add('video');
+    return [...types].sort();
+}
 function runSummary(run, cells) {
     const counts = emptyCounts();
+    const carried = { tags: new Set(run.evidence.tags), artifactTags: new Set(run.evidence.artifactTags), roles: new Set(run.evidence.roles), labels: new Set(run.evidence.labels) };
     const tests = new Set();
     let total = 0, measured = false;
     const failed = [];
@@ -262,6 +331,8 @@ function runSummary(run, cells) {
     for (const cell of cells) {
         if (cell.r.e?.videos)
             videos.push(...cell.r.e.videos);
+        if (cell.r.e)
+            takeEvidence(carried, cell.r.e);
         counts[cell.r.s] += 1;
         tests.add(cell.i.slot);
         if (cell.r.d !== null) {
@@ -281,6 +352,10 @@ function runSummary(run, cells) {
         producers: [...run.producers].sort(), environments: [...run.environments].sort(),
         layers: run.layers, tests: tests.size, counts, totalDurationMs: measured ? total : null, failing,
         ...(video ? { videos: video } : {}),
+        ...(carried.artifactTags.size ? { artifactTags: [...carried.artifactTags].sort() } : {}),
+        ...(carried.roles.size ? { types: typesOf(carried.roles, (video?.recordings ?? 0) > 0) } : {}),
+        ...(carried.labels.size ? { labels: [...carried.labels].sort() } : {}),
+        ...(carried.tags.size ? { tags: [...carried.tags].sort() } : {}),
     };
 }
 function compareText(a, b) {
@@ -374,10 +449,16 @@ function summaryOf(slot, history, known) {
     const labels = {};
     let attachments = 0, steps = 0, attempts = 0;
     const roles = new Set();
+    const profiles = new Map();
     for (const [at, outcome] of outcomes.entries()) {
         const evidence = outcome.evidence;
         if (!evidence)
             continue;
+        for (const video of evidence.videos ?? [])
+            if (video.available) {
+                const tokens = scopeTokens(video.scope);
+                profiles.set(tokens.join('\n'), tokens);
+            }
         if (evidence.attachments > 0)
             for (const tag of normalizeTags([...evidence.tags, ...(slot.outcomes[at]?.layer.evidence?.tags ?? [])]))
                 inheritedTags.add(tag);
@@ -407,6 +488,7 @@ function summaryOf(slot, history, known) {
         tags: tagList,
         ...(artifactTags.size ? { artifactTags: [...artifactTags].sort() } : {}),
         ...([...inheritedTags].some(tag => !artifactTags.has(tag)) ? { inheritedArtifactTags: [...inheritedTags].filter(tag => !artifactTags.has(tag)).sort() } : {}),
+        ...(profiles.size ? { scopeProfiles: [...profiles.values()] } : {}),
         labels,
         evidence: { attachments, steps, attempts, roles: [...roles].sort() },
         lastRecordedAtMs: slot.latestAtMs,
@@ -416,7 +498,7 @@ function summaryOf(slot, history, known) {
 /** The lowercase text a search matches a test against: title, id, spec, domain, tags, labels, environments and producers. */
 export function haystackOf(test) {
     return [
-        test.title, test.id, test.specPath ?? '', test.spec?.title ?? '', test.spec?.domain ?? '', ...test.tags, ...(test.artifactTags ?? []).map(displayTag),
+        test.title, test.id, test.specPath ?? '', test.spec?.title ?? '', test.spec?.domain ?? '', ...test.tags, ...(test.artifactTags ?? []).map(displayTag), ...(test.scopeProfiles ?? []).flat().filter(token => token !== UNKNOWN_SCOPE),
         ...Object.entries(test.labels).flatMap(([key, value]) => [key, value]), ...test.environments, ...test.producers,
     ].join('\n').toLowerCase();
 }
@@ -523,6 +605,7 @@ export function catalogModel(layers, sources = {}) {
         specs: [...specGroups].map(([, list]) => { const g = group(list[0].spec.path, list); return { ...list[0].spec, tests: g.tests, counts: g.counts }; }).sort((a, b) => compareText(a.domain, b.domain) || compareText(a.title, b.title)),
         tags: facet(tests, test => test.tags),
         artifactTags: facet(tests, test => test.artifactTags ?? []),
+        scopes: facet(tests, test => (test.scopeProfiles ?? []).flat()),
         branches: facet(tests, test => test.stats.branches),
         labels: Object.fromEntries([...labelKeys].sort().map(key => [key, facet(tests, test => (test.labels[key] === undefined ? [] : [test.labels[key]]))])),
     };
@@ -559,6 +642,8 @@ export function matchesQuery(model, test, query, inRun) {
         return false;
     if (query.artifactTag?.length && !query.artifactTag.some(tag => { const wanted = normalizeTag(tag); return wanted !== null && ((test.artifactTags?.includes(wanted) ?? false) || (query.inherited === true && (test.inheritedArtifactTags?.includes(wanted) ?? false))); }))
         return false;
+    if (query.scope?.length && !(test.scopeProfiles ?? []).some(tokens => matchesScope(tokens, query.scope)))
+        return false;
     if (inRun && !inRun.has(test.key))
         return false;
     return true;
@@ -592,24 +677,75 @@ export function hasVideo(run) {
 export function runList(history, version, limit = 100, query = {}) {
     const all = history.runs();
     const withVideo = all.filter(hasVideo).length;
-    const runs = query.video ? all.filter(hasVideo) : all;
-    return { schemaVersion: TEST_CATALOG_VERSION, version, total: runs.length, runs: runs.slice(0, Math.max(0, limit)), withVideo };
+    const filter = forRuns(query);
+    const fields = all.map(fieldsOfRun);
+    const runs = all.filter((_, at) => matchesFilter(fields[at], filter));
+    return {
+        schemaVersion: TEST_CATALOG_VERSION, version, total: runs.length, runs: runs.slice(0, Math.max(0, limit)), withVideo,
+        filters: usedFilters(filter), facets: facetsOf(fields, filter),
+    };
+}
+/** What a query looks at in a run: see {@link ArtifactFilter}. A run's outcome is `failed` when any result failed. */
+export function fieldsOfRun(run) {
+    const artifactTags = run.artifactTags ?? [];
+    // A run summary of an older server has no `types`, but it says whether it has a playable video.
+    const types = run.types ?? (hasVideo(run) ? ['video'] : []);
+    const labels = run.labels ?? [];
+    return {
+        text: [run.title ?? '', run.key, run.branch ?? '', run.sha ?? '', ...run.producers, ...run.environments, ...artifactTags, ...(run.tags ?? []), ...labels].join('\n').toLowerCase(),
+        artifactTags, tags: [...new Set([...(run.tags ?? []), ...artifactTags])], types, media: types.map(type => ({ type, tags: artifactTags })), labels,
+        environments: run.environments, producers: run.producers, branches: run.branch ? [run.branch] : [], shas: run.sha ? [run.sha.toLowerCase()] : [],
+        outcomes: [outcomeOfCounts(run.counts)], at: run.endedAtMs,
+    };
+}
+/**
+ * The recorded results that carry evidence and match a query, newest first, across the whole history (`limit` default
+ * 100). `total` and the facet counts apply to every match before `offset` and `limit`. `domainOf` tells the domain of a
+ * spec path when the caller knows more than the layers recorded (a server with a spec set); otherwise it is the
+ * recorded spec's domain, else the folder the path implies.
+ *
+ * @example
+ * ```ts
+ * const list = artifactList(history, 'v1', { artifactTag: ['demo'], type: ['video'], environment: ['ios'] });
+ * list.results.map(r => [r.title, r.run, r.sha, r.status]);
+ * ```
+ */
+export function artifactList(history, version, query = {}, page = {}) {
+    const domains = new Map();
+    const domainOf = (path) => {
+        if (path === null)
+            return null;
+        if (!domains.has(path))
+            domains.set(path, (page.domainOf?.(path) ?? catalogSpec(path, known => history.specs.get(known))?.domain) || null);
+        return domains.get(path);
+    };
+    const items = history.evidenceResults().map(({ result, fields }) => {
+        const domain = domainOf(result.specPath);
+        return { result: { ...result, domain }, fields: { ...fields, domains: domain ? [domain] : [], specs: result.specPath ? [normalSpecPath(result.specPath)] : [] } };
+    });
+    const matched = items.filter(item => matchesFilter(item.fields, query));
+    const limit = Math.max(0, page.limit ?? 100);
+    const offset = Math.max(0, page.offset ?? 0);
+    return {
+        schemaVersion: TEST_CATALOG_VERSION, version, total: matched.length, offset, limit,
+        results: matched.slice(offset, offset + limit).map(item => item.result), filters: usedFilters(query), facets: facetsOf(items.map(item => item.fields), query),
+    };
 }
 /** One run with every result it recorded, or null when no run has that key. */
 export function runDetail(history, version, key) {
     const found = history.run(key);
     return found ? { schemaVersion: TEST_CATALOG_VERSION, version, ...found } : null;
 }
-/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, and `inherited=1`. */
+/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, `scope` (every token must hold for one recording), and `inherited=1`. */
 export function parseTestQuery(params) {
-    const many = (name) => params.getAll(name).flatMap(value => value.split(',')).map(value => value.trim()).filter(Boolean);
+    const many = (name) => params.getAll(name).flatMap(splitList);
     const number = (name) => { const value = params.get(name); return value !== null && /^\d+$/.test(value) ? Number(value) : undefined; };
     const sort = params.get('sort');
     const query = {};
     const q = params.get('q');
     if (q)
         query.q = q;
-    for (const name of ['status', 'environment', 'producer', 'domain', 'spec', 'tag', 'artifactTag']) {
+    for (const name of ['status', 'environment', 'producer', 'domain', 'spec', 'tag', 'artifactTag', 'scope']) {
         const values = many(name);
         if (values.length)
             query[name] = values;
@@ -690,6 +826,7 @@ export function historyMarkdown(document) {
         `- Duration: ${d ? `avg ${formatMs(d.avgMs)}, p50 ${formatMs(d.p50Ms)}, p95 ${formatMs(d.p95Ms)}, min ${formatMs(d.minMs)}, max ${formatMs(d.maxMs)}, last ${formatMs(d.lastMs)}` : 'not measured'}`,
         ...(test.tags.length ? [`- Tags: ${test.tags.join(', ')}`] : []),
         ...(test.artifactTags?.length ? [`- Artifact tags: ${test.artifactTags.map(displayTag).join(', ')}`] : []),
+        ...(test.scopeProfiles?.length ? [`- Recording scope: ${test.scopeProfiles.map(tokens => tokens.join(' ')).join('; ')}`] : []),
         ...(Object.keys(test.labels).length ? [`- Labels: ${Object.entries(test.labels).map(([k, v]) => `${k}=${v}`).join(', ')}`] : []),
         `- Evidence: ${test.evidence.attachments} attachments${test.evidence.roles.length ? ` (${test.evidence.roles.join(', ')})` : ''}, ${test.evidence.steps} steps`,
         '',
@@ -726,6 +863,16 @@ export function runsMarkdown(list) {
         ...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${videosText(r.videos)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
     ].join('\n')}\n`;
 }
+/** The artifact results as Markdown, for an agent or a terminal. */
+export function artifactsMarkdown(list) {
+    return `${[
+        `# Artifacts (${list.results.length} of ${list.total})`,
+        '',
+        '| Recorded | Test | Status | Environment | Branch | Commit | Types | Tags | Videos | Run |',
+        '|---|---|---|---|---|---|---|---|---|---|',
+        ...list.results.map(r => `| ${iso(r.at)} | ${cell(r.title)} | ${r.status} | ${r.environment ?? '—'} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'} | ${r.types.join(', ') || '—'} | ${r.artifactTags.map(displayTag).join(' ') || '—'} | ${videosText(videoCounts(r.videos))} | ${cell(r.runUrl ?? r.run)} |`),
+    ].join('\n')}\n`;
+}
 /** One run as Markdown. */
 export function runMarkdown(detail) {
     const { run } = detail;