@crvouga/atlas 1.16.0 → 1.18.0

package/packages/visualizer/src/explorer/RunsView.tsx

@@ -93,15 +93,39 @@ function ownerText(video: VideoRef): string {
   return 'the test';
 }
 const UNAVAILABLE = 'Listed in the report, but its file was never stored, so it cannot be played.';
-/** The unavailable references of a video summary and why they cannot be watched. */
+const MISSING = 'The run says it stored this file, but its bundle no longer holds it, so it cannot be played.';
+/** The unavailable references of a video summary and why they cannot be watched: never stored, or stored and then gone. */
 function Unavailable({ videos }: { videos: VideoCounts }) {
   if (!videos.unavailable) return null;
+  const missing = Math.min(videos.missing ?? 0, videos.unavailable);
   return (
     <Text component="div" size="xs" c="dimmed" data-video-unavailable>
-      <IconAlertTriangle size={16} aria-hidden="true" /> {plural(videos.unavailable, 'video', 'videos')} unavailable. {UNAVAILABLE}
+      <IconAlertTriangle size={16} aria-hidden="true" /> {plural(videos.unavailable, 'video', 'videos')} unavailable.{' '}
+      {missing > 0 && <Text span inherit data-video-missing>{missing === videos.unavailable ? MISSING : `${plural(missing, 'video', 'videos')}: ${MISSING}`}{' '}</Text>}
+      {missing < videos.unavailable && UNAVAILABLE}
     </Text>
   );
 }
+/** Words for a run's recording completeness, apart from how its tests went. */
+const COMPLETENESS_WORDS: Record<string, string> = { complete: 'complete', incomplete: 'incomplete', 'none-expected': 'none expected', unknown: 'unknown' };
+/** A run's recording completeness in its row: how many expected recordings were published, said in words and independent of its outcomes. Nothing when its layers declared no expectations (unknown is not shown as complete). */
+function RunRecordings({ run }: { run: RunSummary }) {
+  const recordings = run.recordings;
+  if (!recordings) return null;
+  const word = COMPLETENESS_WORDS[recordings.state] ?? 'unknown';
+  const text = recordings.expected > 0 ? `Recordings ${recordings.published} of ${recordings.expected}: ${word}` : `Recordings: ${word}`;
+  return (
+    <Badge
+      size="sm"
+      variant="light"
+      color={recordings.state === 'incomplete' ? 'orange' : 'gray'}
+      data-recordings={recordings.state}
+      title={recordings.state === 'incomplete' ? 'Some expected recordings were not published. The test outcomes are unaffected.' : undefined}
+    >
+      {text}
+    </Badge>
+  );
+}
 /** A run's videos in its row: a link to the recordings, how many references they come from, and what cannot be played. */
 function RunVideos({ run }: { run: RunSummary }) {
   const videos = run.videos;
@@ -210,7 +234,7 @@ const RUN_SORTS = [
   { id: 'relevance', label: 'Best match' }
 ] as const;
 /** What a filter is called when a page says a server could not apply it. */
-const FILTER_WORDS: Partial<Record<FilterName, string>> = { artifactTag: 'tag', type: 'media type', label: 'label', domain: 'domain', spec: 'spec' };
+const FILTER_WORDS: Partial<Record<FilterName, string>> = { artifactTag: 'tag', type: 'media type', label: 'label', domain: 'domain', spec: 'spec', recordings: 'recording completeness' };
 /** Says which filters the list is not narrowed by, because the server that answered ignored them. */
 export function NotApplied({ names, noun }: { names: readonly FilterName[]; noun: string }) {
   if (!names.length) return null;
@@ -383,6 +407,7 @@ function RunTable() {
                 <Table.Td>{formatMs(run.spanMs)}</Table.Td>
                 <Table.Td>
                   <RunVideos run={run} />
+                  <RunRecordings run={run} />
                 </Table.Td>
                 <Table.Td data-tags>{run.artifactTags?.map((tag) => `#${tag}`).join(' ') || '—'}</Table.Td>
                 <Table.Td>

package/packages/visualizer/src/explorer/TestDetail.tsx

@@ -13,12 +13,15 @@ import { environmentLabel } from '@crvouga/atlas/reports/layer-axes';
 import { formatMs, runKey, type TestResultEntry, type TestSummary } from '@crvouga/atlas/reports/test-catalog';
 import { Link } from '@tanstack/react-router';
 import { lazy, Suspense, useState } from 'react';
+import { assessTest } from '../evidence/applicability';
+import { ApplicabilitySummary } from '../evidence/applicability-view';
 import { CheckDetail } from '../evidence/check-detail';
 import { ago, exactTime, plural, shortSha, statusWord, useNow } from '../evidence/format';
 import { useEvidence } from '../evidence/evidence-context';
 import { Origin, StatusMark } from '../evidence/marks';
 import { outcomeHeadline, selectedResult } from '../evidence/outcome';
 import { useTestHistory } from './catalog';
+import { releaseOf } from './applicability';
 import { patched, useExplorerNavigate, useExplorerSearch } from './navigate';
 import { Badges, runName } from './parts';
 import { Reliability } from './Reliability';
@@ -129,7 +132,10 @@ function Scope({ pinned, slot }: { pinned: PinnedResult; slot: CheckSlot | undef
 export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot | undefined }) {
   const now = useNow();
   const history = useTestHistory(test.key);
-  const { layer, source } = useExplorerSearch();
+  const search = useExplorerSearch();
+  const { layer, source } = search;
+  const release = releaseOf(search);
+  const applicability = assessTest(test, release);
   const go = useExplorerNavigate();
   const pinned = usePinnedResult(layer, test.key);
   const [recording, setRecording] = useState(false);
@@ -238,6 +244,13 @@ export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot
 
       <Scope pinned={pinned} slot={slot} />
 
+      <Box aria-label="Evidence applicability" component="section" mb="lg">
+        <Title order={3} mb="sm" c="dimmed">
+          Does this evidence apply?
+        </Title>
+        <ApplicabilitySummary assessment={applicability} release={release} />
+      </Box>
+
       <Box aria-label="Current results" component="section" mb="lg">
         <Title order={3} mb="sm" c="dimmed">
           Current result per environment

package/packages/visualizer/src/explorer/applicability.ts

@@ -0,0 +1,43 @@
+/**
+ * @module Explorer evidence applicability
+ *
+ * The explorer's part of evidence applicability: the release a reader names and the stale-or-unverified filter, held in
+ * the explorer's address (`release`, `releaseSha`, `releasePlatform`, `evidence=stale`, the same names a spec page uses),
+ * and the reading of every test of the catalog against them. The reading is the one a feature page makes
+ * (`evidence/applicability.ts`); it never changes a test's status.
+ */
+import { useMemo } from 'react';
+import type { TestSummary } from '@crvouga/atlas/reports/test-catalog';
+
+import { assessTest, type Assessment, type ReleaseIdentity } from '../evidence/applicability';
+import type { ApplicabilityState } from '../evidence/applicability-view';
+import { useExplorerNavigate, useExplorerSearch } from './navigate';
+import type { ExplorerSearch } from './search';
+
+/** The release an explorer address names; every part is optional. */
+export function releaseOf(search: ExplorerSearch): ReleaseIdentity {
+  return {
+    ...(search.release ? { build: search.release } : {}),
+    ...(search.releaseSha ? { sha: search.releaseSha } : {}),
+    ...(search.releasePlatform ? { platform: search.releasePlatform } : {})
+  };
+}
+
+/** Each test's evidence read against the promise and the release, by test key. */
+export function assessTests(tests: readonly TestSummary[], release: ReleaseIdentity): ReadonlyMap<string, Assessment> {
+  return new Map(tests.map((test) => [test.key, assessTest(test, release)] as const));
+}
+
+/** The release and the evidence filter of the explorer's address, with the ways to change them. */
+export function useExplorerApplicability(): ApplicabilityState {
+  const search = useExplorerSearch();
+  const go = useExplorerNavigate();
+  const { release: build, releaseSha: sha, releasePlatform: platform } = search;
+  const release = useMemo(() => releaseOf({ release: build, releaseSha: sha, releasePlatform: platform }), [build, sha, platform]);
+  return {
+    release,
+    onlyStale: search.evidence === 'stale',
+    setRelease: (next) => go({ release: next.build || undefined, releaseSha: next.sha || undefined, releasePlatform: next.platform || undefined }, { replace: true }),
+    setOnlyStale: (on) => go({ evidence: on ? 'stale' : undefined }, { replace: true })
+  };
+}

package/packages/visualizer/src/explorer/filters.tsx

@@ -2,7 +2,7 @@
  * @module Run and artifact filters
  *
  * The filter bar of the Runs and Artifacts tabs: words, artifact tag, media type, label, environment, branch, outcome,
- * feature domain and a time range, each offering the values the report has with counts in the scope of the list. What is
+ * feature domain and a time range (and, for runs, the completeness of their recordings, apart from their outcome), each offering the values the report has with counts in the scope of the list. What is
  * chosen shows as removable chips and is the address, so copying it or reloading restores the same view; named views keep
  * a query to open again. Choices combine with AND across the controls and OR within one.
  */
@@ -18,7 +18,7 @@ import { patched, useExplorerNavigate } from './navigate';
 import { filtered, joined, NO_FILTERS, toggled, values, type ExplorerSearch } from './search';
 import { filtersOf, isView, readViews, writeViews, type SavedView } from './views';
 
-type Param = 'artifactTag' | 'type' | 'label' | 'env' | 'branch' | 'outcome' | 'domain' | 'producer';
+type Param = 'artifactTag' | 'type' | 'label' | 'env' | 'branch' | 'outcome' | 'recordings' | 'domain' | 'producer';
 
 const NOUN: Record<Param, string> = {
   artifactTag: 'Tag',
@@ -27,13 +27,18 @@ const NOUN: Record<Param, string> = {
   env: 'Environment',
   branch: 'Branch',
   outcome: 'Outcome',
+  recordings: 'Recordings',
   domain: 'Domain',
   producer: 'Producer'
 };
 
+/** How the completeness of a run's recordings reads. */
+const COMPLETENESS: Record<string, string> = { complete: 'Complete', incomplete: 'Incomplete', 'none-expected': 'None expected', unknown: 'Unknown' };
+
 /** How a chosen value reads: tags with their hash, environments by name. */
 function shown(param: Param, value: string): string {
   if (param === 'artifactTag') return `#${value}`;
+  if (param === 'recordings') return COMPLETENESS[value] ?? value;
   if (param === 'env') return environmentLabel(value);
   if (param === 'domain' && value === '(no spec)') return 'No spec';
   return value;
@@ -178,7 +183,7 @@ export function FilterBar({
   const [draft, setDraft] = useState(search.q ?? '');
   // The address may change from elsewhere (a link, a saved view, back): follow it.
   useEffect(() => setDraft((current) => (current.trim() === (search.q ?? '').trim() ? current : (search.q ?? ''))), [search.q]);
-  const params: Param[] = ['artifactTag', 'type', 'label', 'env', 'branch', 'outcome', ...(tab === 'artifacts' ? (['domain'] as const) : []), 'producer'];
+  const params: Param[] = ['artifactTag', 'type', 'label', 'env', 'branch', 'outcome', ...(tab === 'runs' ? (['recordings'] as const) : []), ...(tab === 'artifacts' ? (['domain'] as const) : []), 'producer'];
   const chips = [
     ...params.flatMap((param) =>
       values(search[param]).map((value) => ({ label: `${NOUN[param]}: ${shown(param, value)}`, clear: { [param]: toggled(search[param], value) } as never }))
@@ -233,6 +238,9 @@ export function FilterBar({
           <Facet param="env" facets={facets.environments} search={search} />
           <Facet param="branch" facets={facets.branches} search={search} />
           <Facet param="outcome" facets={facets.outcomes} search={search} />
+          {tab === 'runs' && (facets.recordings || values(search.recordings).length > 0) && (
+            <Facet param="recordings" facets={facets.recordings ?? []} search={search} />
+          )}
           {tab === 'artifacts' && facets.domains && <Facet param="domain" facets={facets.domains} search={search} />}
           {facets.producers.length > 1 && <Facet param="producer" facets={facets.producers} search={search} />}
           <TextInput type="date" label="From" value={search.since ?? ''} onChange={(event) => go({ since: event.target.value || undefined })} />

package/packages/visualizer/src/explorer/search.ts

@@ -7,7 +7,7 @@
  * or `\\`, and the router encodes `#` and spaces. Anything unreadable is dropped, never an error.
  *
  * The Runs and Artifacts tabs share the tests' `q`, `env`, `producer`, `domain` and `spec` and add `artifactTag`,
- * `type`, `label`, `branch`, `outcome`, `since` and `until`, so a link to one tab's filters opens the same query on another.
+ * `type`, `label`, `branch`, `outcome`, `since` and `until` (and the Runs tab alone `recordings`, the completeness of its recordings), so a link to one tab's filters opens the same query on another.
  */
 import { joinList, splitList, type ArtifactFilter } from '@crvouga/atlas/reports/artifact-query';
 import type { ReliabilityScope } from '@crvouga/atlas/reports/reliability';
@@ -19,8 +19,8 @@ export const TEST_GROUPS = ['spec', 'domain', 'environment', 'status', 'producer
 /** One of {@link TEST_GROUPS}. */
 export type TestGroup = (typeof TEST_GROUPS)[number];
 
-/** The orders the explorer offers: `relevance` (best match first, only with words to match) and those of the catalog. */
-export const EXPLORER_SORTS = ['relevance', ...TEST_SORTS] as const;
+/** The orders the explorer offers: `relevance` (best match first, only with words to match), `evidence` (tests whose evidence is stale or unverified first, worked out in the viewer) and those of the catalog. */
+export const EXPLORER_SORTS = ['relevance', 'evidence', ...TEST_SORTS] as const;
 /** One of {@link EXPLORER_SORTS}. */
 export type ExplorerSort = (typeof EXPLORER_SORTS)[number];
 
@@ -55,12 +55,20 @@ export const ExplorerSearchFields = {
   branch: text,
   /** Outcomes of a run or recorded result (`passed`, `failed`…). */
   outcome: text,
+  /** Recording completeness of a run (`complete`, `incomplete`, `none-expected`, `unknown`): runs only, apart from the outcome. */
+  recordings: text,
   /** Only records from this day (`YYYY-MM-DD`, the viewer's local time). */
   since: text,
   /** Only records until the end of this day (`YYYY-MM-DD`, local time). */
   until: text,
   /** Only the tests a CI run recorded (a `RunSummary.key`). Not `run`, which already names a recorded statechart run. */
   inRun: text,
+  /** `stale`: only tests whose evidence is stale or unverified (not that applies to the current promise and, when one is named, the release). Never a test status. */
+  evidence: z.enum(['stale']).optional().catch(undefined),
+  /** The release evidence is read against: its build, commit and platform (the names a spec page uses). A build such as `90` is read from a typed link as a number: coerced back to its text. */
+  release: text,
+  releaseSha: text,
+  releasePlatform: text,
   group: z.enum(TEST_GROUPS).optional().catch(undefined),
   sort: z.enum(EXPLORER_SORTS).optional().catch(undefined),
   tab: z.enum(['runs', 'artifacts', 'collections', 'compare', 'health']).optional().catch(undefined),
@@ -124,9 +132,14 @@ export type ExplorerSearch = {
   readonly label?: string;
   readonly branch?: string;
   readonly outcome?: string;
+  readonly recordings?: string;
   readonly since?: string;
   readonly until?: string;
   readonly inRun?: string;
+  readonly evidence?: 'stale';
+  readonly release?: string;
+  readonly releaseSha?: string;
+  readonly releasePlatform?: string;
   readonly group?: TestGroup;
   readonly sort?: ExplorerSort;
   readonly tab?: 'runs' | 'artifacts' | 'collections' | 'compare' | 'health';
@@ -189,7 +202,7 @@ export function queryOf(search: ExplorerSearch): TestQuery {
     ...(values(search.scope).length ? { scope: values(search.scope) } : {}),
     artifactTag: values(search.artifactTag),
     ...(search.inherited ? { inherited: true } : {}),
-    sort: search.sort === 'relevance' || !search.sort ? 'spec' : search.sort
+    sort: search.sort === 'relevance' || search.sort === 'evidence' || !search.sort ? 'spec' : search.sort
   };
 }
 
@@ -200,7 +213,7 @@ export function explorerSearch(search: Record<string, unknown>): ExplorerSearch
 
 /** True when any filter or search narrows the list. */
 export function narrowed(search: ExplorerSearch): boolean {
-  return !!(search.q || search.status || search.env || search.producer || search.domain || search.spec || search.tag || search.scope || search.artifactTag || search.inRun || search.video);
+  return !!(search.q || search.status || search.env || search.producer || search.domain || search.spec || search.tag || search.scope || search.artifactTag || search.inRun || search.video || search.evidence);
 }
 
 /** The reliability scope an address asks for, in the shape `reliability` reads: days are the viewer's local days. */
@@ -219,7 +232,7 @@ export function reliabilityScopeOf(search: ExplorerSearch): ReliabilityScope {
 }
 
 /** The filters of the Runs and Artifacts tabs, in the address. */
-export const FILTER_FIELDS = ['q', 'artifactTag', 'inherited', 'type', 'label', 'env', 'producer', 'branch', 'outcome', 'domain', 'spec', 'since', 'until', 'video'] as const;
+export const FILTER_FIELDS = ['q', 'artifactTag', 'inherited', 'type', 'label', 'env', 'producer', 'branch', 'outcome', 'recordings', 'domain', 'spec', 'since', 'until', 'video'] as const;
 
 /** A patch that removes every filter of the Runs and Artifacts tabs. */
 export const NO_FILTERS = Object.fromEntries(FILTER_FIELDS.map((field) => [field, undefined])) as { [K in (typeof FILTER_FIELDS)[number]]?: undefined };
@@ -253,6 +266,7 @@ export function filterOf(search: ExplorerSearch): ArtifactFilter {
     producer: list(search.producer),
     branch: list(search.branch),
     outcome: list(search.outcome),
+    recordings: list(search.recordings),
     domain: list(search.domain),
     spec: list(search.spec),
     since,