@crvouga/atlas 1.15.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -1
- package/dist/reports/src/catalog-health.d.ts +15 -2
- package/dist/reports/src/catalog-health.js +23 -2
- package/dist/reports/src/test-catalog.d.ts +13 -0
- package/dist/reports/src/test-catalog.js +21 -5
- package/dist/schema/src/api.d.ts +35 -0
- package/dist/schema/src/api.js +14 -3
- package/dist/schema/src/server-config.d.ts +3 -0
- package/dist/schema/src/server-config.js +5 -0
- package/dist/server/src/app.d.ts +2 -0
- package/dist/server/src/app.js +6 -2
- package/dist/server/src/config.js +4 -0
- package/dist/server/src/main.js +1 -0
- package/dist/server/src/openapi.js +2 -2
- package/docs/extending.md +1 -1
- package/docs/reference/http-api.md +4 -4
- package/docs/reference/reports.md +41 -2
- package/docs/reference/server.md +2 -0
- package/docs/server.md +11 -2
- package/docs/ui.md +28 -1
- package/llms-full.txt +44 -8
- package/package.json +1 -1
- package/packages/schema/schemas/catalog-health.schema.json +20 -0
- package/packages/schema/schemas/server-config.schema.json +17 -0
- package/packages/schema/schemas/test-catalog.schema.json +26 -0
- package/packages/schema/schemas/test-history.schema.json +40 -0
- package/packages/server/openapi.json +90 -4
- package/packages/visualizer/src/evidence/applicability-view.tsx +46 -20
- package/packages/visualizer/src/evidence/applicability.test.ts +68 -1
- package/packages/visualizer/src/evidence/applicability.ts +121 -26
- package/packages/visualizer/src/explorer/Explorer.tsx +105 -10
- package/packages/visualizer/src/explorer/HealthView.tsx +402 -0
- package/packages/visualizer/src/explorer/TestDetail.tsx +14 -1
- package/packages/visualizer/src/explorer/applicability.ts +43 -0
- package/packages/visualizer/src/explorer/catalog.tsx +33 -0
- package/packages/visualizer/src/explorer/search.ts +20 -7
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Catalog health view
|
|
3
|
+
*
|
|
4
|
+
* The Catalog health tab: why the Specs and the Tests of a report disagree, and what a producer or an operator can change.
|
|
5
|
+
* It shows the denominators apart (spec files, defined specs, specs with an executed test, tests, outlines, examples and
|
|
6
|
+
* per-environment observations), the steps from the Specs count to the Tests count, the domains in use, specs seen under
|
|
7
|
+
* more than one path, and every finding with the paths it affects. Two paths are one logical spec only when a root mapping
|
|
8
|
+
* relates them: the server's configured mappings are shown as such, and further `from>to` mappings can be tried here
|
|
9
|
+
* (they are the address's `root`, so a link reproduces the figures). Ambiguous paths are never merged by similar title.
|
|
10
|
+
* The tab only reads: it never changes a status, a count of passed checks or a domain.
|
|
11
|
+
*/
|
|
12
|
+
import { parseRootMappings, type CatalogHealth, type HealthFinding, type RootMapping } from '@crvouga/atlas/reports/catalog-health';
|
|
13
|
+
import { Accordion, ActionIcon, Alert, Badge, Box, Button, Code, Group, List, Paper, SimpleGrid, Stack, Table, Text, TextInput, Title, Tooltip } from '@mantine/core';
|
|
14
|
+
import { IconTrash } from '@tabler/icons-react';
|
|
15
|
+
import { useMemo, useState } from 'react';
|
|
16
|
+
|
|
17
|
+
import { plural } from '../evidence/format';
|
|
18
|
+
import { useCatalogHealth } from './catalog';
|
|
19
|
+
import { useExplorerNavigate, useExplorerSearch } from './navigate';
|
|
20
|
+
|
|
21
|
+
const label = (root: { readonly from: string; readonly to: string }) => `${root.from}>${root.to}`;
|
|
22
|
+
|
|
23
|
+
function Figure({ name, value, hint }: { name: string; value: number; hint: string }) {
|
|
24
|
+
return (
|
|
25
|
+
<Paper withBorder p="sm" title={hint}>
|
|
26
|
+
<Text size="xs" c="dimmed" component="div">
|
|
27
|
+
{name}
|
|
28
|
+
</Text>
|
|
29
|
+
<Text fw={700} size="xl" component="div" data-health-figure={name}>
|
|
30
|
+
{value}
|
|
31
|
+
</Text>
|
|
32
|
+
</Paper>
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function Counts({ health }: { health: CatalogHealth }) {
|
|
37
|
+
const { specs, tests, scenarios } = health.counts;
|
|
38
|
+
return (
|
|
39
|
+
<Stack gap="md">
|
|
40
|
+
<Box component="section" aria-label="Spec counts">
|
|
41
|
+
<Title order={4} mb="xs">
|
|
42
|
+
Specs
|
|
43
|
+
</Title>
|
|
44
|
+
<SimpleGrid cols={{ base: 2, sm: 3, md: 5 }}>
|
|
45
|
+
<Figure name="Spec files" value={specs.files} hint="Files the report is laid over, before any root mapping. This is the number the Specs view shows." />
|
|
46
|
+
<Figure name="Defined specs" value={specs.defined} hint="Spec files, with those joined by a declared root mapping counted once." />
|
|
47
|
+
<Figure name="Specs with tests" value={specs.withTests} hint="Specs, defined or not, with at least one test. This is what the Tests view groups by." />
|
|
48
|
+
<Figure name="Specs executed" value={specs.executed} hint="Defined specs with a test that passed or failed: the denominator of execution coverage." />
|
|
49
|
+
<Figure name="Defined, not executed" value={specs.withoutExecution} hint="Defined specs with no passed or failed test: the coverage gaps. They add to no passed count." />
|
|
50
|
+
</SimpleGrid>
|
|
51
|
+
</Box>
|
|
52
|
+
<Box component="section" aria-label="Test counts">
|
|
53
|
+
<Title order={4} mb="xs">
|
|
54
|
+
Tests
|
|
55
|
+
</Title>
|
|
56
|
+
<SimpleGrid cols={{ base: 2, sm: 3, md: 5 }}>
|
|
57
|
+
<Figure name="Tests" value={tests.total} hint="Check identities: a spec path and an id, however many environments ran them." />
|
|
58
|
+
<Figure name="Tests passed" value={tests.passed} hint="Tests whose current status is passed." />
|
|
59
|
+
<Figure name="Tests with a spec" value={tests.withSpec} hint="Tests that name the spec they belong to." />
|
|
60
|
+
<Figure name="Tests without a spec" value={tests.withoutSpec} hint="Tests that name no spec; they are in no spec or domain group." />
|
|
61
|
+
<Figure name="Only in history" value={tests.historicalOnly} hint="Tests the history holds but the report has no current result for (removed, renamed or not re-run)." />
|
|
62
|
+
</SimpleGrid>
|
|
63
|
+
</Box>
|
|
64
|
+
<Box component="section" aria-label="Scenario counts">
|
|
65
|
+
<Title order={4} mb="xs">
|
|
66
|
+
Outlines, examples and observations
|
|
67
|
+
</Title>
|
|
68
|
+
<SimpleGrid cols={{ base: 2, sm: 3, md: 5 }}>
|
|
69
|
+
<Figure name="Outlines" value={scenarios.outlines} hint="Tests sharing the label outline count once; a test without it is its own outline." />
|
|
70
|
+
<Figure name="Examples" value={scenarios.examples} hint="Concrete examples: tests, however many environments ran them." />
|
|
71
|
+
<Figure name="Observations" value={scenarios.observations} hint="Current results per producer, environment and parameter set." />
|
|
72
|
+
<Figure name="Observations executed" value={scenarios.executedObservations} hint="Observations that passed or failed." />
|
|
73
|
+
<Figure name="Observations passed" value={scenarios.passedObservations} hint="Observations that passed." />
|
|
74
|
+
</SimpleGrid>
|
|
75
|
+
</Box>
|
|
76
|
+
</Stack>
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function Reconciliation({ health }: { health: CatalogHealth }) {
|
|
81
|
+
const { reconciliation } = health;
|
|
82
|
+
let total = reconciliation.specs;
|
|
83
|
+
const rows = reconciliation.steps.map((step) => {
|
|
84
|
+
total += step.delta;
|
|
85
|
+
return (
|
|
86
|
+
<Table.Tr key={step.code}>
|
|
87
|
+
<Table.Td>{step.label}</Table.Td>
|
|
88
|
+
<Table.Td ta="right">{step.delta > 0 ? `+${step.delta}` : step.delta}</Table.Td>
|
|
89
|
+
<Table.Td ta="right">{total}</Table.Td>
|
|
90
|
+
</Table.Tr>
|
|
91
|
+
);
|
|
92
|
+
});
|
|
93
|
+
return (
|
|
94
|
+
<Box component="section" aria-label="Why Specs and Tests differ">
|
|
95
|
+
<Title order={3} mb="xs">
|
|
96
|
+
Why Specs and Tests differ
|
|
97
|
+
</Title>
|
|
98
|
+
<Table.ScrollContainer minWidth={420}>
|
|
99
|
+
<Table data-health-reconciliation>
|
|
100
|
+
<Table.Thead>
|
|
101
|
+
<Table.Tr>
|
|
102
|
+
<Table.Th scope="col">Step</Table.Th>
|
|
103
|
+
<Table.Th scope="col" ta="right">
|
|
104
|
+
Change
|
|
105
|
+
</Table.Th>
|
|
106
|
+
<Table.Th scope="col" ta="right">
|
|
107
|
+
Total
|
|
108
|
+
</Table.Th>
|
|
109
|
+
</Table.Tr>
|
|
110
|
+
</Table.Thead>
|
|
111
|
+
<Table.Tbody>
|
|
112
|
+
<Table.Tr>
|
|
113
|
+
<Table.Td>Spec files the Specs view lists</Table.Td>
|
|
114
|
+
<Table.Td />
|
|
115
|
+
<Table.Td ta="right">{reconciliation.specs}</Table.Td>
|
|
116
|
+
</Table.Tr>
|
|
117
|
+
{rows}
|
|
118
|
+
<Table.Tr>
|
|
119
|
+
<Table.Td fw={600}>Specs the Tests view groups its tests into</Table.Td>
|
|
120
|
+
<Table.Td />
|
|
121
|
+
<Table.Td ta="right" fw={600}>
|
|
122
|
+
{reconciliation.testSpecs}
|
|
123
|
+
</Table.Td>
|
|
124
|
+
</Table.Tr>
|
|
125
|
+
</Table.Tbody>
|
|
126
|
+
</Table>
|
|
127
|
+
</Table.ScrollContainer>
|
|
128
|
+
<List size="sm" mt="sm" data-health-explanation>
|
|
129
|
+
{reconciliation.explanation.map((line) => (
|
|
130
|
+
<List.Item key={line}>{line}</List.Item>
|
|
131
|
+
))}
|
|
132
|
+
</List>
|
|
133
|
+
</Box>
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function Finding({ item }: { item: HealthFinding }) {
|
|
138
|
+
return (
|
|
139
|
+
<Accordion.Item value={item.code} data-health-finding={item.code}>
|
|
140
|
+
<Accordion.Control>
|
|
141
|
+
<Group gap="xs" wrap="wrap">
|
|
142
|
+
<Badge variant="light" color={item.severity === 'warning' ? 'orange' : 'blue'} tt="none">
|
|
143
|
+
{item.severity === 'warning' ? 'Warning' : 'Note'}
|
|
144
|
+
</Badge>
|
|
145
|
+
<Code>{item.code}</Code>
|
|
146
|
+
<Text span size="sm" c="dimmed">
|
|
147
|
+
{item.count} affected
|
|
148
|
+
</Text>
|
|
149
|
+
</Group>
|
|
150
|
+
</Accordion.Control>
|
|
151
|
+
<Accordion.Panel>
|
|
152
|
+
<Stack gap="xs">
|
|
153
|
+
<Text component="div" size="sm">
|
|
154
|
+
{item.message}
|
|
155
|
+
</Text>
|
|
156
|
+
{item.items.length > 0 && (
|
|
157
|
+
<List size="sm" listStyleType="none">
|
|
158
|
+
{item.items.map((entry) => (
|
|
159
|
+
<List.Item key={entry.path}>
|
|
160
|
+
<Code>{entry.path}</Code>
|
|
161
|
+
{entry.candidates?.map((candidate) => (
|
|
162
|
+
<Text key={candidate.path} component="div" size="xs" c="dimmed" ml="md">
|
|
163
|
+
may be the same promise as <Code>{candidate.path}</Code> ({candidate.reason}); not merged
|
|
164
|
+
</Text>
|
|
165
|
+
))}
|
|
166
|
+
</List.Item>
|
|
167
|
+
))}
|
|
168
|
+
</List>
|
|
169
|
+
)}
|
|
170
|
+
{item.count > item.items.length && (
|
|
171
|
+
<Text size="xs" c="dimmed" component="div">
|
|
172
|
+
and {item.count - item.items.length} more
|
|
173
|
+
</Text>
|
|
174
|
+
)}
|
|
175
|
+
</Stack>
|
|
176
|
+
</Accordion.Panel>
|
|
177
|
+
</Accordion.Item>
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function Mappings({ health, tried }: { health: CatalogHealth | undefined; tried: readonly RootMapping[] }) {
|
|
182
|
+
const go = useExplorerNavigate();
|
|
183
|
+
const [from, setFrom] = useState('');
|
|
184
|
+
const [to, setTo] = useState('');
|
|
185
|
+
const configured = (health?.roots ?? []).filter((root) => root.source === 'configuration');
|
|
186
|
+
const write = (list: readonly RootMapping[]) => go({ root: list.length ? list.map(label).join(',') : undefined }, { replace: true });
|
|
187
|
+
const known = [...configured, ...tried].some((root) => root.from === from.trim() && root.to === to.trim());
|
|
188
|
+
const matched = (root: RootMapping) => health?.roots.find((item) => item.from === root.from && item.to === root.to && item.source !== 'configuration')?.matched;
|
|
189
|
+
return (
|
|
190
|
+
<Box component="section" aria-label="Root mappings">
|
|
191
|
+
<Title order={3} mb="xs">
|
|
192
|
+
Root mappings
|
|
193
|
+
</Title>
|
|
194
|
+
<Text size="sm" c="dimmed" mb="sm">
|
|
195
|
+
Two spec paths are one spec only when a mapping says so. A mapping states that paths under one folder are the same specs as paths under another; Atlas never merges paths by a similar title. A mapping tried here is kept in the address and changes nothing on the server. To keep one, set <Code>catalog.roots</Code> or <Code>ATLAS_CATALOG_ROOTS</Code> on the server.
|
|
196
|
+
</Text>
|
|
197
|
+
{configured.length + tried.length > 0 && (
|
|
198
|
+
<List size="sm" listStyleType="none" mb="sm" aria-label="Declared root mappings">
|
|
199
|
+
{configured.map((root) => (
|
|
200
|
+
<List.Item key={`c${label(root)}`}>
|
|
201
|
+
<Group gap="xs" wrap="wrap">
|
|
202
|
+
<Code>{label(root)}</Code>
|
|
203
|
+
<Badge variant="light" tt="none">
|
|
204
|
+
Configured on the server
|
|
205
|
+
</Badge>
|
|
206
|
+
<Text span size="xs" c="dimmed">
|
|
207
|
+
changed {plural(root.matched, 'path', 'paths')}
|
|
208
|
+
</Text>
|
|
209
|
+
</Group>
|
|
210
|
+
</List.Item>
|
|
211
|
+
))}
|
|
212
|
+
{tried.map((root) => (
|
|
213
|
+
<List.Item key={`t${label(root)}`}>
|
|
214
|
+
<Group gap="xs" wrap="wrap">
|
|
215
|
+
<Code>{label(root)}</Code>
|
|
216
|
+
<Badge variant="light" color="gray" tt="none">
|
|
217
|
+
Tried here
|
|
218
|
+
</Badge>
|
|
219
|
+
{matched(root) !== undefined && (
|
|
220
|
+
<Text span size="xs" c="dimmed">
|
|
221
|
+
changed {plural(matched(root)!, 'path', 'paths')}
|
|
222
|
+
</Text>
|
|
223
|
+
)}
|
|
224
|
+
<Tooltip label="Stop trying this mapping">
|
|
225
|
+
<ActionIcon variant="subtle" color="gray" aria-label={`Stop trying ${label(root)}`} onClick={() => write(tried.filter((item) => item !== root))}>
|
|
226
|
+
<IconTrash size={16} aria-hidden="true" />
|
|
227
|
+
</ActionIcon>
|
|
228
|
+
</Tooltip>
|
|
229
|
+
</Group>
|
|
230
|
+
</List.Item>
|
|
231
|
+
))}
|
|
232
|
+
</List>
|
|
233
|
+
)}
|
|
234
|
+
<form
|
|
235
|
+
onSubmit={(event) => {
|
|
236
|
+
event.preventDefault();
|
|
237
|
+
if (!to.trim() || known) return;
|
|
238
|
+
write([...tried, { from: from.trim(), to: to.trim() }]);
|
|
239
|
+
setFrom('');
|
|
240
|
+
setTo('');
|
|
241
|
+
}}
|
|
242
|
+
>
|
|
243
|
+
<Group align="flex-end" gap="md" wrap="wrap">
|
|
244
|
+
<TextInput label="Paths under" description="Leave empty to match every path" placeholder="checkout/" value={from} onChange={(event) => setFrom(event.currentTarget.value)} />
|
|
245
|
+
<TextInput label="are the same specs as paths under" description="Where the specs are recorded" placeholder="specs/checkout/" value={to} onChange={(event) => setTo(event.currentTarget.value)} />
|
|
246
|
+
<Button type="submit" disabled={!to.trim() || known}>
|
|
247
|
+
Try this mapping
|
|
248
|
+
</Button>
|
|
249
|
+
</Group>
|
|
250
|
+
</form>
|
|
251
|
+
</Box>
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** The Catalog health tab. */
|
|
256
|
+
export function HealthView() {
|
|
257
|
+
const search = useExplorerSearch();
|
|
258
|
+
const tried = useMemo(() => (search.root ? parseRootMappings(new URLSearchParams([['root', search.root]])) : []), [search.root]);
|
|
259
|
+
const answer = useCatalogHealth(tried);
|
|
260
|
+
const health = answer.data && !answer.data.unsupported ? answer.data.health : undefined;
|
|
261
|
+
const warnings = health?.findings.filter((item) => item.severity === 'warning') ?? [];
|
|
262
|
+
return (
|
|
263
|
+
<Box aria-label="Catalog health" component="section" mb="lg">
|
|
264
|
+
<Title order={2} mb="sm">
|
|
265
|
+
Catalog health
|
|
266
|
+
</Title>
|
|
267
|
+
<Text component="div" c="dimmed" mb="md">
|
|
268
|
+
Specs, domains and coverage counts can disagree because of how specs are named and described, not because of a defect in a test. This explains each number, where the totals differ, and what to change at the source. It only reads: it changes no result.
|
|
269
|
+
</Text>
|
|
270
|
+
<Stack gap="lg">
|
|
271
|
+
<Mappings health={health} tried={tried} />
|
|
272
|
+
{answer.isPending && (
|
|
273
|
+
<Text role="status" component="div" c="dimmed">
|
|
274
|
+
Reading the catalog…
|
|
275
|
+
</Text>
|
|
276
|
+
)}
|
|
277
|
+
{answer.isError && (
|
|
278
|
+
<Alert role="alert" color="red">
|
|
279
|
+
The catalog health could not be read. {answer.error.message}
|
|
280
|
+
<Group mt="xs">
|
|
281
|
+
<Button type="button" variant="light" onClick={() => void answer.refetch()}>
|
|
282
|
+
Try again
|
|
283
|
+
</Button>
|
|
284
|
+
</Group>
|
|
285
|
+
</Alert>
|
|
286
|
+
)}
|
|
287
|
+
{answer.data?.unsupported && (
|
|
288
|
+
<Alert role="status" color="blue">
|
|
289
|
+
This server is older than the catalog health report, so it cannot explain these counts. Upgrade the server to read it.
|
|
290
|
+
</Alert>
|
|
291
|
+
)}
|
|
292
|
+
{health && (
|
|
293
|
+
<>
|
|
294
|
+
<Alert role="status" color={health.ok ? 'green' : 'orange'} title={health.ok ? 'No warnings' : plural(warnings.length, 'warning', 'warnings')} data-health-summary>
|
|
295
|
+
{health.ok ? 'Nothing found that makes a count or a domain filter misleading.' : 'These make a count or a domain filter misleading. Each says what to change at its source.'}
|
|
296
|
+
</Alert>
|
|
297
|
+
<Reconciliation health={health} />
|
|
298
|
+
<Counts health={health} />
|
|
299
|
+
{health.aliases.length > 0 && (
|
|
300
|
+
<Box component="section" aria-label="Specs under more than one path">
|
|
301
|
+
<Title order={3} mb="xs">
|
|
302
|
+
Specs under more than one path
|
|
303
|
+
</Title>
|
|
304
|
+
<Text size="sm" c="dimmed" mb="xs">
|
|
305
|
+
Joined because a root mapping relates them. Every original path stays inspectable.
|
|
306
|
+
</Text>
|
|
307
|
+
<Table.ScrollContainer minWidth={420}>
|
|
308
|
+
<Table data-health-aliases>
|
|
309
|
+
<Table.Thead>
|
|
310
|
+
<Table.Tr>
|
|
311
|
+
<Table.Th scope="col">Spec</Table.Th>
|
|
312
|
+
<Table.Th scope="col">Original paths</Table.Th>
|
|
313
|
+
</Table.Tr>
|
|
314
|
+
</Table.Thead>
|
|
315
|
+
<Table.Tbody>
|
|
316
|
+
{health.aliases.map((alias) => (
|
|
317
|
+
<Table.Tr key={alias.path}>
|
|
318
|
+
<Table.Td>
|
|
319
|
+
<Code>{alias.path}</Code>
|
|
320
|
+
</Table.Td>
|
|
321
|
+
<Table.Td>
|
|
322
|
+
<Stack gap={2}>
|
|
323
|
+
{alias.sources.map((source) => (
|
|
324
|
+
<Code key={source}>{source}</Code>
|
|
325
|
+
))}
|
|
326
|
+
</Stack>
|
|
327
|
+
</Table.Td>
|
|
328
|
+
</Table.Tr>
|
|
329
|
+
))}
|
|
330
|
+
</Table.Tbody>
|
|
331
|
+
</Table>
|
|
332
|
+
</Table.ScrollContainer>
|
|
333
|
+
</Box>
|
|
334
|
+
)}
|
|
335
|
+
<Box component="section" aria-label="Domains">
|
|
336
|
+
<Title order={3} mb="xs">
|
|
337
|
+
Domains
|
|
338
|
+
</Title>
|
|
339
|
+
<Text size="sm" c="dimmed" mb="xs">
|
|
340
|
+
A domain is what a spec's own metadata says. Explicit metadata is always authoritative; a domain shared by nearly every spec is reported as a finding below, not changed.
|
|
341
|
+
</Text>
|
|
342
|
+
{health.domains.length === 0 ? (
|
|
343
|
+
<Text component="div" c="dimmed">
|
|
344
|
+
No spec declares a domain.
|
|
345
|
+
</Text>
|
|
346
|
+
) : (
|
|
347
|
+
<Table.ScrollContainer minWidth={320}>
|
|
348
|
+
<Table data-health-domains>
|
|
349
|
+
<Table.Thead>
|
|
350
|
+
<Table.Tr>
|
|
351
|
+
<Table.Th scope="col">Domain</Table.Th>
|
|
352
|
+
<Table.Th scope="col" ta="right">
|
|
353
|
+
Specs
|
|
354
|
+
</Table.Th>
|
|
355
|
+
</Table.Tr>
|
|
356
|
+
</Table.Thead>
|
|
357
|
+
<Table.Tbody>
|
|
358
|
+
{health.domains.map((domain) => (
|
|
359
|
+
<Table.Tr key={domain.name}>
|
|
360
|
+
<Table.Td>{domain.name}</Table.Td>
|
|
361
|
+
<Table.Td ta="right">{domain.specs}</Table.Td>
|
|
362
|
+
</Table.Tr>
|
|
363
|
+
))}
|
|
364
|
+
</Table.Tbody>
|
|
365
|
+
</Table>
|
|
366
|
+
</Table.ScrollContainer>
|
|
367
|
+
)}
|
|
368
|
+
</Box>
|
|
369
|
+
<Box component="section" aria-label="Findings">
|
|
370
|
+
<Title order={3} mb="xs">
|
|
371
|
+
Findings
|
|
372
|
+
</Title>
|
|
373
|
+
{health.findings.length === 0 ? (
|
|
374
|
+
<Text component="div" c="dimmed">
|
|
375
|
+
Nothing to report.
|
|
376
|
+
</Text>
|
|
377
|
+
) : (
|
|
378
|
+
<Accordion multiple defaultValue={warnings.map((item) => item.code)}>
|
|
379
|
+
{health.findings.map((item) => (
|
|
380
|
+
<Finding key={item.code} item={item} />
|
|
381
|
+
))}
|
|
382
|
+
</Accordion>
|
|
383
|
+
)}
|
|
384
|
+
</Box>
|
|
385
|
+
<Box component="section" aria-label="Identity">
|
|
386
|
+
<Title order={3} mb="xs">
|
|
387
|
+
Identity
|
|
388
|
+
</Title>
|
|
389
|
+
<Text size="sm" component="div" data-health-identity>
|
|
390
|
+
Fallback identity (known by title only): {health.identity.fallbackIdentity} of {health.identity.checks} checks.
|
|
391
|
+
{health.identity.declaredBehaviorIds !== undefined && ` Declared behavior ids: ${health.identity.declaredBehaviorIds}. Declared example ids: ${health.identity.declaredExampleIds ?? 0}.`}
|
|
392
|
+
</Text>
|
|
393
|
+
<Text size="sm" c="dimmed" component="div" mt="xs">
|
|
394
|
+
{health.identity.basis}
|
|
395
|
+
</Text>
|
|
396
|
+
</Box>
|
|
397
|
+
</>
|
|
398
|
+
)}
|
|
399
|
+
</Stack>
|
|
400
|
+
</Box>
|
|
401
|
+
);
|
|
402
|
+
}
|
|
@@ -13,12 +13,15 @@ import { environmentLabel } from '@crvouga/atlas/reports/layer-axes';
|
|
|
13
13
|
import { formatMs, runKey, type TestResultEntry, type TestSummary } from '@crvouga/atlas/reports/test-catalog';
|
|
14
14
|
import { Link } from '@tanstack/react-router';
|
|
15
15
|
import { lazy, Suspense, useState } from 'react';
|
|
16
|
+
import { assessTest } from '../evidence/applicability';
|
|
17
|
+
import { ApplicabilitySummary } from '../evidence/applicability-view';
|
|
16
18
|
import { CheckDetail } from '../evidence/check-detail';
|
|
17
19
|
import { ago, exactTime, plural, shortSha, statusWord, useNow } from '../evidence/format';
|
|
18
20
|
import { useEvidence } from '../evidence/evidence-context';
|
|
19
21
|
import { Origin, StatusMark } from '../evidence/marks';
|
|
20
22
|
import { outcomeHeadline, selectedResult } from '../evidence/outcome';
|
|
21
23
|
import { useTestHistory } from './catalog';
|
|
24
|
+
import { releaseOf } from './applicability';
|
|
22
25
|
import { patched, useExplorerNavigate, useExplorerSearch } from './navigate';
|
|
23
26
|
import { Badges, runName } from './parts';
|
|
24
27
|
import { Reliability } from './Reliability';
|
|
@@ -129,7 +132,10 @@ function Scope({ pinned, slot }: { pinned: PinnedResult; slot: CheckSlot | undef
|
|
|
129
132
|
export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot | undefined }) {
|
|
130
133
|
const now = useNow();
|
|
131
134
|
const history = useTestHistory(test.key);
|
|
132
|
-
const
|
|
135
|
+
const search = useExplorerSearch();
|
|
136
|
+
const { layer, source } = search;
|
|
137
|
+
const release = releaseOf(search);
|
|
138
|
+
const applicability = assessTest(test, release);
|
|
133
139
|
const go = useExplorerNavigate();
|
|
134
140
|
const pinned = usePinnedResult(layer, test.key);
|
|
135
141
|
const [recording, setRecording] = useState(false);
|
|
@@ -238,6 +244,13 @@ export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot
|
|
|
238
244
|
|
|
239
245
|
<Scope pinned={pinned} slot={slot} />
|
|
240
246
|
|
|
247
|
+
<Box aria-label="Evidence applicability" component="section" mb="lg">
|
|
248
|
+
<Title order={3} mb="sm" c="dimmed">
|
|
249
|
+
Does this evidence apply?
|
|
250
|
+
</Title>
|
|
251
|
+
<ApplicabilitySummary assessment={applicability} release={release} />
|
|
252
|
+
</Box>
|
|
253
|
+
|
|
241
254
|
<Box aria-label="Current results" component="section" mb="lg">
|
|
242
255
|
<Title order={3} mb="sm" c="dimmed">
|
|
243
256
|
Current result per environment
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Explorer evidence applicability
|
|
3
|
+
*
|
|
4
|
+
* The explorer's part of evidence applicability: the release a reader names and the stale-or-unverified filter, held in
|
|
5
|
+
* the explorer's address (`release`, `releaseSha`, `releasePlatform`, `evidence=stale`, the same names a spec page uses),
|
|
6
|
+
* and the reading of every test of the catalog against them. The reading is the one a feature page makes
|
|
7
|
+
* (`evidence/applicability.ts`); it never changes a test's status.
|
|
8
|
+
*/
|
|
9
|
+
import { useMemo } from 'react';
|
|
10
|
+
import type { TestSummary } from '@crvouga/atlas/reports/test-catalog';
|
|
11
|
+
|
|
12
|
+
import { assessTest, type Assessment, type ReleaseIdentity } from '../evidence/applicability';
|
|
13
|
+
import type { ApplicabilityState } from '../evidence/applicability-view';
|
|
14
|
+
import { useExplorerNavigate, useExplorerSearch } from './navigate';
|
|
15
|
+
import type { ExplorerSearch } from './search';
|
|
16
|
+
|
|
17
|
+
/** The release an explorer address names; every part is optional. */
|
|
18
|
+
export function releaseOf(search: ExplorerSearch): ReleaseIdentity {
|
|
19
|
+
return {
|
|
20
|
+
...(search.release ? { build: search.release } : {}),
|
|
21
|
+
...(search.releaseSha ? { sha: search.releaseSha } : {}),
|
|
22
|
+
...(search.releasePlatform ? { platform: search.releasePlatform } : {})
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Each test's evidence read against the promise and the release, by test key. */
|
|
27
|
+
export function assessTests(tests: readonly TestSummary[], release: ReleaseIdentity): ReadonlyMap<string, Assessment> {
|
|
28
|
+
return new Map(tests.map((test) => [test.key, assessTest(test, release)] as const));
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The release and the evidence filter of the explorer's address, with the ways to change them. */
|
|
32
|
+
export function useExplorerApplicability(): ApplicabilityState {
|
|
33
|
+
const search = useExplorerSearch();
|
|
34
|
+
const go = useExplorerNavigate();
|
|
35
|
+
const { release: build, releaseSha: sha, releasePlatform: platform } = search;
|
|
36
|
+
const release = useMemo(() => releaseOf({ release: build, releaseSha: sha, releasePlatform: platform }), [build, sha, platform]);
|
|
37
|
+
return {
|
|
38
|
+
release,
|
|
39
|
+
onlyStale: search.evidence === 'stale',
|
|
40
|
+
setRelease: (next) => go({ release: next.build || undefined, releaseSha: next.sha || undefined, releasePlatform: next.platform || undefined }, { replace: true }),
|
|
41
|
+
setOnlyStale: (on) => go({ evidence: on ? 'stale' : undefined }, { replace: true })
|
|
42
|
+
};
|
|
43
|
+
}
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
type FilterName,
|
|
27
27
|
type QueryFacets
|
|
28
28
|
} from '@crvouga/atlas/reports/artifact-query';
|
|
29
|
+
import { catalogHealth, type CatalogHealth, type RootMapping } from '@crvouga/atlas/reports/catalog-health';
|
|
29
30
|
import {
|
|
30
31
|
artifactList,
|
|
31
32
|
catalogModel,
|
|
@@ -371,3 +372,35 @@ export function useRunDetail(key: string | undefined) {
|
|
|
371
372
|
retry: false
|
|
372
373
|
});
|
|
373
374
|
}
|
|
375
|
+
|
|
376
|
+
/** What the Catalog health tab reads: the diagnostics, or that this server is too old to give them. */
|
|
377
|
+
export type HealthAnswer = { readonly unsupported: false; readonly health: CatalogHealth } | { readonly unsupported: true };
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Why the Specs, domains and counts of the report disagree. From the server when it has one (it also applies the root
|
|
381
|
+
* mappings it is configured with, and says so in `roots[].source`); otherwise worked out here from the specs and
|
|
382
|
+
* results this page holds. A server without the route answers `unsupported`, and the page says so.
|
|
383
|
+
*/
|
|
384
|
+
export function useCatalogHealth(roots: readonly RootMapping[]) {
|
|
385
|
+
const catalog = useTestCatalog();
|
|
386
|
+
const asked = roots.map((root) => `${root.from}>${root.to}`);
|
|
387
|
+
return useQuery({
|
|
388
|
+
queryKey: ['catalog-health', catalog.version, asked.join('\u0001')],
|
|
389
|
+
queryFn: async ({ signal }): Promise<HealthAnswer> => {
|
|
390
|
+
if (catalog.source === 'server') {
|
|
391
|
+
const params = new URLSearchParams(asked.map((root) => ['root', root]));
|
|
392
|
+
try {
|
|
393
|
+
return { unsupported: false, health: await reportJson<CatalogHealth>(`/tests/health${params.size ? `?${params}` : ''}`, signal) };
|
|
394
|
+
} catch (error) {
|
|
395
|
+
if (notFound(error)) return { unsupported: true };
|
|
396
|
+
throw error;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
return { unsupported: false, health: catalogHealth(catalog.model!, roots) };
|
|
400
|
+
},
|
|
401
|
+
enabled: catalog.model !== null,
|
|
402
|
+
placeholderData: keepPreviousData,
|
|
403
|
+
staleTime: Number.POSITIVE_INFINITY,
|
|
404
|
+
retry: false
|
|
405
|
+
});
|
|
406
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @module Explorer address
|
|
3
3
|
*
|
|
4
4
|
* Everything the test explorer shows is in the address of the `/` route, so a link reproduces the view: the open test,
|
|
5
|
-
* the words searched, the filters, the grouping, the order, the Runs tab with the run opened in it, the Compare tab with its two pinned runs (`base`, `cand`) and the behavior opened (`pair`), the exact layer a test's detail is pinned to, the recording and position of a shared moment in it (`clip`, `clipAt`), and the one file of a run's layer its Artifacts select. Filters that
|
|
5
|
+
* the words searched, the filters, the grouping, the order, the Runs tab with the run opened in it, the Compare tab with its two pinned runs (`base`, `cand`) and the behavior opened (`pair`), the Catalog health tab with the root mappings it tries (`root`), the exact layer a test's detail is pinned to, the recording and position of a shared moment in it (`clip`, `clipAt`), and the one file of a run's layer its Artifacts select. Filters that
|
|
6
6
|
* take several values are comma-separated (`status=failed,flaky`); a comma or backslash inside a value is written `\,`
|
|
7
7
|
* or `\\`, and the router encodes `#` and spaces. Anything unreadable is dropped, never an error.
|
|
8
8
|
*
|
|
@@ -19,8 +19,8 @@ export const TEST_GROUPS = ['spec', 'domain', 'environment', 'status', 'producer
|
|
|
19
19
|
/** One of {@link TEST_GROUPS}. */
|
|
20
20
|
export type TestGroup = (typeof TEST_GROUPS)[number];
|
|
21
21
|
|
|
22
|
-
/** The orders the explorer offers: `relevance` (best match first, only with words to match) and those of the catalog. */
|
|
23
|
-
export const EXPLORER_SORTS = ['relevance', ...TEST_SORTS] as const;
|
|
22
|
+
/** The orders the explorer offers: `relevance` (best match first, only with words to match), `evidence` (tests whose evidence is stale or unverified first, worked out in the viewer) and those of the catalog. */
|
|
23
|
+
export const EXPLORER_SORTS = ['relevance', 'evidence', ...TEST_SORTS] as const;
|
|
24
24
|
/** One of {@link EXPLORER_SORTS}. */
|
|
25
25
|
export type ExplorerSort = (typeof EXPLORER_SORTS)[number];
|
|
26
26
|
|
|
@@ -61,9 +61,17 @@ export const ExplorerSearchFields = {
|
|
|
61
61
|
until: text,
|
|
62
62
|
/** Only the tests a CI run recorded (a `RunSummary.key`). Not `run`, which already names a recorded statechart run. */
|
|
63
63
|
inRun: text,
|
|
64
|
+
/** `stale`: only tests whose evidence is stale or unverified (not that applies to the current promise and, when one is named, the release). Never a test status. */
|
|
65
|
+
evidence: z.enum(['stale']).optional().catch(undefined),
|
|
66
|
+
/** The release evidence is read against: its build, commit and platform (the names a spec page uses). A build such as `90` is read from a typed link as a number: coerced back to its text. */
|
|
67
|
+
release: text,
|
|
68
|
+
releaseSha: text,
|
|
69
|
+
releasePlatform: text,
|
|
64
70
|
group: z.enum(TEST_GROUPS).optional().catch(undefined),
|
|
65
71
|
sort: z.enum(EXPLORER_SORTS).optional().catch(undefined),
|
|
66
|
-
tab: z.enum(['runs', 'artifacts', 'collections', 'compare']).optional().catch(undefined),
|
|
72
|
+
tab: z.enum(['runs', 'artifacts', 'collections', 'compare', 'health']).optional().catch(undefined),
|
|
73
|
+
/** The root mappings (`from>to`, comma-separated) the Catalog health tab tries, beside those the server is configured with. */
|
|
74
|
+
root: text,
|
|
67
75
|
/** The review collection opened in the Collections tab (its stable id), and the revision of it (default: the newest). */
|
|
68
76
|
collection: text,
|
|
69
77
|
rev: z.coerce.number().int().positive().optional().catch(undefined),
|
|
@@ -125,9 +133,14 @@ export type ExplorerSearch = {
|
|
|
125
133
|
readonly since?: string;
|
|
126
134
|
readonly until?: string;
|
|
127
135
|
readonly inRun?: string;
|
|
136
|
+
readonly evidence?: 'stale';
|
|
137
|
+
readonly release?: string;
|
|
138
|
+
readonly releaseSha?: string;
|
|
139
|
+
readonly releasePlatform?: string;
|
|
128
140
|
readonly group?: TestGroup;
|
|
129
141
|
readonly sort?: ExplorerSort;
|
|
130
|
-
readonly tab?: 'runs' | 'artifacts' | 'collections' | 'compare';
|
|
142
|
+
readonly tab?: 'runs' | 'artifacts' | 'collections' | 'compare' | 'health';
|
|
143
|
+
readonly root?: string;
|
|
131
144
|
readonly collection?: string;
|
|
132
145
|
readonly rev?: number;
|
|
133
146
|
readonly base?: string;
|
|
@@ -186,7 +199,7 @@ export function queryOf(search: ExplorerSearch): TestQuery {
|
|
|
186
199
|
...(values(search.scope).length ? { scope: values(search.scope) } : {}),
|
|
187
200
|
artifactTag: values(search.artifactTag),
|
|
188
201
|
...(search.inherited ? { inherited: true } : {}),
|
|
189
|
-
sort: search.sort === 'relevance' || !search.sort ? 'spec' : search.sort
|
|
202
|
+
sort: search.sort === 'relevance' || search.sort === 'evidence' || !search.sort ? 'spec' : search.sort
|
|
190
203
|
};
|
|
191
204
|
}
|
|
192
205
|
|
|
@@ -197,7 +210,7 @@ export function explorerSearch(search: Record<string, unknown>): ExplorerSearch
|
|
|
197
210
|
|
|
198
211
|
/** True when any filter or search narrows the list. */
|
|
199
212
|
export function narrowed(search: ExplorerSearch): boolean {
|
|
200
|
-
return !!(search.q || search.status || search.env || search.producer || search.domain || search.spec || search.tag || search.scope || search.artifactTag || search.inRun || search.video);
|
|
213
|
+
return !!(search.q || search.status || search.env || search.producer || search.domain || search.spec || search.tag || search.scope || search.artifactTag || search.inRun || search.video || search.evidence);
|
|
201
214
|
}
|
|
202
215
|
|
|
203
216
|
/** The reliability scope an address asks for, in the shape `reliability` reads: days are the viewer's local days. */
|