@crvouga/atlas 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -1
- package/docs/extending.md +17 -0
- package/docs/ui.md +14 -0
- package/llms-full.txt +31 -0
- package/package.json +1 -1
- package/packages/visualizer/src/evidence/behavior-panel.tsx +10 -16
- package/packages/visualizer/src/evidence/check-detail.tsx +50 -15
- package/packages/visualizer/src/evidence/outcome-summary.tsx +208 -0
- package/packages/visualizer/src/evidence/outcome.test.ts +129 -0
- package/packages/visualizer/src/evidence/outcome.ts +309 -0
- package/packages/visualizer/src/explorer/TestDetail.tsx +23 -1
- package/packages/visualizer/src/explorer/search.ts +3 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
# Changelog: @crvouga/atlas
|
|
2
2
|
|
|
3
|
-
## 1.
|
|
3
|
+
## 1.7.0 (2026-10-29)
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
|
|
7
|
+
- explain selected, aggregate and historical outcomes without contradictory summaries ([018d2fe](https://github.com/crvouga/atlas/commit/018d2fe91a9c17cbefa8f9d24ad6f3d70550c5d1))
|
|
8
|
+
|
|
9
|
+
## 1.6.0 (2026-10-28)
|
|
4
10
|
|
|
5
11
|
### Features
|
|
6
12
|
|
package/docs/extending.md
CHANGED
|
@@ -104,6 +104,23 @@ every kind of evidence and values from the future) and `layer-extensibility.test
|
|
|
104
104
|
fields survive at every level, malformed evidence is dropped without losing the result, and a reader of the
|
|
105
105
|
first version of the format still reads it all.
|
|
106
106
|
|
|
107
|
+
### Saying what failed
|
|
108
|
+
|
|
109
|
+
A failed check or step may declare the expectation it broke under the `atlas.failure` extension, so a reviewer sees
|
|
110
|
+
it before the raw log. Every field is optional, and the declaration never changes the result.
|
|
111
|
+
|
|
112
|
+
```json
|
|
113
|
+
{
|
|
114
|
+
"name": "The list includes tea", "kind": "then", "status": "failed",
|
|
115
|
+
"extensions": { "atlas.failure": { "expected": "tea in the list", "actual": "the list is empty", "attribution": "product" } }
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
`expected` and `actual` are shown as written. `attribution` is `product`, `test-harness` or `environment`; leave it out
|
|
120
|
+
when you do not know, and the viewer says the cause is unknown. It is never inferred from the message. Whether setup
|
|
121
|
+
failed before the behavior started comes from the failed step's `kind`: `hook`, `setup`, `fixture`, `before`,
|
|
122
|
+
`beforeAll`, `beforeEach` or `given` mean preparation; `when`, `then`, `assertion` or `expect` mean the behavior.
|
|
123
|
+
|
|
107
124
|
## Attach evidence
|
|
108
125
|
|
|
109
126
|
### From the command line
|
package/docs/ui.md
CHANGED
|
@@ -87,6 +87,20 @@ when Watch is on screen.
|
|
|
87
87
|
- The spec index searches titles, paths and declared domains, and can group by the domain from spec metadata (specs with
|
|
88
88
|
none are listed under "No domain declared"; the Tests view reports domain problems).
|
|
89
89
|
|
|
90
|
+
## Outcome summary
|
|
91
|
+
|
|
92
|
+
A test or behavior detail states what the **selected** result says apart from what the other current sources say
|
|
93
|
+
(`evidence/outcome.ts`, `outcome-summary.tsx`). The heading is the selected source's outcome first ("Selected result
|
|
94
|
+
passed; another scoped source failed"), never the combined status; the combined status is labelled as the worst current
|
|
95
|
+
result. Each other current source is one click away, and sources that share an environment are told apart by status,
|
|
96
|
+
time, revision and their place among them. A failure leads with whether the behavior was reached (decided only by the
|
|
97
|
+
recorded step `kind`: a `hook`, `setup`, `fixture`, `before*` or `given` step failing is "Behavior not verified: setup
|
|
98
|
+
failed", `when`, `then`, `assertion` or `expect` is "Expectation not met", anything else is unknown), then the expected
|
|
99
|
+
and observed values and the closest step and video, then the original output. The cause (product, test harness,
|
|
100
|
+
environment) is unknown unless the producer declared it under the `atlas.failure` extension; it is never read from error
|
|
101
|
+
wording. Records of one source whose outcome changed are listed oldest first, and a later pass is never called a fix.
|
|
102
|
+
In the explorer the chosen source is the `source` search parameter (a layer id; unlike `layer` it does not pin a run).
|
|
103
|
+
|
|
90
104
|
## Icons and notation marks
|
|
91
105
|
|
|
92
106
|
Use named imports from `@tabler/icons-react` for all UI glyphs. The icon package supplies SVG paths;
|
package/llms-full.txt
CHANGED
|
@@ -2900,6 +2900,23 @@ every kind of evidence and values from the future) and `layer-extensibility.test
|
|
|
2900
2900
|
fields survive at every level, malformed evidence is dropped without losing the result, and a reader of the
|
|
2901
2901
|
first version of the format still reads it all.
|
|
2902
2902
|
|
|
2903
|
+
### Saying what failed
|
|
2904
|
+
|
|
2905
|
+
A failed check or step may declare the expectation it broke under the `atlas.failure` extension, so a reviewer sees
|
|
2906
|
+
it before the raw log. Every field is optional, and the declaration never changes the result.
|
|
2907
|
+
|
|
2908
|
+
```json
|
|
2909
|
+
{
|
|
2910
|
+
"name": "The list includes tea", "kind": "then", "status": "failed",
|
|
2911
|
+
"extensions": { "atlas.failure": { "expected": "tea in the list", "actual": "the list is empty", "attribution": "product" } }
|
|
2912
|
+
}
|
|
2913
|
+
```
|
|
2914
|
+
|
|
2915
|
+
`expected` and `actual` are shown as written. `attribution` is `product`, `test-harness` or `environment`; leave it out
|
|
2916
|
+
when you do not know, and the viewer says the cause is unknown. It is never inferred from the message. Whether setup
|
|
2917
|
+
failed before the behavior started comes from the failed step's `kind`: `hook`, `setup`, `fixture`, `before`,
|
|
2918
|
+
`beforeAll`, `beforeEach` or `given` mean preparation; `when`, `then`, `assertion` or `expect` mean the behavior.
|
|
2919
|
+
|
|
2903
2920
|
## Attach evidence
|
|
2904
2921
|
|
|
2905
2922
|
### From the command line
|
|
@@ -3919,6 +3936,20 @@ when Watch is on screen.
|
|
|
3919
3936
|
- The spec index searches titles, paths and declared domains, and can group by the domain from spec metadata (specs with
|
|
3920
3937
|
none are listed under "No domain declared"; the Tests view reports domain problems).
|
|
3921
3938
|
|
|
3939
|
+
## Outcome summary
|
|
3940
|
+
|
|
3941
|
+
A test or behavior detail states what the **selected** result says apart from what the other current sources say
|
|
3942
|
+
(`evidence/outcome.ts`, `outcome-summary.tsx`). The heading is the selected source's outcome first ("Selected result
|
|
3943
|
+
passed; another scoped source failed"), never the combined status; the combined status is labelled as the worst current
|
|
3944
|
+
result. Each other current source is one click away, and sources that share an environment are told apart by status,
|
|
3945
|
+
time, revision and their place among them. A failure leads with whether the behavior was reached (decided only by the
|
|
3946
|
+
recorded step `kind`: a `hook`, `setup`, `fixture`, `before*` or `given` step failing is "Behavior not verified: setup
|
|
3947
|
+
failed", `when`, `then`, `assertion` or `expect` is "Expectation not met", anything else is unknown), then the expected
|
|
3948
|
+
and observed values and the closest step and video, then the original output. The cause (product, test harness,
|
|
3949
|
+
environment) is unknown unless the producer declared it under the `atlas.failure` extension; it is never read from error
|
|
3950
|
+
wording. Records of one source whose outcome changed are listed oldest first, and a later pass is never called a fix.
|
|
3951
|
+
In the explorer the chosen source is the `source` search parameter (a layer id; unlike `layer` it does not pin a run).
|
|
3952
|
+
|
|
3922
3953
|
## Icons and notation marks
|
|
3923
3954
|
|
|
3924
3955
|
Use named imports from `@tabler/icons-react` for all UI glyphs. The icon package supplies SVG paths;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crvouga/atlas",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.0",
|
|
4
4
|
"description": "Atlas: specs of any kind, reported as layered evidence. Publish checks from any runner (JUnit, CTRF, a script), combine them across nodes, branches, environments and backends, gate CI on the result, and run statecharts against the real app.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -6,12 +6,9 @@
|
|
|
6
6
|
* record: run, environment, source, attempts). Also the coverage summary of the whole feature. A recording is always
|
|
7
7
|
* labelled as illustrative evidence with its own status, provenance and revision; it never stands for approval.
|
|
8
8
|
*/
|
|
9
|
-
import type { EvidenceHolder } from '@crvouga/atlas/reports/evidence-detail';
|
|
10
|
-
import { isVideoAttachment } from '@crvouga/atlas/reports/evidence-detail';
|
|
11
9
|
import { effectiveScope } from '@crvouga/atlas/reports/evidence-scope';
|
|
12
10
|
import type { EvidenceResult } from '@crvouga/atlas/reports/layer-merge';
|
|
13
11
|
import { environmentLabel } from '@crvouga/atlas/reports/layer-axes';
|
|
14
|
-
import type { Attachment } from '@crvouga/atlas/schema';
|
|
15
12
|
import { Alert, Badge, Box, Group, List, Paper, Stack, Tabs, Text, Title } from '@mantine/core';
|
|
16
13
|
import { useMemo, type ReactNode } from 'react';
|
|
17
14
|
|
|
@@ -23,6 +20,7 @@ import { ScopeNotice } from './evidence-scope';
|
|
|
23
20
|
import { plural, statusWord, useNow } from './format';
|
|
24
21
|
import { exampleIndexFor, exampleRows, fillExample, type FeatureStep } from './gherkin';
|
|
25
22
|
import { Origin, StatusMark } from './marks';
|
|
23
|
+
import { outcomeHeadline, videosIn } from './outcome';
|
|
26
24
|
|
|
27
25
|
/** The short, readable form of a spec fingerprint (`<length>:<hash>:<hash>`): its length and the start of its hash. */
|
|
28
26
|
export function shortRevision(fingerprint: string): string {
|
|
@@ -80,19 +78,6 @@ function expectedSteps(steps: readonly FeatureStep[]): FeatureStep[] {
|
|
|
80
78
|
return at < 0 ? [] : steps.slice(at);
|
|
81
79
|
}
|
|
82
80
|
|
|
83
|
-
function videosIn(holder: EvidenceHolder | undefined): Attachment[] {
|
|
84
|
-
if (!holder) return [];
|
|
85
|
-
const own = (holder.attachments ?? []).filter(isVideoAttachment);
|
|
86
|
-
return [
|
|
87
|
-
...own,
|
|
88
|
-
...(holder.steps ?? []).flatMap((step) => (step.attachments ?? []).filter(isVideoAttachment)),
|
|
89
|
-
...(holder.attempts ?? []).flatMap((attempt) => [
|
|
90
|
-
...(attempt.attachments ?? []).filter(isVideoAttachment),
|
|
91
|
-
...(attempt.steps ?? []).flatMap((step) => (step.attachments ?? []).filter(isVideoAttachment))
|
|
92
|
-
])
|
|
93
|
-
];
|
|
94
|
-
}
|
|
95
|
-
|
|
96
81
|
/** The recording of one result, read only when this is on screen. */
|
|
97
82
|
function Player({ result }: { result: EvidenceResult }) {
|
|
98
83
|
const detail = useCheckDetail(result.layer, result.row);
|
|
@@ -171,6 +156,13 @@ export function BehaviorPanel({
|
|
|
171
156
|
const steps = useMemo(() => (entry ? fillExample(scenario.steps, entry) : scenario.steps), [scenario, entry]);
|
|
172
157
|
const expected = expectedSteps(steps);
|
|
173
158
|
const named = example !== null && entry === null;
|
|
159
|
+
// The coverage state is the check's combined status; this says what the recording or result shown here recorded, apart from it.
|
|
160
|
+
const outcome =
|
|
161
|
+
subject && behavior.slot ? (
|
|
162
|
+
<Text component="div" fw={600} data-outcome-headline>
|
|
163
|
+
{outcomeHeadline({ layer: subject.layer, row: subject.row }, behavior.slot).text}
|
|
164
|
+
</Text>
|
|
165
|
+
) : null;
|
|
174
166
|
return (
|
|
175
167
|
<Tabs
|
|
176
168
|
defaultValue={start ?? (demo ? 'watch' : state === 'running' || behavior.slot?.live ? 'inspect' : 'read')}
|
|
@@ -190,6 +182,7 @@ export function BehaviorPanel({
|
|
|
190
182
|
{COVERAGE[state].meaning}
|
|
191
183
|
</Text>
|
|
192
184
|
</Group>
|
|
185
|
+
{outcome}
|
|
193
186
|
{example !== null && (
|
|
194
187
|
<Text component="div" c="dimmed">
|
|
195
188
|
Example: {example}
|
|
@@ -209,6 +202,7 @@ export function BehaviorPanel({
|
|
|
209
202
|
<CoverageBadge state={state} />
|
|
210
203
|
{behavior.representative && <Badge variant="outline">Representative demo</Badge>}
|
|
211
204
|
</Group>
|
|
205
|
+
{outcome}
|
|
212
206
|
{expected.length > 0 && (
|
|
213
207
|
<Box component="section" aria-label="Expected outcome">
|
|
214
208
|
<Title order={4} mb="xs">
|
|
@@ -9,12 +9,14 @@ import { useQuery } from '@tanstack/react-query';
|
|
|
9
9
|
import { useMemo, useState, type ReactNode } from 'react';
|
|
10
10
|
import { TestProgress, useTestProgress } from '../live/test-progress';
|
|
11
11
|
import { reportJson } from './api';
|
|
12
|
-
import { ResultEvidence } from './artifacts';
|
|
12
|
+
import { ResultEvidence, useCheckDetail } from './artifacts';
|
|
13
13
|
import { useEvidence } from './evidence-context';
|
|
14
14
|
import { ago, duration, plural, statusWord, useNow } from './format';
|
|
15
15
|
import type { FeatureExamples, FeatureStep } from './gherkin';
|
|
16
16
|
import { Bounded, Origin, StatusMark } from './marks';
|
|
17
17
|
import { ModelTrace } from './model-trace';
|
|
18
|
+
import { OutcomeSummary } from './outcome-summary';
|
|
19
|
+
import { failedReportStep, failedStep, recordOf, sourceLabels, type ResultRecord } from './outcome';
|
|
18
20
|
import { recordedModelStates } from './quint';
|
|
19
21
|
/** What the address says is open: a check, and optionally one particular layer's record of it. */
|
|
20
22
|
export type CheckSelection = {
|
|
@@ -49,13 +51,6 @@ function environmentMarks(slot: CheckSlot) {
|
|
|
49
51
|
if (seen.size < 2) return [];
|
|
50
52
|
return [...seen].map(([id, status]) => ({ id, status }));
|
|
51
53
|
}
|
|
52
|
-
function outcomeName(outcome: EvidenceResult) {
|
|
53
|
-
const environment = outcome.layer.environment ? environmentLabel(outcome.layer.environment) : '';
|
|
54
|
-
const parameters = Object.entries(outcome.parameters)
|
|
55
|
-
.map(([name, value]) => `${name}: ${value}`)
|
|
56
|
-
.join(', ');
|
|
57
|
-
return [environment, parameters].filter(Boolean).join(' · ') || outcome.layer.producer;
|
|
58
|
-
}
|
|
59
54
|
export function ResultRow({
|
|
60
55
|
label,
|
|
61
56
|
slot,
|
|
@@ -93,7 +88,7 @@ export function ResultRow({
|
|
|
93
88
|
<Flex component="span" display="inline-flex" gap="xs" align="center" wrap="wrap">
|
|
94
89
|
{marks.map((mark) => (
|
|
95
90
|
<Text key={mark.id || 'unspecified'} data-status={mark.status} span inherit>
|
|
96
|
-
{environmentLabel(mark.id || null)}
|
|
91
|
+
{environmentLabel(mark.id || null)} {statusWord(mark.status).toLowerCase()}
|
|
97
92
|
</Text>
|
|
98
93
|
))}
|
|
99
94
|
</Flex>
|
|
@@ -276,6 +271,14 @@ type Props = {
|
|
|
276
271
|
pinned?: HistoryEntry;
|
|
277
272
|
/** Choosing a run in the history always names its layer, even when that result is a current one, so the choice stays pinned. */
|
|
278
273
|
pin?: boolean;
|
|
274
|
+
/** The layer id of the current source the address selected (without pinning a run). Used with `onSource`; otherwise the choice is local. */
|
|
275
|
+
source?: string;
|
|
276
|
+
/** Called with the layer id when a source is chosen (null for the newest), so the page can keep the choice in its address. */
|
|
277
|
+
onSource?: (layerId: string | null) => void;
|
|
278
|
+
/** Whether the outcome summary states its own heading. False when the page already shows it. */
|
|
279
|
+
headline?: boolean;
|
|
280
|
+
/** Every record of this check the page knows (a node's history of it), when the layers held here are only the winners. */
|
|
281
|
+
records?: readonly ResultRecord[];
|
|
279
282
|
};
|
|
280
283
|
/** Everything recorded for one check: the chosen result, how it went, and every other run of it. */
|
|
281
284
|
export function CheckDetail(props: Props) {
|
|
@@ -306,16 +309,28 @@ export function CheckDetail(props: Props) {
|
|
|
306
309
|
</>
|
|
307
310
|
);
|
|
308
311
|
}
|
|
309
|
-
function RecordedCheckDetail({ slot, layerId, select, reads, written, pinned, pin }: Props) {
|
|
312
|
+
function RecordedCheckDetail({ slot, layerId, select, reads, written, pinned, pin, source, onSource, headline = true, records }: Props) {
|
|
310
313
|
const { evidence, backends, ready, pending } = useEvidence();
|
|
311
314
|
const now = useNow();
|
|
312
315
|
const [variant, setVariant] = useState(0);
|
|
313
316
|
const history = useMemo(() => checkHistory(evidence, slot.specPath, slot.id), [evidence, slot.specPath, slot.id]);
|
|
314
|
-
const
|
|
317
|
+
const sourceAt = onSource && source ? slot.outcomes.findIndex((outcome) => outcome.layer.id === source) : -1;
|
|
318
|
+
const index = Math.min(onSource ? Math.max(sourceAt, 0) : variant, slot.outcomes.length - 1);
|
|
319
|
+
const winner = slot.outcomes[index]!;
|
|
315
320
|
const earlier = pinned ?? (layerId ? history.find((entry) => entry.layer.id === layerId) : undefined);
|
|
316
321
|
const chosen: HistoryEntry = earlier ?? { layer: winner.layer, row: winner.row };
|
|
317
322
|
const superseded = !slot.outcomes.some((outcome) => outcome.layer.id === chosen.layer.id);
|
|
318
323
|
const { paths, report } = useRecordedReport(chosen);
|
|
324
|
+
const recorded = useCheckDetail(chosen.layer, chosen.row);
|
|
325
|
+
const point = useMemo(
|
|
326
|
+
() => (chosen.row.status === 'failed' ? (failedStep(recorded.data?.check.steps) ?? failedReportStep(report.data)) : undefined),
|
|
327
|
+
[chosen.row.status, recorded.data, report.data]
|
|
328
|
+
);
|
|
329
|
+
const labels = sourceLabels(slot.outcomes, now);
|
|
330
|
+
const choose = (at: number) => {
|
|
331
|
+
setVariant(at);
|
|
332
|
+
onSource?.(at === 0 ? null : slot.outcomes[at]!.layer.id);
|
|
333
|
+
};
|
|
319
334
|
const moments = report.data ? flattenFrames(report.data.steps).length : 0;
|
|
320
335
|
const timings = superseded ? [] : slot.diagnostics.filter((entry) => entry.layer.id === chosen.layer.id || slot.diagnostics.length === 1);
|
|
321
336
|
const note = superseded ? 'An earlier result. A newer run replaced it in the combined report.' : qualification(winner.warning);
|
|
@@ -328,12 +343,32 @@ function RecordedCheckDetail({ slot, layerId, select, reads, written, pinned, pi
|
|
|
328
343
|
: '';
|
|
329
344
|
return (
|
|
330
345
|
<Stack gap="sm" miw={0}>
|
|
346
|
+
<OutcomeSummary
|
|
347
|
+
slot={slot}
|
|
348
|
+
selected={chosen}
|
|
349
|
+
history={records && records.length > history.length ? records : history.map(recordOf)}
|
|
350
|
+
point={point}
|
|
351
|
+
detail={recorded.data}
|
|
352
|
+
now={now}
|
|
353
|
+
showHeadline={headline}
|
|
354
|
+
choose={(layer) => {
|
|
355
|
+
const at = slot.outcomes.findIndex((outcome) => outcome.layer.id === layer);
|
|
356
|
+
if (at >= 0) choose(at);
|
|
357
|
+
else select({ layer });
|
|
358
|
+
}}
|
|
359
|
+
/>
|
|
331
360
|
{slot.outcomes.length > 1 && !earlier && (
|
|
332
361
|
<Flex role="group" aria-label="Environments" gap="xs" align="center" wrap="wrap">
|
|
333
|
-
{slot.outcomes.map((outcome,
|
|
334
|
-
<Button
|
|
362
|
+
{slot.outcomes.map((outcome, at) => (
|
|
363
|
+
<Button
|
|
364
|
+
key={outcome.key}
|
|
365
|
+
type="button"
|
|
366
|
+
aria-pressed={at === index}
|
|
367
|
+
onClick={() => choose(at)}
|
|
368
|
+
variant="light"
|
|
369
|
+
>
|
|
335
370
|
<StatusMark status={outcome.row.status} size={12} />
|
|
336
|
-
{
|
|
371
|
+
{labels[at]!.text}
|
|
337
372
|
</Button>
|
|
338
373
|
))}
|
|
339
374
|
</Flex>
|
|
@@ -410,7 +445,7 @@ function RecordedCheckDetail({ slot, layerId, select, reads, written, pinned, pi
|
|
|
410
445
|
{moments > 0 && (
|
|
411
446
|
<Button
|
|
412
447
|
type="button"
|
|
413
|
-
onClick={() => select({ view: 'recording', check: slot.id, layer: earlier ||
|
|
448
|
+
onClick={() => select({ view: 'recording', check: slot.id, layer: earlier || index > 0 ? chosen.layer.id : null })}
|
|
414
449
|
variant="filled"
|
|
415
450
|
>
|
|
416
451
|
Open recording{' '}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Outcome summary
|
|
3
|
+
*
|
|
4
|
+
* What the selected result says, stated apart from the other current sources, and the failure explained before its
|
|
5
|
+
* raw output: the heading names the selected source's outcome and what the other scoped sources recorded (each one
|
|
6
|
+
* click away), whether the behavior was reached, the recorded expected and observed values, the closest video and
|
|
7
|
+
* step, the chronology of the source, and who is to blame, which stays unknown unless the producer declared it. The
|
|
8
|
+
* original diagnostics remain below, untouched.
|
|
9
|
+
*/
|
|
10
|
+
import type { CheckSlot, HistoryEntry } from '@crvouga/atlas/reports/evidence-view';
|
|
11
|
+
import type { CheckDetail } from '@crvouga/atlas/reports/evidence-detail';
|
|
12
|
+
import { effectiveScope, scopeBadges } from '@crvouga/atlas/reports/evidence-scope';
|
|
13
|
+
import { Badge, Box, Button, Flex, List, Paper, Stack, Text, Title } from '@mantine/core';
|
|
14
|
+
|
|
15
|
+
import { ago, duration, shortSha, statusWord } from './format';
|
|
16
|
+
import { StatusMark } from './marks';
|
|
17
|
+
import {
|
|
18
|
+
chronology,
|
|
19
|
+
failureNote,
|
|
20
|
+
outcomeHeadline,
|
|
21
|
+
passedAfterFailure,
|
|
22
|
+
recordOf,
|
|
23
|
+
type ResultRecord,
|
|
24
|
+
reachOf,
|
|
25
|
+
sourceLabels,
|
|
26
|
+
videosIn,
|
|
27
|
+
type FailurePoint
|
|
28
|
+
} from './outcome';
|
|
29
|
+
|
|
30
|
+
const BLAME: Record<string, string> = {
|
|
31
|
+
product: 'the product',
|
|
32
|
+
'test-harness': 'the test harness',
|
|
33
|
+
environment: 'the environment'
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The outcome of the selected result in words, the other current sources beside it, and, for a failure, where and
|
|
38
|
+
* how it failed. `point` is the failed step found in the recording (undefined when none was recorded); `detail` is the
|
|
39
|
+
* check's full record, for its videos and scope. `choose` selects one of the other current sources; `showHeadline`
|
|
40
|
+
* is false when the page already shows the heading.
|
|
41
|
+
*/
|
|
42
|
+
export function OutcomeSummary({
|
|
43
|
+
slot,
|
|
44
|
+
selected,
|
|
45
|
+
history,
|
|
46
|
+
point,
|
|
47
|
+
detail,
|
|
48
|
+
now,
|
|
49
|
+
choose,
|
|
50
|
+
showHeadline = true
|
|
51
|
+
}: {
|
|
52
|
+
slot: CheckSlot;
|
|
53
|
+
selected: HistoryEntry;
|
|
54
|
+
history: readonly ResultRecord[];
|
|
55
|
+
point: FailurePoint | undefined;
|
|
56
|
+
detail: CheckDetail | undefined;
|
|
57
|
+
now: number;
|
|
58
|
+
choose: (layerId: string) => void;
|
|
59
|
+
showHeadline?: boolean;
|
|
60
|
+
}) {
|
|
61
|
+
const headline = outcomeHeadline(selected, slot);
|
|
62
|
+
const failed = selected.row.status === 'failed';
|
|
63
|
+
const reach = failed ? reachOf(point) : undefined;
|
|
64
|
+
const note = point?.note ?? failureNote(detail?.check.extensions);
|
|
65
|
+
const every = sourceLabels(slot.outcomes, now);
|
|
66
|
+
const labels = headline.others.map((outcome) => every[slot.outcomes.indexOf(outcome)]!);
|
|
67
|
+
const video = point?.videos[0] ?? videosIn(detail?.check)[0];
|
|
68
|
+
const scope = video ? effectiveScope(video.scope, detail?.check.scope, detail?.inheritedScope) : undefined;
|
|
69
|
+
const timeline = chronology(recordOf(selected), history);
|
|
70
|
+
return (
|
|
71
|
+
<Paper component="section" aria-label="Outcome summary" data-outcome={selected.row.status} withBorder p="md">
|
|
72
|
+
<Stack gap="sm" miw={0}>
|
|
73
|
+
{showHeadline && (
|
|
74
|
+
<Title order={3} data-outcome-headline>
|
|
75
|
+
{headline.text}
|
|
76
|
+
</Title>
|
|
77
|
+
)}
|
|
78
|
+
{headline.others.length > 0 && (
|
|
79
|
+
<Stack gap="xs" component="section" aria-label="Other current sources">
|
|
80
|
+
<Text component="div" c="dimmed">
|
|
81
|
+
Other current sources of this check
|
|
82
|
+
</Text>
|
|
83
|
+
<List listStyleType="none" spacing="xs">
|
|
84
|
+
{headline.others.map((outcome, index) => (
|
|
85
|
+
<List.Item key={outcome.key}>
|
|
86
|
+
<Button type="button" variant="light" onClick={() => choose(outcome.layer.id)} aria-label={`Open ${labels[index]!.text}`}>
|
|
87
|
+
<StatusMark status={outcome.row.status} size={12} />
|
|
88
|
+
<Text span inherit>
|
|
89
|
+
{labels[index]!.text}
|
|
90
|
+
</Text>
|
|
91
|
+
</Button>
|
|
92
|
+
</List.Item>
|
|
93
|
+
))}
|
|
94
|
+
</List>
|
|
95
|
+
</Stack>
|
|
96
|
+
)}
|
|
97
|
+
{reach && (
|
|
98
|
+
<Flex gap="xs" align="center" wrap="wrap" data-reach={reach.state}>
|
|
99
|
+
<Badge variant={reach.state === 'setup-failed' ? 'filled' : 'light'} color={reach.state === 'unknown' ? 'gray' : 'red'}>
|
|
100
|
+
{reach.state === 'setup-failed'
|
|
101
|
+
? 'Behavior not verified'
|
|
102
|
+
: reach.state === 'expectation-failed'
|
|
103
|
+
? 'Expectation not met'
|
|
104
|
+
: 'Failure point unknown'}
|
|
105
|
+
</Badge>
|
|
106
|
+
<Text span inherit>
|
|
107
|
+
{reach.text}
|
|
108
|
+
</Text>
|
|
109
|
+
</Flex>
|
|
110
|
+
)}
|
|
111
|
+
{failed && (note?.expected !== undefined || note?.actual !== undefined) && (
|
|
112
|
+
<Box component="dl" aria-label="Expected and observed" m={0}>
|
|
113
|
+
{note?.expected !== undefined && (
|
|
114
|
+
<>
|
|
115
|
+
<Text component="dt" fw={600}>
|
|
116
|
+
Expected
|
|
117
|
+
</Text>
|
|
118
|
+
<Text component="dd" m={0}>
|
|
119
|
+
{note.expected}
|
|
120
|
+
</Text>
|
|
121
|
+
</>
|
|
122
|
+
)}
|
|
123
|
+
{note?.actual !== undefined && (
|
|
124
|
+
<>
|
|
125
|
+
<Text component="dt" fw={600}>
|
|
126
|
+
Observed
|
|
127
|
+
</Text>
|
|
128
|
+
<Text component="dd" m={0}>
|
|
129
|
+
{note.actual}
|
|
130
|
+
</Text>
|
|
131
|
+
</>
|
|
132
|
+
)}
|
|
133
|
+
</Box>
|
|
134
|
+
)}
|
|
135
|
+
{failed && (point || video) && (
|
|
136
|
+
<Stack gap="xs" component="section" aria-label="Closest evidence">
|
|
137
|
+
{point && (
|
|
138
|
+
<Text component="div">
|
|
139
|
+
Closest step:{' '}
|
|
140
|
+
<Text span inherit fw={600}>
|
|
141
|
+
{point.name}
|
|
142
|
+
</Text>
|
|
143
|
+
{point.offsetMs !== undefined && (
|
|
144
|
+
<Text span inherit c="dimmed">
|
|
145
|
+
{' '}
|
|
146
|
+
at {duration(point.offsetMs) || '0 ms'} into the check
|
|
147
|
+
</Text>
|
|
148
|
+
)}
|
|
149
|
+
</Text>
|
|
150
|
+
)}
|
|
151
|
+
{video && (
|
|
152
|
+
<Text component="div">
|
|
153
|
+
Closest video:{' '}
|
|
154
|
+
<Text span inherit fw={600}>
|
|
155
|
+
{video.description ?? video.name}
|
|
156
|
+
</Text>
|
|
157
|
+
<Text span inherit c="dimmed">
|
|
158
|
+
{' '}
|
|
159
|
+
{point?.videos.length ? 'attached to that step' : 'recorded for the whole check'}
|
|
160
|
+
{' · '}
|
|
161
|
+
{scopeBadges(scope)
|
|
162
|
+
.map((badge) => badge.text)
|
|
163
|
+
.join(', ')}
|
|
164
|
+
</Text>
|
|
165
|
+
</Text>
|
|
166
|
+
)}
|
|
167
|
+
</Stack>
|
|
168
|
+
)}
|
|
169
|
+
{failed && (
|
|
170
|
+
<Text component="div" c="dimmed" data-attribution={note?.attribution ?? 'unknown'}>
|
|
171
|
+
Cause:{' '}
|
|
172
|
+
{note?.attribution
|
|
173
|
+
? `${BLAME[note.attribution]}, as declared by the producer`
|
|
174
|
+
: 'unknown. Nothing recorded says whether the product, the test harness or the environment is at fault.'}
|
|
175
|
+
</Text>
|
|
176
|
+
)}
|
|
177
|
+
{timeline.length > 0 && (
|
|
178
|
+
<Stack gap="xs" component="section" aria-label="Chronology of this source">
|
|
179
|
+
<Text component="div" c="dimmed">
|
|
180
|
+
Records of this source, oldest first
|
|
181
|
+
</Text>
|
|
182
|
+
<List listStyleType="none" spacing="xs" type="ordered">
|
|
183
|
+
{timeline.map(({ record, selected: current }) => (
|
|
184
|
+
<List.Item key={`${record.layer}|${record.kind}`} aria-current={current || undefined} data-status={record.status}>
|
|
185
|
+
<Flex gap="xs" align="center" wrap="wrap">
|
|
186
|
+
<StatusMark status={record.status} size={12} />
|
|
187
|
+
<Text span inherit fw={current ? 600 : undefined}>
|
|
188
|
+
{statusWord(record.status)}
|
|
189
|
+
</Text>
|
|
190
|
+
<Text span inherit c="dimmed">
|
|
191
|
+
{ago(record.at, now)} · {shortSha(record.sha) ?? 'revision not recorded'}
|
|
192
|
+
{current ? ' · selected' : ''}
|
|
193
|
+
</Text>
|
|
194
|
+
</Flex>
|
|
195
|
+
</List.Item>
|
|
196
|
+
))}
|
|
197
|
+
</List>
|
|
198
|
+
{passedAfterFailure(timeline) && (
|
|
199
|
+
<Text component="div" role="status">
|
|
200
|
+
The latest run passed after an earlier failure. Nothing recorded shows that the failure was fixed or why it passed.
|
|
201
|
+
</Text>
|
|
202
|
+
)}
|
|
203
|
+
</Stack>
|
|
204
|
+
)}
|
|
205
|
+
</Stack>
|
|
206
|
+
</Paper>
|
|
207
|
+
);
|
|
208
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import type { LayerHeader, LayerRow } from '@crvouga/atlas/reports/evidence-types';
|
|
2
|
+
import type { CheckSlot } from '@crvouga/atlas/reports/evidence-view';
|
|
3
|
+
import type { EvidenceResult } from '@crvouga/atlas/reports/layer-merge';
|
|
4
|
+
import { expect, test } from 'bun:test';
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
chronology,
|
|
8
|
+
failedStep,
|
|
9
|
+
failureNote,
|
|
10
|
+
outcomeHeadline,
|
|
11
|
+
passedAfterFailure,
|
|
12
|
+
phaseOf,
|
|
13
|
+
reachOf,
|
|
14
|
+
recordOf,
|
|
15
|
+
selectedResult,
|
|
16
|
+
sourceLabels,
|
|
17
|
+
type ResultRecord
|
|
18
|
+
} from './outcome';
|
|
19
|
+
|
|
20
|
+
const header = (id: string, at: number, extra: Partial<LayerHeader> = {}): LayerHeader => ({
|
|
21
|
+
id, backend: 'b', producer: 'p', packageDir: null, recordedAtMs: at, sha: null, branch: null, environment: 'desktop-web', runUrl: null,
|
|
22
|
+
parameters: {}, recording: null, live: false, counts: { passed: 0, failed: 0, running: 0, 'not-run': 0 }, execution: id, ...extra
|
|
23
|
+
});
|
|
24
|
+
const row = (status: LayerRow['status']): LayerRow => ({
|
|
25
|
+
kind: 'checks', id: 'x', title: 'X', status, specPath: null, detail: '', measurements: [], testId: null, variant: {}, durationMs: null
|
|
26
|
+
});
|
|
27
|
+
const result = (id: string, status: LayerRow['status'], extra: Partial<LayerHeader> = {}): EvidenceResult => ({
|
|
28
|
+
key: id, slot: 's', layer: header(id, 1, extra), parameters: {}, warning: null, row: row(status)
|
|
29
|
+
});
|
|
30
|
+
const slot = (status: CheckSlot['status'], outcomes: EvidenceResult[]): CheckSlot => ({
|
|
31
|
+
key: 's', id: 'x', title: 'X', specPath: null, status, outcomes, diagnostics: [], latestAtMs: 1, live: false
|
|
32
|
+
});
|
|
33
|
+
const entry = (r: EvidenceResult) => ({ layer: r.layer, row: r.row });
|
|
34
|
+
|
|
35
|
+
test('the heading names the selected outcome before the other sources, and never reads it as the combined status', () => {
|
|
36
|
+
const passed = result('a', 'passed');
|
|
37
|
+
const failed = result('b', 'failed', { environment: 'api' });
|
|
38
|
+
const both = slot('failed', [passed, failed]);
|
|
39
|
+
expect(outcomeHeadline(entry(passed), both).text).toBe('Selected result passed; another scoped source failed');
|
|
40
|
+
expect(outcomeHeadline(entry(failed), both).text).toBe('Selected result failed; the other scoped source: 1 passed');
|
|
41
|
+
expect(outcomeHeadline(entry(passed), slot('passed', [passed])).text).toBe('Selected result passed');
|
|
42
|
+
expect(outcomeHeadline(entry(result('c', 'running')), slot('running', [result('c', 'running')])).text).toBe('Selected result is still running');
|
|
43
|
+
expect(outcomeHeadline(entry(passed), slot('failed', [passed, failed, result('d', 'failed')])).text).toContain('2 other scoped sources failed');
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test('an earlier result says so and names what the combined report now shows', () => {
|
|
47
|
+
const current = result('new', 'passed');
|
|
48
|
+
const old = result('old', 'failed');
|
|
49
|
+
const headline = outcomeHeadline(entry(old), slot('passed', [current]));
|
|
50
|
+
expect(headline.current).toBe(false);
|
|
51
|
+
expect(headline.text).toBe('This earlier result failed; the combined report now shows passed across its current source');
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test('the selected result is the pinned run, else the named source, else the newest', () => {
|
|
55
|
+
const a = result('a', 'passed');
|
|
56
|
+
const b = result('b', 'failed');
|
|
57
|
+
const both = slot('failed', [a, b]);
|
|
58
|
+
expect(selectedResult(both)?.layer.id).toBe('a');
|
|
59
|
+
expect(selectedResult(both, undefined, 'b')?.layer.id).toBe('b');
|
|
60
|
+
expect(selectedResult(both, undefined, 'unknown')?.layer.id).toBe('a');
|
|
61
|
+
expect(selectedResult(both, entry(b), 'a')?.layer.id).toBe('b');
|
|
62
|
+
expect(selectedResult(slot('passed', []))).toBeUndefined();
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test('sources that share a name are told apart by status, revision and their place among them', () => {
|
|
66
|
+
const labels = sourceLabels(
|
|
67
|
+
[result('a', 'passed', { sha: 'b'.repeat(40), producer: 'one' }), result('b', 'failed', { sha: 'a'.repeat(40), producer: 'two' }), result('c', 'passed', { environment: 'api' })],
|
|
68
|
+
1_000
|
|
69
|
+
);
|
|
70
|
+
expect(labels.map((label) => label.text)).toEqual([
|
|
71
|
+
'Desktop web · Passed · just now · bbbbbbb · result 1 of 2 from one',
|
|
72
|
+
'Desktop web · Failed · just now · aaaaaaa · result 2 of 2 from two',
|
|
73
|
+
'Api · Passed · just now · revision not recorded'
|
|
74
|
+
]);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('the phase comes from the recorded kind, never from the wording', () => {
|
|
78
|
+
expect(phaseOf('hook')).toBe('setup');
|
|
79
|
+
expect(phaseOf('Given')).toBe('setup');
|
|
80
|
+
expect(phaseOf('then')).toBe('behavior');
|
|
81
|
+
expect(phaseOf('action')).toBe('unspecified');
|
|
82
|
+
expect(phaseOf(undefined)).toBe('unspecified');
|
|
83
|
+
const point = failedStep([{ name: 'setup failed to connect', status: 'failed', offsetMs: 0 }]);
|
|
84
|
+
expect(point?.phase).toBe('unspecified');
|
|
85
|
+
expect(reachOf(point).state).toBe('unknown');
|
|
86
|
+
expect(reachOf(undefined).state).toBe('unknown');
|
|
87
|
+
expect(reachOf(failedStep([{ name: 'Seed', kind: 'hook', status: 'failed' }])).state).toBe('setup-failed');
|
|
88
|
+
expect(reachOf(failedStep([{ name: 'Check', kind: 'assertion', status: 'failed' }])).state).toBe('expectation-failed');
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test('the failed step is the first innermost one in time order', () => {
|
|
92
|
+
const point = failedStep([
|
|
93
|
+
{ id: 'late', name: 'Late', status: 'failed', offsetMs: 900 },
|
|
94
|
+
{ id: 'group', name: 'Group', status: 'failed', offsetMs: 100 },
|
|
95
|
+
{ id: 'leaf', parentId: 'group', name: 'Leaf', status: 'failed', offsetMs: 150, detail: 'nope' },
|
|
96
|
+
{ name: 'Fine', status: 'passed', offsetMs: 0 }
|
|
97
|
+
]);
|
|
98
|
+
expect(point?.name).toBe('Leaf');
|
|
99
|
+
expect(point?.detail).toBe('nope');
|
|
100
|
+
expect(failedStep([{ name: 'ok', status: 'passed' }])).toBeUndefined();
|
|
101
|
+
expect(failedStep(undefined)).toBeUndefined();
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('only a producer-declared failure is read, and only a known attribution counts', () => {
|
|
105
|
+
expect(failureNote({ 'atlas.failure': { expected: 200, actual: { status: 500 }, attribution: 'environment' } })).toEqual({
|
|
106
|
+
expected: '200',
|
|
107
|
+
actual: '{"status":500}',
|
|
108
|
+
attribution: 'environment'
|
|
109
|
+
});
|
|
110
|
+
expect(failureNote({ 'atlas.failure': { attribution: 'the intern' } })).toBeUndefined();
|
|
111
|
+
expect(failureNote({ other: {} })).toBeUndefined();
|
|
112
|
+
expect(failureNote(undefined)).toBeUndefined();
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('a chronology is only told for one source whose outcome changed, oldest first, without declaring a cause', () => {
|
|
116
|
+
const record = (layer: string, status: ResultRecord['status'], at: number, env: string | null = 'desktop-web'): ResultRecord => ({
|
|
117
|
+
...recordOf(entry(result(layer, status, { environment: env, recordedAtMs: at }))),
|
|
118
|
+
at
|
|
119
|
+
});
|
|
120
|
+
const early = record('1', 'failed', 100);
|
|
121
|
+
const late = record('2', 'passed', 200);
|
|
122
|
+
const timeline = chronology(late, [late, early, record('3', 'passed', 150, 'api')]);
|
|
123
|
+
expect(timeline.map((each) => each.record.layer)).toEqual(['1', '2']);
|
|
124
|
+
expect(timeline[1]!.selected).toBe(true);
|
|
125
|
+
expect(passedAfterFailure(timeline)).toBe(true);
|
|
126
|
+
expect(passedAfterFailure(chronology(early, [early, late]).slice().reverse())).toBe(false);
|
|
127
|
+
expect(chronology(late, [late])).toEqual([]);
|
|
128
|
+
expect(chronology(late, [late, record('4', 'passed', 50)])).toEqual([]);
|
|
129
|
+
});
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Outcome explanation
|
|
3
|
+
*
|
|
4
|
+
* Pure reading of what one selected result says, set beside everything else recorded for the same check, so a
|
|
5
|
+
* detail never leads with a summary that contradicts its evidence. The selected source's outcome, the other current
|
|
6
|
+
* sources, where a failure happened, whether the behavior was reached, and the chronology of one source are all
|
|
7
|
+
* stated separately. Nothing here changes a status or a winner: it only words what was recorded, and anything that
|
|
8
|
+
* was not recorded stays unknown. In particular a failure is never attributed to the product, the test harness or
|
|
9
|
+
* the environment from the wording of its message; only a producer's own `atlas.failure` declaration is shown.
|
|
10
|
+
*/
|
|
11
|
+
import type { HistoryEntry, CheckSlot } from '@crvouga/atlas/reports/evidence-view';
|
|
12
|
+
import { isVideoAttachment, type EvidenceHolder } from '@crvouga/atlas/reports/evidence-detail';
|
|
13
|
+
import { environmentLabel } from '@crvouga/atlas/reports/layer-axes';
|
|
14
|
+
import type { EvidenceResult } from '@crvouga/atlas/reports/layer-merge';
|
|
15
|
+
import type { ReportStep, RunReport } from '@crvouga/atlas/reports/report-types';
|
|
16
|
+
import type { Attachment, Step } from '@crvouga/atlas/schema';
|
|
17
|
+
import { ago, shortSha, statusWord } from './format';
|
|
18
|
+
|
|
19
|
+
/** The extension key under which a producer may declare why and where a check failed. */
|
|
20
|
+
export const FAILURE_EXTENSION = 'atlas.failure';
|
|
21
|
+
|
|
22
|
+
/** Who a producer declared a failure to be the fault of. Absent is unknown, never inferred. */
|
|
23
|
+
export type Attribution = 'product' | 'test-harness' | 'environment';
|
|
24
|
+
|
|
25
|
+
const ATTRIBUTIONS: readonly string[] = ['product', 'test-harness', 'environment'];
|
|
26
|
+
|
|
27
|
+
/** What a producer declared about a failure: the expectation, what was observed, and optionally whose fault it was. */
|
|
28
|
+
export type FailureNote = {
|
|
29
|
+
readonly expected?: string;
|
|
30
|
+
readonly actual?: string;
|
|
31
|
+
readonly attribution?: Attribution;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/** Whether the failed step belongs to getting ready for the behavior, to the behavior itself, or is not known to be either. */
|
|
35
|
+
export type Phase = 'setup' | 'behavior' | 'unspecified';
|
|
36
|
+
|
|
37
|
+
/** Step kinds (an open vocabulary) that are preparation, not the behavior under test. */
|
|
38
|
+
const SETUP_KINDS = new Set(['hook', 'setup', 'fixture', 'before', 'beforeall', 'beforeeach', 'given']);
|
|
39
|
+
/** Step kinds that are the behavior being exercised or an expectation about it. */
|
|
40
|
+
const BEHAVIOR_KINDS = new Set(['when', 'then', 'assertion', 'expect']);
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Which phase a step of this `kind` belongs to. Decided by the recorded kind only, never by a step's name or message;
|
|
44
|
+
* a kind that is not recognized, or none, is `unspecified`.
|
|
45
|
+
*/
|
|
46
|
+
export function phaseOf(kind: string | undefined): Phase {
|
|
47
|
+
const key = kind?.toLowerCase();
|
|
48
|
+
if (!key) return 'unspecified';
|
|
49
|
+
return SETUP_KINDS.has(key) ? 'setup' : BEHAVIOR_KINDS.has(key) ? 'behavior' : 'unspecified';
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The step at which a result failed, as far as the recording says. */
|
|
53
|
+
export type FailurePoint = {
|
|
54
|
+
readonly name: string;
|
|
55
|
+
readonly kind: string | undefined;
|
|
56
|
+
readonly phase: Phase;
|
|
57
|
+
/** The step's own message, as recorded. */
|
|
58
|
+
readonly detail: string | undefined;
|
|
59
|
+
/** How long after the check started it began, when recorded. */
|
|
60
|
+
readonly offsetMs: number | undefined;
|
|
61
|
+
readonly note: FailureNote | undefined;
|
|
62
|
+
/** The videos on the step itself, if it has any. */
|
|
63
|
+
readonly videos: readonly Attachment[];
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
function text(value: unknown): string | undefined {
|
|
67
|
+
if (value === undefined || value === null) return undefined;
|
|
68
|
+
if (typeof value === 'string') return value;
|
|
69
|
+
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
|
70
|
+
return JSON.stringify(value);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The failure a producer declared under {@link FAILURE_EXTENSION} in `extensions`: `expected`, `actual` and an
|
|
75
|
+
* `attribution` of `product`, `test-harness` or `environment`. Anything else is ignored; undefined when nothing usable is there.
|
|
76
|
+
*/
|
|
77
|
+
export function failureNote(extensions: Readonly<Record<string, unknown>> | undefined): FailureNote | undefined {
|
|
78
|
+
const raw = extensions?.[FAILURE_EXTENSION];
|
|
79
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined;
|
|
80
|
+
const { expected, actual, attribution } = raw as Record<string, unknown>;
|
|
81
|
+
const note: { expected?: string; actual?: string; attribution?: Attribution } = {};
|
|
82
|
+
const e = text(expected);
|
|
83
|
+
const a = text(actual);
|
|
84
|
+
if (e !== undefined) note.expected = e;
|
|
85
|
+
if (a !== undefined) note.actual = a;
|
|
86
|
+
if (typeof attribution === 'string' && ATTRIBUTIONS.includes(attribution)) note.attribution = attribution as Attribution;
|
|
87
|
+
return Object.keys(note).length ? note : undefined;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The first failed step of a recorded check, preferring the innermost one (a failed action inside a failed group is
|
|
92
|
+
* the action). Steps are ordered by their start offset, else by their position. Undefined when no step failed or none
|
|
93
|
+
* was recorded.
|
|
94
|
+
*/
|
|
95
|
+
export function failedStep(steps: readonly Step[] | undefined): FailurePoint | undefined {
|
|
96
|
+
if (!steps?.length) return undefined;
|
|
97
|
+
const failedParents = new Set(steps.filter((step) => step.status === 'failed' && step.parentId).map((step) => step.parentId));
|
|
98
|
+
const order = steps.map((step, index) => ({ step, index }));
|
|
99
|
+
order.sort((a, b) => (a.step.offsetMs ?? 0) - (b.step.offsetMs ?? 0) || a.index - b.index);
|
|
100
|
+
const found = order.find(({ step }) => step.status === 'failed' && !(step.id !== undefined && failedParents.has(step.id)))?.step;
|
|
101
|
+
if (!found) return undefined;
|
|
102
|
+
return {
|
|
103
|
+
name: found.name,
|
|
104
|
+
kind: found.kind,
|
|
105
|
+
phase: phaseOf(found.kind),
|
|
106
|
+
detail: found.detail,
|
|
107
|
+
offsetMs: found.offsetMs,
|
|
108
|
+
note: failureNote(found.extensions),
|
|
109
|
+
videos: (found.attachments ?? []).filter(isVideoAttachment)
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** The first innermost failed step of an Atlas run's own timeline (see {@link failedStep}). */
|
|
114
|
+
export function failedReportStep(report: RunReport | undefined): FailurePoint | undefined {
|
|
115
|
+
if (!report) return undefined;
|
|
116
|
+
const walk = (steps: readonly ReportStep[]): ReportStep | undefined => {
|
|
117
|
+
for (const step of steps) {
|
|
118
|
+
if (step.status !== 'failed') continue;
|
|
119
|
+
return walk(step.steps) ?? step;
|
|
120
|
+
}
|
|
121
|
+
return undefined;
|
|
122
|
+
};
|
|
123
|
+
const found = walk(report.steps);
|
|
124
|
+
if (!found) return undefined;
|
|
125
|
+
return {
|
|
126
|
+
name: found.title,
|
|
127
|
+
kind: found.kind,
|
|
128
|
+
phase: phaseOf(found.kind),
|
|
129
|
+
detail: found.error,
|
|
130
|
+
offsetMs: Math.max(0, found.startedAtMs - report.meta.startedAtMs),
|
|
131
|
+
note: undefined,
|
|
132
|
+
videos: []
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The recorded videos a holder carries on itself, its steps and its attempts. */
|
|
137
|
+
export function videosIn(holder: EvidenceHolder | undefined): Attachment[] {
|
|
138
|
+
if (!holder) return [];
|
|
139
|
+
return [
|
|
140
|
+
...(holder.attachments ?? []).filter(isVideoAttachment),
|
|
141
|
+
...(holder.steps ?? []).flatMap((step) => (step.attachments ?? []).filter(isVideoAttachment)),
|
|
142
|
+
...(holder.attempts ?? []).flatMap((attempt) => [
|
|
143
|
+
...(attempt.attachments ?? []).filter(isVideoAttachment),
|
|
144
|
+
...(attempt.steps ?? []).flatMap((step) => (step.attachments ?? []).filter(isVideoAttachment))
|
|
145
|
+
])
|
|
146
|
+
];
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** The one-line statement of what the failure point says about the behavior. */
|
|
150
|
+
export function reachOf(point: FailurePoint | undefined): { readonly state: 'setup-failed' | 'expectation-failed' | 'unknown'; readonly text: string } {
|
|
151
|
+
if (!point) return { state: 'unknown', text: 'Where it failed was not recorded, so whether the behavior was reached is unknown.' };
|
|
152
|
+
if (point.phase === 'setup')
|
|
153
|
+
return { state: 'setup-failed', text: `Behavior not verified: setup failed at “${point.name}”. The execution is still recorded as failed.` };
|
|
154
|
+
if (point.phase === 'behavior')
|
|
155
|
+
return { state: 'expectation-failed', text: `The behavior was reached and an expectation was not met at “${point.name}”.` };
|
|
156
|
+
return { state: 'unknown', text: `Failed at “${point.name}”. Whether the behavior was reached is not recorded.` };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const WORD = { passed: 'passed', failed: 'failed', running: 'is still running', 'not-run': 'was not run' } as const;
|
|
160
|
+
|
|
161
|
+
/** Counts of results by status, in words: `1 passed, 2 failed`. */
|
|
162
|
+
function tally(results: readonly EvidenceResult[]): string {
|
|
163
|
+
const counts = new Map<string, number>();
|
|
164
|
+
for (const { row } of results) counts.set(row.status, (counts.get(row.status) ?? 0) + 1);
|
|
165
|
+
return (['passed', 'failed', 'running', 'not-run'] as const)
|
|
166
|
+
.filter((status) => counts.has(status))
|
|
167
|
+
.map((status) => `${counts.get(status)} ${status === 'not-run' ? 'not run' : status}`)
|
|
168
|
+
.join(', ');
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** What the selected result says, and the current results of the other sources it must be read beside. */
|
|
172
|
+
export type Headline = {
|
|
173
|
+
readonly text: string;
|
|
174
|
+
/** The current results from other sources than the selected one, newest first. */
|
|
175
|
+
readonly others: readonly EvidenceResult[];
|
|
176
|
+
/** False when the selected result is an earlier one that a newer run has replaced in the combined report. */
|
|
177
|
+
readonly current: boolean;
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* The heading for a selected result: its own outcome first, then what the other current sources recorded. A passing
|
|
182
|
+
* selection beside a failing source says both; an earlier result says it is earlier and names the combined status.
|
|
183
|
+
* Never reads the selected outcome as the check's status.
|
|
184
|
+
*/
|
|
185
|
+
export function outcomeHeadline(selected: HistoryEntry, slot: CheckSlot): Headline {
|
|
186
|
+
const current = slot.outcomes.some((outcome) => outcome.layer.id === selected.layer.id);
|
|
187
|
+
const others = slot.outcomes.filter((outcome) => outcome.layer.id !== selected.layer.id);
|
|
188
|
+
const own = WORD[selected.row.status];
|
|
189
|
+
if (!current) {
|
|
190
|
+
return {
|
|
191
|
+
current,
|
|
192
|
+
others,
|
|
193
|
+
text: `This earlier result ${own}; the combined report now shows ${statusWord(slot.status).toLowerCase()} across ${others.length === 1 ? 'its current source' : `its ${others.length} current sources`}`
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
const failed = others.filter((outcome) => outcome.row.status === 'failed').length;
|
|
197
|
+
let text = `Selected result ${own}`;
|
|
198
|
+
if (selected.row.status === 'passed' && failed > 0) text += `; ${failed === 1 ? 'another scoped source' : `${failed} other scoped sources`} failed`;
|
|
199
|
+
else if (others.length > 0) text += `; ${others.length === 1 ? 'the other scoped source' : 'other scoped sources'}: ${tally(others)}`;
|
|
200
|
+
return { current, others, text };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* The result a detail shows: the run the address pins, else the current result of the source it names, else the newest.
|
|
205
|
+
* Undefined only for a check with no result.
|
|
206
|
+
*/
|
|
207
|
+
export function selectedResult(slot: CheckSlot, pinned?: HistoryEntry, source?: string): HistoryEntry | undefined {
|
|
208
|
+
if (pinned) return pinned;
|
|
209
|
+
const found = (source ? slot.outcomes.find((outcome) => outcome.layer.id === source) : undefined) ?? slot.outcomes[0];
|
|
210
|
+
return found ? { layer: found.layer, row: found.row } : undefined;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** How a choosable source is told apart from its siblings without reading opaque producer ids. */
|
|
214
|
+
export type SourceLabel = {
|
|
215
|
+
/** The environment and parameters, or the producer when neither is named. */
|
|
216
|
+
readonly name: string;
|
|
217
|
+
/** Status in words. */
|
|
218
|
+
readonly status: string;
|
|
219
|
+
/** When it was recorded, relative. */
|
|
220
|
+
readonly when: string;
|
|
221
|
+
/** The commit, or the branch, or `revision not recorded`. */
|
|
222
|
+
readonly revision: string;
|
|
223
|
+
/** Present when other sources share the same name: its place among them, newest first, and its producer. */
|
|
224
|
+
readonly attempt: string | undefined;
|
|
225
|
+
/** All of the above in one line. */
|
|
226
|
+
readonly text: string;
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Labels for the current results of a check so each can be told from the others by status, time, revision and,
|
|
231
|
+
* when several share a name, its place among them. `now` is epoch milliseconds.
|
|
232
|
+
*/
|
|
233
|
+
export function sourceLabels(outcomes: readonly EvidenceResult[], now: number): SourceLabel[] {
|
|
234
|
+
const names = outcomes.map((outcome) => {
|
|
235
|
+
const environment = outcome.layer.environment ? environmentLabel(outcome.layer.environment) : '';
|
|
236
|
+
const parameters = Object.entries(outcome.parameters)
|
|
237
|
+
.map(([name, value]) => `${name}: ${value}`)
|
|
238
|
+
.join(', ');
|
|
239
|
+
return [environment, parameters].filter(Boolean).join(' · ') || outcome.layer.producer;
|
|
240
|
+
});
|
|
241
|
+
return outcomes.map((outcome, index) => {
|
|
242
|
+
const name = names[index]!;
|
|
243
|
+
const siblings = names.flatMap((each, at) => (each === name ? [at] : []));
|
|
244
|
+
const attempt =
|
|
245
|
+
siblings.length > 1 ? `result ${siblings.indexOf(index) + 1} of ${siblings.length} from ${outcome.layer.producer}` : undefined;
|
|
246
|
+
const revision = shortSha(outcome.layer.sha) ?? outcome.layer.branch ?? 'revision not recorded';
|
|
247
|
+
const status = statusWord(outcome.row.status);
|
|
248
|
+
const when = ago(outcome.layer.recordedAtMs, now);
|
|
249
|
+
return { name, status, when, revision, attempt, text: [name, status, when, revision, attempt].filter(Boolean).join(' · ') };
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** What the chronology needs to know of one recorded result, whether it came from the held layers or from a test's history. */
|
|
254
|
+
export type ResultRecord = {
|
|
255
|
+
/** The id of the layer that recorded it. */
|
|
256
|
+
readonly layer: string;
|
|
257
|
+
readonly kind: string;
|
|
258
|
+
readonly status: HistoryEntry['row']['status'];
|
|
259
|
+
readonly at: number;
|
|
260
|
+
readonly sha: string | null;
|
|
261
|
+
readonly producer: string;
|
|
262
|
+
readonly environment: string | null;
|
|
263
|
+
readonly parameters: Readonly<Record<string, string>>;
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
/** The record of a layer's result. */
|
|
267
|
+
export function recordOf(entry: HistoryEntry): ResultRecord {
|
|
268
|
+
const { layer, row } = entry;
|
|
269
|
+
return {
|
|
270
|
+
layer: layer.id,
|
|
271
|
+
kind: row.kind,
|
|
272
|
+
status: row.status,
|
|
273
|
+
at: layer.recordedAtMs,
|
|
274
|
+
sha: layer.sha,
|
|
275
|
+
producer: layer.producer,
|
|
276
|
+
environment: layer.environment,
|
|
277
|
+
parameters: layer.parameters
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** One record of the selected result's own source, for the chronology. */
|
|
282
|
+
export type ChronologyEntry = { readonly record: ResultRecord; readonly selected: boolean };
|
|
283
|
+
|
|
284
|
+
function samePlace(a: ResultRecord, b: ResultRecord): boolean {
|
|
285
|
+
return (
|
|
286
|
+
a.producer === b.producer &&
|
|
287
|
+
a.environment === b.environment &&
|
|
288
|
+
JSON.stringify(Object.entries(a.parameters).sort()) === JSON.stringify(Object.entries(b.parameters).sort())
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* The records of the selected result's own source (same producer, environment and parameters), oldest first, one per
|
|
294
|
+
* layer. Empty when there is only one, or when they all have the same outcome, so there is no chronology to tell.
|
|
295
|
+
*/
|
|
296
|
+
export function chronology(selected: ResultRecord, history: readonly ResultRecord[]): ChronologyEntry[] {
|
|
297
|
+
const seen = new Set<string>();
|
|
298
|
+
const own = history
|
|
299
|
+
.filter((record) => samePlace(record, selected) && !seen.has(`${record.layer}|${record.kind}`) && seen.add(`${record.layer}|${record.kind}`))
|
|
300
|
+
.sort((a, b) => a.at - b.at);
|
|
301
|
+
if (own.length < 2 || new Set(own.map((record) => record.status)).size < 2) return [];
|
|
302
|
+
return own.map((record) => ({ record, selected: record.layer === selected.layer && record.kind === selected.kind }));
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Whether the newest record of a chronology passed after an earlier one failed. */
|
|
306
|
+
export function passedAfterFailure(entries: readonly ChronologyEntry[]): boolean {
|
|
307
|
+
const last = entries.at(-1);
|
|
308
|
+
return last !== undefined && last.record.status === 'passed' && entries.slice(0, -1).some((each) => each.record.status === 'failed');
|
|
309
|
+
}
|
|
@@ -17,6 +17,7 @@ import { CheckDetail } from '../evidence/check-detail';
|
|
|
17
17
|
import { ago, exactTime, plural, shortSha, statusWord, useNow } from '../evidence/format';
|
|
18
18
|
import { useEvidence } from '../evidence/evidence-context';
|
|
19
19
|
import { Origin, StatusMark } from '../evidence/marks';
|
|
20
|
+
import { outcomeHeadline, selectedResult } from '../evidence/outcome';
|
|
20
21
|
import { useTestHistory } from './catalog';
|
|
21
22
|
import { patched, useExplorerNavigate, useExplorerSearch } from './navigate';
|
|
22
23
|
import { Badges, runName } from './parts';
|
|
@@ -126,7 +127,7 @@ function Scope({ pinned, slot }: { pinned: PinnedResult; slot: CheckSlot | undef
|
|
|
126
127
|
export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot | undefined }) {
|
|
127
128
|
const now = useNow();
|
|
128
129
|
const history = useTestHistory(test.key);
|
|
129
|
-
const { layer } = useExplorerSearch();
|
|
130
|
+
const { layer, source } = useExplorerSearch();
|
|
130
131
|
const go = useExplorerNavigate();
|
|
131
132
|
const pinned = usePinnedResult(layer, test.key);
|
|
132
133
|
const [recording, setRecording] = useState(false);
|
|
@@ -134,6 +135,7 @@ export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot
|
|
|
134
135
|
const [moment, setMoment] = useState<number | null>(null);
|
|
135
136
|
const [rows, setRows] = useState(HISTORY_ROWS);
|
|
136
137
|
const entries = history.data?.history ?? [];
|
|
138
|
+
const selected = slot && (!layer || pinned.state === 'ready') ? selectedResult(slot, pinned.state === 'ready' ? pinned.entry : undefined, source) : undefined;
|
|
137
139
|
const stats = test.stats;
|
|
138
140
|
const d = stats.durations;
|
|
139
141
|
if (recording && slot) {
|
|
@@ -211,6 +213,9 @@ export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot
|
|
|
211
213
|
<Text data-status={test.status} span inherit fw={600}>
|
|
212
214
|
{statusWord(test.status)}
|
|
213
215
|
</Text>
|
|
216
|
+
<Text span inherit c="dimmed">
|
|
217
|
+
worst current result of {plural(test.outcomes.length, 'source', 'sources')}
|
|
218
|
+
</Text>
|
|
214
219
|
{stats.flaky && (
|
|
215
220
|
<Text title={stats.flakyReason} span inherit>
|
|
216
221
|
Flaky: {stats.flakyReason}
|
|
@@ -218,6 +223,11 @@ export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot
|
|
|
218
223
|
)}
|
|
219
224
|
<Badges test={test} />
|
|
220
225
|
</Flex>
|
|
226
|
+
{selected && slot && (
|
|
227
|
+
<Text component="p" role="status" fw={600} data-outcome-headline>
|
|
228
|
+
{outcomeHeadline(selected, slot).text}
|
|
229
|
+
</Text>
|
|
230
|
+
)}
|
|
221
231
|
{test.spec && <Code c="dimmed">{test.spec!.path}</Code>}
|
|
222
232
|
<Code title="Check id" c="dimmed">
|
|
223
233
|
{test.id}
|
|
@@ -425,6 +435,18 @@ export function TestDetail({ test, slot }: { test: TestSummary; slot: CheckSlot
|
|
|
425
435
|
layerId={layer}
|
|
426
436
|
{...(pinned.state === 'ready' ? { pinned: pinned.entry, pin: true } : {})}
|
|
427
437
|
reads="steps"
|
|
438
|
+
headline={false}
|
|
439
|
+
records={entries.map((entry) => ({
|
|
440
|
+
layer: entry.layer,
|
|
441
|
+
kind: entry.kind,
|
|
442
|
+
status: entry.status,
|
|
443
|
+
at: entry.at,
|
|
444
|
+
sha: entry.sha,
|
|
445
|
+
producer: entry.producer,
|
|
446
|
+
environment: entry.environment,
|
|
447
|
+
parameters: entry.parameters
|
|
448
|
+
}))}
|
|
449
|
+
{...(layer ? {} : { source, onSource: (next: string | null) => go({ source: next ?? undefined }) })}
|
|
428
450
|
select={(patch) => {
|
|
429
451
|
if (patch.view === 'recording') {
|
|
430
452
|
setRecordingLayer(patch.layer ?? undefined);
|
|
@@ -67,6 +67,8 @@ export const ExplorerSearchFields = {
|
|
|
67
67
|
openRun: text,
|
|
68
68
|
/** Pins the open test's detail to one exact layer (a run's record of it), instead of the latest combined evidence. With `openRun` and `artifact` it names the layer the file is in. */
|
|
69
69
|
layer: text,
|
|
70
|
+
/** Which current source (a layer id) of the open test the detail's outcome summary, media and failure describe, without pinning a run. Unknown or absent: the newest. */
|
|
71
|
+
source: text,
|
|
70
72
|
/** With `openRun` and `layer`: the one file of that layer (an `artifactRef`: `id:…`, `key:…` or `name:…`) the run's Artifacts select. */
|
|
71
73
|
artifact: text,
|
|
72
74
|
/** Only the runs, and the results of an opened run, that have a playable video recording. */
|
|
@@ -98,6 +100,7 @@ export type ExplorerSearch = {
|
|
|
98
100
|
readonly tab?: 'runs' | 'artifacts';
|
|
99
101
|
readonly openRun?: string;
|
|
100
102
|
readonly layer?: string;
|
|
103
|
+
readonly source?: string;
|
|
101
104
|
readonly artifact?: string;
|
|
102
105
|
readonly video?: true;
|
|
103
106
|
};
|