@crvouga/atlas 1.17.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -1
- package/dist/reports/src/artifact-query.d.ts +14 -2
- package/dist/reports/src/artifact-query.js +13 -4
- package/dist/reports/src/evidence-detail.js +19 -6
- package/dist/reports/src/evidence-types.d.ts +8 -0
- package/dist/reports/src/recording-receipt.d.ts +23 -0
- package/dist/reports/src/recording-receipt.js +16 -0
- package/dist/reports/src/recording-verify.d.ts +74 -0
- package/dist/reports/src/recording-verify.js +142 -0
- package/dist/reports/src/report-cli.d.ts +2 -2
- package/dist/reports/src/report-cli.js +53 -3
- package/dist/reports/src/test-catalog.d.ts +7 -1
- package/dist/reports/src/test-catalog.js +10 -4
- package/dist/schema/src/api.d.ts +134 -0
- package/dist/schema/src/api.js +13 -2
- package/dist/schema/src/cli-results.d.ts +51 -1
- package/dist/schema/src/cli-results.js +25 -1
- package/dist/server/src/openapi.js +1 -0
- package/docs/agents.md +1 -0
- package/docs/reference/cli.md +11 -2
- package/docs/reference/http-api.md +2 -0
- package/docs/reference/reports.md +189 -9
- package/docs/reference/schema.md +19 -1
- package/docs/rich-evidence.md +33 -5
- package/docs/server.md +1 -1
- package/docs/ui.md +40 -1
- package/llms-full.txt +88 -9
- package/package.json +1 -1
- package/packages/schema/schemas/effective-delta.schema.json +26 -0
- package/packages/schema/schemas/effective-evidence.schema.json +26 -0
- package/packages/schema/schemas/evidence-page.schema.json +13 -0
- package/packages/schema/schemas/layer-rows.schema.json +13 -0
- package/packages/schema/schemas/report-run.schema.json +83 -0
- package/packages/schema/schemas/report-runs.schema.json +86 -0
- package/packages/schema/schemas/review-collection.schema.json +13 -0
- package/packages/schema/schemas/review-manifest.schema.json +13 -0
- package/packages/schema/schemas/test-catalog.schema.json +13 -0
- package/packages/schema/schemas/test-history.schema.json +26 -0
- package/packages/schema/schemas/verify-recordings-result.schema.json +97 -0
- package/packages/server/openapi.json +378 -0
- package/packages/visualizer/src/app/AppShell.tsx +2 -2
- package/packages/visualizer/src/app/router.tsx +25 -1
- package/packages/visualizer/src/components/Header.tsx +15 -4
- package/packages/visualizer/src/evidence/behavior-index.test.ts +119 -0
- package/packages/visualizer/src/evidence/behavior-index.ts +162 -0
- package/packages/visualizer/src/evidence/behavior-panel.tsx +36 -22
- package/packages/visualizer/src/evidence/behavior-return.test.ts +41 -0
- package/packages/visualizer/src/evidence/behavior-return.ts +59 -0
- package/packages/visualizer/src/evidence/behavior.ts +32 -1
- package/packages/visualizer/src/evidence/behaviors-page.tsx +465 -0
- package/packages/visualizer/src/evidence/feature-page.tsx +5 -26
- package/packages/visualizer/src/evidence/scoped-summary-view.tsx +110 -0
- package/packages/visualizer/src/evidence/scoped-summary.test.ts +62 -0
- package/packages/visualizer/src/evidence/scoped-summary.ts +118 -0
- package/packages/visualizer/src/evidence/spec-texts.ts +88 -0
- package/packages/visualizer/src/explorer/ArtifactsView.tsx +7 -6
- package/packages/visualizer/src/explorer/Explorer.tsx +1 -1
- package/packages/visualizer/src/explorer/RunsView.tsx +32 -7
- package/packages/visualizer/src/explorer/collections.tsx +5 -5
- package/packages/visualizer/src/explorer/filters.tsx +11 -3
- package/packages/visualizer/src/explorer/navigate.ts +17 -0
- package/packages/visualizer/src/explorer/playlist-entries.test.ts +10 -1
- package/packages/visualizer/src/explorer/playlist-entries.ts +10 -0
- package/packages/visualizer/src/explorer/playlist.tsx +43 -8
- package/packages/visualizer/src/explorer/search.ts +13 -3
- package/packages/visualizer/src/find/Finder.tsx +1 -62
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { TestResultEntry } from '@crvouga/atlas/reports/test-catalog';
|
|
2
|
+
import { expect, test } from 'bun:test';
|
|
3
|
+
|
|
4
|
+
import { entriesInScope, scopedFigures } from './scoped-summary';
|
|
5
|
+
|
|
6
|
+
const entry = (i: number, over: Partial<TestResultEntry> = {}): TestResultEntry => ({
|
|
7
|
+
at: 1_000 + i, status: 'passed', durationMs: 9_000, environment: 'desktop-web', producer: 'web', kind: 'checks', parameters: {}, branch: 'main',
|
|
8
|
+
sha: 'a'.repeat(40), runUrl: null, run: `run:r${i}`, layer: `l${i}`, backend: 'one', detail: '', ...over
|
|
9
|
+
});
|
|
10
|
+
const KEY = '["specs/a.feature","a"]';
|
|
11
|
+
const latency = (value: number, over: Record<string, unknown> = {}) => ({ name: 'save', kind: 'product-latency', unit: 'ms', value, from: 'tap', to: 'saved', ...over });
|
|
12
|
+
|
|
13
|
+
test('no history leaves reliability and product latency unknown, never good', () => {
|
|
14
|
+
const figures = scopedFigures(KEY, [], { environment: 'desktop-web' });
|
|
15
|
+
expect(figures.reliability.tone).toBe('unknown');
|
|
16
|
+
expect(figures.reliability.headline).toContain('unknown');
|
|
17
|
+
expect(figures.latency).toHaveLength(1);
|
|
18
|
+
expect(figures.latency[0]!.headline).toContain('unknown');
|
|
19
|
+
// The duration of the test is not a stand-in for the product's response time.
|
|
20
|
+
expect(figures.latency[0]!.qualifiers.join(' ')).toContain('not product latency');
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test('a rate always names its independent executions and a small sample is insufficient evidence', () => {
|
|
24
|
+
const figures = scopedFigures(KEY, [entry(0), entry(1), entry(2, { status: 'failed' })], { environment: 'desktop-web' });
|
|
25
|
+
expect(figures.reliability.headline).toBe('2 of 3 independent executions passed in the end (66.7%).');
|
|
26
|
+
expect(figures.reliability.tone).toBe('weak');
|
|
27
|
+
expect(figures.reliability.qualifiers.join(' ')).toContain('Insufficient evidence');
|
|
28
|
+
expect(figures.reliability.qualifiers.join(' ')).toContain('First-try success is unknown');
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('the scope is the shown result\'s environment: another environment is not pooled into it', () => {
|
|
32
|
+
const history = [entry(0), entry(1, { environment: 'ios', status: 'failed' }), entry(2, { environment: 'ios', status: 'failed' })];
|
|
33
|
+
expect(entriesInScope(history, { environment: 'desktop-web' })).toHaveLength(1);
|
|
34
|
+
expect(entriesInScope(history, { environment: null })).toHaveLength(3);
|
|
35
|
+
const scoped = scopedFigures(KEY, history, { environment: 'desktop-web' });
|
|
36
|
+
expect(scoped.reliability.headline).toContain('1 of 1 independent execution passed');
|
|
37
|
+
expect(scoped.reliability.qualifiers[0]).toContain('desktop-web');
|
|
38
|
+
// With no environment named the executions are pooled, and the figure says so.
|
|
39
|
+
const pooled = scopedFigures(KEY, history, { environment: null });
|
|
40
|
+
expect(pooled.reliability.qualifiers.join(' ')).toMatch(/environments are pooled/i);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('product latency is read only from named measurements, with withheld percentiles explained', () => {
|
|
44
|
+
const history = [entry(0, { latency: [latency(300)] }), entry(1, { latency: [latency(500, { limit: 400 })] }), entry(2)];
|
|
45
|
+
const [line] = scopedFigures(KEY, history, { environment: 'desktop-web' }).latency;
|
|
46
|
+
expect(line!.headline).toContain('save');
|
|
47
|
+
expect(line!.headline).toContain('no percentile can be stated');
|
|
48
|
+
expect(line!.tone).toBe('weak');
|
|
49
|
+
const text = line!.qualifiers.join(' ');
|
|
50
|
+
expect(text).toContain('2 observations');
|
|
51
|
+
expect(text).toContain('1 observation over it');
|
|
52
|
+
expect(text).toContain('p50 needs 5 observations');
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('enough observations state p50 and p95 with the budget and its breaches', () => {
|
|
56
|
+
const history = Array.from({ length: 20 }, (_, i) => entry(i, { latency: [latency(100 + i * 10, { limit: 250 })] }));
|
|
57
|
+
const [line] = scopedFigures(KEY, history, { environment: 'desktop-web' }).latency;
|
|
58
|
+
expect(line!.headline).toContain('p50');
|
|
59
|
+
expect(line!.headline).toContain('p95');
|
|
60
|
+
expect(line!.tone).toBe('figure');
|
|
61
|
+
expect(line!.qualifiers.join(' ')).toContain('20 observations');
|
|
62
|
+
});
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Scoped behavior summary
|
|
3
|
+
*
|
|
4
|
+
* The short account that sits beside a behavior's written promise and recording: what the selected result says, how
|
|
5
|
+
* reliable the check has been, and how quickly the product responded. It does not compute anything of its own. The
|
|
6
|
+
* outcome is {@link outcomeHeadline}; reliability is `reliability` and product latency is `latencyReview` from the
|
|
7
|
+
* reports library, read over the history of the same check in the same environment as the recording shown. This module
|
|
8
|
+
* only chooses the scope and puts the qualified figures into short sentences, keeping every qualification that makes a
|
|
9
|
+
* figure honest: how many independent executions it rests on, that a small sample is not established, which notes warn
|
|
10
|
+
* against reading it as representative, and that absent evidence is unknown, never good.
|
|
11
|
+
*/
|
|
12
|
+
import { formatLatency, latencyReview, type LatencyReview } from '@crvouga/atlas/reports/latency';
|
|
13
|
+
import { reliability, scopeText, type ReliabilityDocument, type ReliabilityScope } from '@crvouga/atlas/reports/reliability';
|
|
14
|
+
import type { TestResultEntry } from '@crvouga/atlas/reports/test-catalog';
|
|
15
|
+
|
|
16
|
+
import { plural } from './format';
|
|
17
|
+
|
|
18
|
+
/** One line of the summary: the figure in words, and what qualifies it. */
|
|
19
|
+
export type SummaryLine = {
|
|
20
|
+
readonly headline: string;
|
|
21
|
+
/** What a reader must keep in mind: sample size, scope, warnings. Never empty for a figure. */
|
|
22
|
+
readonly qualifiers: readonly string[];
|
|
23
|
+
/** `figure`: a number supported by evidence. `unknown`: there is no evidence to state. `weak`: a figure that its own qualifiers say not to rely on. */
|
|
24
|
+
readonly tone: 'figure' | 'weak' | 'unknown';
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** The scope the summary describes: the environment of the shown result, so environments are never pooled silently. */
|
|
28
|
+
export type SummaryScope = {
|
|
29
|
+
/** The environment of the result shown; null when it recorded none (then every environment is pooled and the summary says so). */
|
|
30
|
+
readonly environment: string | null;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const percent = (rate: number | null) => (rate === null ? 'no rate' : `${Math.round(rate * 1000) / 10}%`);
|
|
34
|
+
|
|
35
|
+
/** The reliability scope for a summary scope. */
|
|
36
|
+
export function reliabilityScope(scope: SummaryScope): ReliabilityScope {
|
|
37
|
+
return scope.environment === null ? {} : { environment: [scope.environment] };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The recorded results of the summary's scope: those of its environment, or all when it names none. */
|
|
41
|
+
export function entriesInScope(entries: readonly TestResultEntry[], scope: SummaryScope): readonly TestResultEntry[] {
|
|
42
|
+
return scope.environment === null ? entries : entries.filter((entry) => entry.environment === scope.environment);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Reliability in one sentence and its qualifiers. A count of independent executions always comes with the figure;
|
|
47
|
+
* fewer than the minimum is "Insufficient evidence"; none is unknown. The library's warning notes are carried over.
|
|
48
|
+
*/
|
|
49
|
+
export function reliabilityLine(document: ReliabilityDocument): SummaryLine {
|
|
50
|
+
const { eventual, firstAttempt } = document;
|
|
51
|
+
const where = scopeText(document.scope);
|
|
52
|
+
if (document.sample === 'none') {
|
|
53
|
+
return {
|
|
54
|
+
headline: 'Reliability is unknown: no independent execution is recorded in this scope.',
|
|
55
|
+
qualifiers: [`Scope: ${where}.`],
|
|
56
|
+
tone: 'unknown'
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
const qualifiers = [`Scope: ${where}.`];
|
|
60
|
+
if (document.sample === 'insufficient') {
|
|
61
|
+
qualifiers.push(`Insufficient evidence: at least ${document.minimumExecutions} independent executions are needed before a rate means much.`);
|
|
62
|
+
}
|
|
63
|
+
qualifiers.push(
|
|
64
|
+
firstAttempt.executions
|
|
65
|
+
? `${firstAttempt.passed} of ${plural(firstAttempt.executions, 'execution', 'executions')} passed first time.`
|
|
66
|
+
: 'First-try success is unknown: no tries were recorded.'
|
|
67
|
+
);
|
|
68
|
+
for (const note of document.notes) if (note.severity === 'warning') qualifiers.push(note.message);
|
|
69
|
+
return {
|
|
70
|
+
headline: `${eventual.passed} of ${plural(eventual.executions, 'independent execution', 'independent executions')} passed in the end (${percent(eventual.rate)}).`,
|
|
71
|
+
qualifiers,
|
|
72
|
+
tone: document.sample === 'insufficient' || document.notes.some((note) => note.severity === 'warning') ? 'weak' : 'figure'
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Product latency: one line per measurement a producer named as product latency. The test's own duration is never read
|
|
78
|
+
* here. Percentiles the evidence cannot support are said to be withheld with their reason; with no measurement the
|
|
79
|
+
* answer is unknown, not fast.
|
|
80
|
+
*/
|
|
81
|
+
export function latencyLines(review: LatencyReview): readonly SummaryLine[] {
|
|
82
|
+
if (review.metrics.length === 0) {
|
|
83
|
+
return [
|
|
84
|
+
{
|
|
85
|
+
headline: 'Product latency is unknown: no product-latency measurement is recorded.',
|
|
86
|
+
qualifiers: ['The duration of the test is not product latency and is not shown here.'],
|
|
87
|
+
tone: 'unknown'
|
|
88
|
+
}
|
|
89
|
+
];
|
|
90
|
+
}
|
|
91
|
+
return review.metrics.flatMap(({ metric, groups }) =>
|
|
92
|
+
groups.map<SummaryLine>((summary) => {
|
|
93
|
+
const figures = [
|
|
94
|
+
summary.p50 === undefined ? undefined : `p50 ${formatLatency(summary.p50, metric.unit)}`,
|
|
95
|
+
summary.p95 === undefined ? undefined : `p95 ${formatLatency(summary.p95, metric.unit)}`
|
|
96
|
+
].filter((figure): figure is string => figure !== undefined);
|
|
97
|
+
const qualifiers = [`${plural(summary.samples, 'observation', 'observations')}${summary.lowerBound ? ', including some that did not complete (lower bounds)' : ''}.`];
|
|
98
|
+
if (summary.budget !== undefined) {
|
|
99
|
+
qualifiers.push(`Budget ${formatLatency(summary.budget, metric.unit)}: ${plural(summary.breaches, 'observation', 'observations')} over it.`);
|
|
100
|
+
}
|
|
101
|
+
qualifiers.push(...summary.withheld, ...summary.gaps);
|
|
102
|
+
return {
|
|
103
|
+
headline: `${metric.name}${metric.boundary.label ? ` (${metric.boundary.label})` : ''}: ${figures.length ? figures.join(', ') : 'no percentile can be stated'}.`,
|
|
104
|
+
qualifiers,
|
|
105
|
+
tone: figures.length > 0 && summary.gaps.length === 0 ? 'figure' : 'weak'
|
|
106
|
+
};
|
|
107
|
+
})
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Reliability and product latency of one check in a scope, worked out from its recorded history. */
|
|
112
|
+
export function scopedFigures(key: string, history: readonly TestResultEntry[], scope: SummaryScope) {
|
|
113
|
+
const entries = entriesInScope(history, scope);
|
|
114
|
+
return {
|
|
115
|
+
reliability: reliabilityLine(reliability(key, history, reliabilityScope(scope))),
|
|
116
|
+
latency: latencyLines(latencyReview(entries))
|
|
117
|
+
};
|
|
118
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Reading spec texts for search and indexes
|
|
3
|
+
*
|
|
4
|
+
* The finder and the project-wide behavior index both need the text of many specs, not only the one on screen. They
|
|
5
|
+
* read it here: a few files at a time, each landing as it arrives so a search can use what is read so far, through the
|
|
6
|
+
* same query cache the spec pages use (so a file is never fetched twice and a page opened afterwards finds it ready).
|
|
7
|
+
*/
|
|
8
|
+
import { useQueryClient } from '@tanstack/react-query';
|
|
9
|
+
import { useEffect, useMemo, useRef, useState } from 'react';
|
|
10
|
+
|
|
11
|
+
import { keys, specKey, useAdapter } from '../data/queries';
|
|
12
|
+
|
|
13
|
+
/** How many spec files are read at once. */
|
|
14
|
+
const READERS = 4;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Reads the text of every spec in `files`, a few at a time, once `enabled`. Each file lands as it arrives, so a search
|
|
18
|
+
* can use what is read so far; `read` of `total` say how far it is, and `failed` how many could not be read at all
|
|
19
|
+
* (`texts` holds an empty text for them). Files the query cache already holds are available at once, without a request.
|
|
20
|
+
* Setting `open` (the finder opening) gives the files that could not be read another chance.
|
|
21
|
+
*/
|
|
22
|
+
export function useSpecTexts(files: readonly string[], enabled: boolean, open: boolean) {
|
|
23
|
+
const adapter = useAdapter();
|
|
24
|
+
const client = useQueryClient();
|
|
25
|
+
const source = specKey(adapter);
|
|
26
|
+
const cache = useRef(new Map<string, string>());
|
|
27
|
+
const unreadable = useRef(new Set<string>());
|
|
28
|
+
const [texts, setTexts] = useState<ReadonlyMap<string, string>>(() => new Map());
|
|
29
|
+
const [pending, setPending] = useState(false);
|
|
30
|
+
const [failedFiles, setFailedFiles] = useState<ReadonlySet<string>>(() => new Set());
|
|
31
|
+
const key = files.join('\n');
|
|
32
|
+
// Opening the finder is another chance for the files that could not be read.
|
|
33
|
+
useEffect(() => {
|
|
34
|
+
if (!open || unreadable.current.size === 0) return;
|
|
35
|
+
for (const file of unreadable.current) cache.current.delete(file);
|
|
36
|
+
unreadable.current.clear();
|
|
37
|
+
setFailedFiles(new Set());
|
|
38
|
+
}, [open]);
|
|
39
|
+
useEffect(() => {
|
|
40
|
+
if (!enabled) return;
|
|
41
|
+
const wanted = key.length === 0 ? [] : key.split('\n');
|
|
42
|
+
let cancelled = false;
|
|
43
|
+
// What the pages already read is not asked for again.
|
|
44
|
+
let seeded = false;
|
|
45
|
+
for (const file of wanted) {
|
|
46
|
+
if (cache.current.has(file)) continue;
|
|
47
|
+
const held = client.getQueryData<string>(keys.specFile(source, file));
|
|
48
|
+
if (typeof held === 'string') {
|
|
49
|
+
cache.current.set(file, held);
|
|
50
|
+
seeded = true;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (seeded) setTexts(new Map(cache.current));
|
|
54
|
+
const missing = wanted.filter((file) => !cache.current.has(file));
|
|
55
|
+
if (missing.length === 0) {
|
|
56
|
+
setPending(false);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
setPending(true);
|
|
60
|
+
let cursor = 0;
|
|
61
|
+
const pull = async () => {
|
|
62
|
+
for (;;) {
|
|
63
|
+
const file = missing[cursor++];
|
|
64
|
+
if (file === undefined || cancelled) return;
|
|
65
|
+
let text = '';
|
|
66
|
+
try {
|
|
67
|
+
text = await client.fetchQuery({ queryKey: keys.specFile(source, file), queryFn: () => adapter.specFile(file), retry: false });
|
|
68
|
+
} catch {
|
|
69
|
+
text = '';
|
|
70
|
+
unreadable.current.add(file);
|
|
71
|
+
}
|
|
72
|
+
if (cancelled) return;
|
|
73
|
+
cache.current.set(file, text);
|
|
74
|
+
setTexts(new Map(cache.current));
|
|
75
|
+
if (unreadable.current.has(file)) setFailedFiles(new Set(unreadable.current));
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
void Promise.all(Array.from({ length: READERS }, pull)).then(() => {
|
|
79
|
+
if (!cancelled) setPending(false);
|
|
80
|
+
});
|
|
81
|
+
return () => {
|
|
82
|
+
cancelled = true;
|
|
83
|
+
};
|
|
84
|
+
}, [adapter, client, enabled, key, source]);
|
|
85
|
+
const read = useMemo(() => files.filter((file) => texts.has(file)).length, [files, texts]);
|
|
86
|
+
const failed = useMemo(() => files.filter((file) => failedFiles.has(file)).length, [files, failedFiles]);
|
|
87
|
+
return { texts, pending, total: files.length, read, failed };
|
|
88
|
+
}
|
|
@@ -20,13 +20,13 @@ import { StatusMark } from '../evidence/marks';
|
|
|
20
20
|
import { useArtifactList } from './catalog';
|
|
21
21
|
import { FreezeCollection } from './collections';
|
|
22
22
|
import { FilterBar } from './filters';
|
|
23
|
-
import { patched, useExplorerNavigate, useExplorerSearch } from './navigate';
|
|
23
|
+
import { patched, useExplorerNavigate, useExplorerSearch, useWatchAddress } from './navigate';
|
|
24
24
|
import { runName } from './parts';
|
|
25
25
|
import { entriesFromArtifacts } from './playlist-entries';
|
|
26
26
|
import { WatchPlaylist } from './playlist';
|
|
27
27
|
import { RecordingTextSearch } from './RecordingTextSearch';
|
|
28
28
|
import { filtered, filterOf, NO_FILTERS } from './search';
|
|
29
|
-
import { filterParams } from '@crvouga/atlas/reports/artifact-query';
|
|
29
|
+
import { filterParams, forArtifacts } from '@crvouga/atlas/reports/artifact-query';
|
|
30
30
|
|
|
31
31
|
const ARTIFACT_ROWS = 50;
|
|
32
32
|
|
|
@@ -34,12 +34,13 @@ const ARTIFACT_ROWS = 50;
|
|
|
34
34
|
export function ArtifactsView() {
|
|
35
35
|
const search = useExplorerSearch();
|
|
36
36
|
const go = useExplorerNavigate();
|
|
37
|
-
|
|
37
|
+
// Recording completeness belongs to a run, not to a recorded result: the Artifacts tab never asks for it.
|
|
38
|
+
const filter = useMemo(() => forArtifacts(filterOf(search)), [search]);
|
|
38
39
|
const narrow = filtered(search);
|
|
39
40
|
const answer = useArtifactList(filter);
|
|
40
41
|
const now = useNow();
|
|
41
42
|
const [shown, setShown] = useState(ARTIFACT_ROWS);
|
|
42
|
-
const
|
|
43
|
+
const watch = useWatchAddress();
|
|
43
44
|
const [freezing, setFreezing] = useState(false);
|
|
44
45
|
const entries = useMemo(() => (answer.data && !answer.data.unsupported ? entriesFromArtifacts(answer.data.list.results) : []), [answer.data]);
|
|
45
46
|
if (answer.isPending)
|
|
@@ -88,7 +89,7 @@ export function ArtifactsView() {
|
|
|
88
89
|
)}
|
|
89
90
|
{list.results.length > 0 && (
|
|
90
91
|
<Group my="sm" gap="sm">
|
|
91
|
-
<Button type="button" variant="filled" leftSection={<IconPlayerPlay size={16} aria-hidden="true" />} onClick={() =>
|
|
92
|
+
<Button type="button" variant="filled" leftSection={<IconPlayerPlay size={16} aria-hidden="true" />} onClick={() => watch.open()}>
|
|
92
93
|
Watch these {plural(list.results.length, 'result', 'results')}
|
|
93
94
|
</Button>
|
|
94
95
|
{!staticEvidence && (
|
|
@@ -98,7 +99,7 @@ export function ArtifactsView() {
|
|
|
98
99
|
)}
|
|
99
100
|
</Group>
|
|
100
101
|
)}
|
|
101
|
-
{watching && <WatchPlaylist entries={entries} total={list.total}
|
|
102
|
+
{watch.watching && <WatchPlaylist entries={entries} total={list.total} initial={watch.clip} onSelect={watch.select} onClose={watch.close} />}
|
|
102
103
|
{freezing && <FreezeCollection results={list.results} total={list.total} filterQuery={filterParams(filter).toString()} onClose={() => setFreezing(false)} />}
|
|
103
104
|
{list.results.length > 0 && <RecordingTextSearch entries={entries} total={list.total} />}
|
|
104
105
|
{list.results.length > 0 && (
|
|
@@ -198,7 +198,7 @@ export function Explorer() {
|
|
|
198
198
|
{model && (model.tests.length > 0 || model.summary.runs > 0) && (
|
|
199
199
|
<Tabs
|
|
200
200
|
value={search.tab ?? 'tests'}
|
|
201
|
-
onChange={(value) => go({ tab: value === 'runs' || value === 'artifacts' || value === 'collections' || value === 'compare' || value === 'health' ? value : undefined, test: undefined, openRun: undefined, layer: undefined, artifact: undefined, collection: undefined, rev: undefined, pair: undefined })}
|
|
201
|
+
onChange={(value) => go({ tab: value === 'runs' || value === 'artifacts' || value === 'collections' || value === 'compare' || value === 'health' ? value : undefined, test: undefined, openRun: undefined, layer: undefined, artifact: undefined, collection: undefined, rev: undefined, pair: undefined, watch: undefined, watchClip: undefined })}
|
|
202
202
|
>
|
|
203
203
|
<Tabs.List aria-label="Explorer view">
|
|
204
204
|
<Tabs.Tab
|
|
@@ -29,7 +29,7 @@ import { entriesFromRun } from './playlist-entries';
|
|
|
29
29
|
import { WatchPlaylist } from './playlist';
|
|
30
30
|
import { FilterBar } from './filters';
|
|
31
31
|
import { SearchNotes, TypedFilters } from './search-notes';
|
|
32
|
-
import { patched, useExplorerNavigate, useExplorerSearch } from './navigate';
|
|
32
|
+
import { patched, useExplorerNavigate, useExplorerSearch, useWatchAddress } from './navigate';
|
|
33
33
|
import { runName } from './parts';
|
|
34
34
|
import { filtered, filterOf, NO_FILTERS } from './search';
|
|
35
35
|
const RUN_ROWS = 50;
|
|
@@ -93,15 +93,39 @@ function ownerText(video: VideoRef): string {
|
|
|
93
93
|
return 'the test';
|
|
94
94
|
}
|
|
95
95
|
const UNAVAILABLE = 'Listed in the report, but its file was never stored, so it cannot be played.';
|
|
96
|
-
|
|
96
|
+
const MISSING = 'The run says it stored this file, but its bundle no longer holds it, so it cannot be played.';
|
|
97
|
+
/** The unavailable references of a video summary and why they cannot be watched: never stored, or stored and then gone. */
|
|
97
98
|
function Unavailable({ videos }: { videos: VideoCounts }) {
|
|
98
99
|
if (!videos.unavailable) return null;
|
|
100
|
+
const missing = Math.min(videos.missing ?? 0, videos.unavailable);
|
|
99
101
|
return (
|
|
100
102
|
<Text component="div" size="xs" c="dimmed" data-video-unavailable>
|
|
101
|
-
<IconAlertTriangle size={16} aria-hidden="true" /> {plural(videos.unavailable, 'video', 'videos')} unavailable.
|
|
103
|
+
<IconAlertTriangle size={16} aria-hidden="true" /> {plural(videos.unavailable, 'video', 'videos')} unavailable.{' '}
|
|
104
|
+
{missing > 0 && <Text span inherit data-video-missing>{missing === videos.unavailable ? MISSING : `${plural(missing, 'video', 'videos')}: ${MISSING}`}{' '}</Text>}
|
|
105
|
+
{missing < videos.unavailable && UNAVAILABLE}
|
|
102
106
|
</Text>
|
|
103
107
|
);
|
|
104
108
|
}
|
|
109
|
+
/** Words for a run's recording completeness, apart from how its tests went. */
|
|
110
|
+
const COMPLETENESS_WORDS: Record<string, string> = { complete: 'complete', incomplete: 'incomplete', 'none-expected': 'none expected', unknown: 'unknown' };
|
|
111
|
+
/** A run's recording completeness in its row: how many expected recordings were published, said in words and independent of its outcomes. Nothing when its layers declared no expectations (unknown is not shown as complete). */
|
|
112
|
+
function RunRecordings({ run }: { run: RunSummary }) {
|
|
113
|
+
const recordings = run.recordings;
|
|
114
|
+
if (!recordings) return null;
|
|
115
|
+
const word = COMPLETENESS_WORDS[recordings.state] ?? 'unknown';
|
|
116
|
+
const text = recordings.expected > 0 ? `Recordings ${recordings.published} of ${recordings.expected}: ${word}` : `Recordings: ${word}`;
|
|
117
|
+
return (
|
|
118
|
+
<Badge
|
|
119
|
+
size="sm"
|
|
120
|
+
variant="light"
|
|
121
|
+
color={recordings.state === 'incomplete' ? 'orange' : 'gray'}
|
|
122
|
+
data-recordings={recordings.state}
|
|
123
|
+
title={recordings.state === 'incomplete' ? 'Some expected recordings were not published. The test outcomes are unaffected.' : undefined}
|
|
124
|
+
>
|
|
125
|
+
{text}
|
|
126
|
+
</Badge>
|
|
127
|
+
);
|
|
128
|
+
}
|
|
105
129
|
/** A run's videos in its row: a link to the recordings, how many references they come from, and what cannot be played. */
|
|
106
130
|
function RunVideos({ run }: { run: RunSummary }) {
|
|
107
131
|
const videos = run.videos;
|
|
@@ -210,7 +234,7 @@ const RUN_SORTS = [
|
|
|
210
234
|
{ id: 'relevance', label: 'Best match' }
|
|
211
235
|
] as const;
|
|
212
236
|
/** What a filter is called when a page says a server could not apply it. */
|
|
213
|
-
const FILTER_WORDS: Partial<Record<FilterName, string>> = { artifactTag: 'tag', type: 'media type', label: 'label', domain: 'domain', spec: 'spec' };
|
|
237
|
+
const FILTER_WORDS: Partial<Record<FilterName, string>> = { artifactTag: 'tag', type: 'media type', label: 'label', domain: 'domain', spec: 'spec', recordings: 'recording completeness' };
|
|
214
238
|
/** Says which filters the list is not narrowed by, because the server that answered ignored them. */
|
|
215
239
|
export function NotApplied({ names, noun }: { names: readonly FilterName[]; noun: string }) {
|
|
216
240
|
if (!names.length) return null;
|
|
@@ -383,6 +407,7 @@ function RunTable() {
|
|
|
383
407
|
<Table.Td>{formatMs(run.spanMs)}</Table.Td>
|
|
384
408
|
<Table.Td>
|
|
385
409
|
<RunVideos run={run} />
|
|
410
|
+
<RunRecordings run={run} />
|
|
386
411
|
</Table.Td>
|
|
387
412
|
<Table.Td data-tags>{run.artifactTags?.map((tag) => `#${tag}`).join(' ') || '—'}</Table.Td>
|
|
388
413
|
<Table.Td>
|
|
@@ -471,7 +496,7 @@ function RunDetailView({ runKey }: { runKey: string }) {
|
|
|
471
496
|
),
|
|
472
497
|
[narrowedResults, status, statusFilters, model]
|
|
473
498
|
);
|
|
474
|
-
const
|
|
499
|
+
const watch = useWatchAddress();
|
|
475
500
|
const playlist = useMemo(() => entriesFromRun(filtered, detail.data?.layers), [filtered, detail.data]);
|
|
476
501
|
const bySpec = useMemo(() => {
|
|
477
502
|
const groups = new Map<string, RunResult[]>();
|
|
@@ -651,11 +676,11 @@ function RunDetailView({ runKey }: { runKey: string }) {
|
|
|
651
676
|
</Text>
|
|
652
677
|
)}
|
|
653
678
|
{filtered.length > 0 && (
|
|
654
|
-
<Button type="button" variant="filled" my="sm" leftSection={<IconPlayerPlay size={16} aria-hidden="true" />} onClick={() =>
|
|
679
|
+
<Button type="button" variant="filled" my="sm" leftSection={<IconPlayerPlay size={16} aria-hidden="true" />} onClick={() => watch.open()}>
|
|
655
680
|
Watch these {plural(filtered.length, 'result', 'results')}
|
|
656
681
|
</Button>
|
|
657
682
|
)}
|
|
658
|
-
{watching && <WatchPlaylist entries={playlist}
|
|
683
|
+
{watch.watching && <WatchPlaylist entries={playlist} initial={watch.clip} onSelect={watch.select} onClose={watch.close} />}
|
|
659
684
|
{bySpec.map(([spec, list]) => (
|
|
660
685
|
<Box key={spec} data-open aria-label={spec || 'No spec'} component="section" mb="lg">
|
|
661
686
|
<Button data-static variant="default" fullWidth h="auto" py="sm" justify="flex-start">
|
|
@@ -22,7 +22,7 @@ import { reportAssetUrl, reportJson, ReportsError } from '../evidence/api';
|
|
|
22
22
|
import { exactTime, plural, shortSha, statusWord } from '../evidence/format';
|
|
23
23
|
import { StatusMark } from '../evidence/marks';
|
|
24
24
|
import { formatTimecode, parseTagsField, parseTimecode, sourceSpread } from './collection-model';
|
|
25
|
-
import { patched, useExplorerNavigate, useExplorerSearch } from './navigate';
|
|
25
|
+
import { patched, useExplorerNavigate, useExplorerSearch, useWatchAddress } from './navigate';
|
|
26
26
|
import { runName } from './parts';
|
|
27
27
|
import { WatchPlaylist } from './playlist';
|
|
28
28
|
import { entriesFromCollection, RECORDING_STATE_TEXT, recordingState } from './playlist-entries';
|
|
@@ -155,7 +155,7 @@ function CollectionPage({ id, revision }: { id: string; revision: number | undef
|
|
|
155
155
|
const go = useExplorerNavigate();
|
|
156
156
|
const client = useQueryClient();
|
|
157
157
|
const [token, setToken] = usePublishToken();
|
|
158
|
-
const
|
|
158
|
+
const watch = useWatchAddress();
|
|
159
159
|
const detail = useQuery({
|
|
160
160
|
queryKey: ['review-collection', id, revision ?? 'newest'],
|
|
161
161
|
queryFn: ({ signal }) => reportJson<ReviewCollectionDetail>(`/review-collections/detail?id=${encodeURIComponent(id)}${revision ? `&revision=${revision}` : ''}`, signal),
|
|
@@ -235,7 +235,7 @@ function CollectionPage({ id, revision }: { id: string; revision: number | undef
|
|
|
235
235
|
</Text>
|
|
236
236
|
</Alert>
|
|
237
237
|
<Group gap="sm" align="flex-end" wrap="wrap">
|
|
238
|
-
<Button type="button" variant="filled" leftSection={<IconPlayerPlay size={16} aria-hidden="true" />} onClick={() =>
|
|
238
|
+
<Button type="button" variant="filled" leftSection={<IconPlayerPlay size={16} aria-hidden="true" />} onClick={() => watch.open()} disabled={collection.entries.length === 0}>
|
|
239
239
|
Watch these {plural(collection.entries.length, 'entry', 'entries')}
|
|
240
240
|
</Button>
|
|
241
241
|
<Button component="a" href={exportHref} download leftSection={<IconDownload size={16} aria-hidden="true" />}>
|
|
@@ -244,7 +244,7 @@ function CollectionPage({ id, revision }: { id: string; revision: number | undef
|
|
|
244
244
|
<TextInput label="Share link" description="Opening it needs the same read access as this report." value={shareLink} readOnly onFocus={(event) => event.currentTarget.select()} miw={280} />
|
|
245
245
|
</Group>
|
|
246
246
|
<TokenField token={token} onChange={setToken} />
|
|
247
|
-
{watching
|
|
247
|
+
{watch.watching && <WatchPlaylist title={collection.title} entries={playlist} initial={watch.clip} onSelect={watch.select} onClose={watch.close} />}
|
|
248
248
|
{notesOf(undefined).length > 0 && (
|
|
249
249
|
<Stack gap="xs" component="section" aria-label="Notes on the collection">
|
|
250
250
|
<Title order={3}>Notes on the collection</Title>
|
|
@@ -309,7 +309,7 @@ function CollectionPage({ id, revision }: { id: string; revision: number | undef
|
|
|
309
309
|
{RECORDING_STATE_TEXT[state].label}
|
|
310
310
|
</Text>
|
|
311
311
|
{state === 'playable' && (
|
|
312
|
-
<Button type="button" size="compact-sm" variant="light" leftSection={<IconPlayerPlay size={14} aria-hidden="true" />} onClick={() =>
|
|
312
|
+
<Button type="button" size="compact-sm" variant="light" leftSection={<IconPlayerPlay size={14} aria-hidden="true" />} onClick={() => watch.open(entry.id)}>
|
|
313
313
|
Watch entry {index + 1}
|
|
314
314
|
</Button>
|
|
315
315
|
)}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @module Run and artifact filters
|
|
3
3
|
*
|
|
4
4
|
* The filter bar of the Runs and Artifacts tabs: words, artifact tag, media type, label, environment, branch, outcome,
|
|
5
|
-
* feature domain and a time range, each offering the values the report has with counts in the scope of the list. What is
|
|
5
|
+
* feature domain and a time range (and, for runs, the completeness of their recordings, apart from their outcome), each offering the values the report has with counts in the scope of the list. What is
|
|
6
6
|
* chosen shows as removable chips and is the address, so copying it or reloading restores the same view; named views keep
|
|
7
7
|
* a query to open again. Choices combine with AND across the controls and OR within one.
|
|
8
8
|
*/
|
|
@@ -18,7 +18,7 @@ import { patched, useExplorerNavigate } from './navigate';
|
|
|
18
18
|
import { filtered, joined, NO_FILTERS, toggled, values, type ExplorerSearch } from './search';
|
|
19
19
|
import { filtersOf, isView, readViews, writeViews, type SavedView } from './views';
|
|
20
20
|
|
|
21
|
-
type Param = 'artifactTag' | 'type' | 'label' | 'env' | 'branch' | 'outcome' | 'domain' | 'producer';
|
|
21
|
+
type Param = 'artifactTag' | 'type' | 'label' | 'env' | 'branch' | 'outcome' | 'recordings' | 'domain' | 'producer';
|
|
22
22
|
|
|
23
23
|
const NOUN: Record<Param, string> = {
|
|
24
24
|
artifactTag: 'Tag',
|
|
@@ -27,13 +27,18 @@ const NOUN: Record<Param, string> = {
|
|
|
27
27
|
env: 'Environment',
|
|
28
28
|
branch: 'Branch',
|
|
29
29
|
outcome: 'Outcome',
|
|
30
|
+
recordings: 'Recordings',
|
|
30
31
|
domain: 'Domain',
|
|
31
32
|
producer: 'Producer'
|
|
32
33
|
};
|
|
33
34
|
|
|
35
|
+
/** How the completeness of a run's recordings reads. */
|
|
36
|
+
const COMPLETENESS: Record<string, string> = { complete: 'Complete', incomplete: 'Incomplete', 'none-expected': 'None expected', unknown: 'Unknown' };
|
|
37
|
+
|
|
34
38
|
/** How a chosen value reads: tags with their hash, environments by name. */
|
|
35
39
|
function shown(param: Param, value: string): string {
|
|
36
40
|
if (param === 'artifactTag') return `#${value}`;
|
|
41
|
+
if (param === 'recordings') return COMPLETENESS[value] ?? value;
|
|
37
42
|
if (param === 'env') return environmentLabel(value);
|
|
38
43
|
if (param === 'domain' && value === '(no spec)') return 'No spec';
|
|
39
44
|
return value;
|
|
@@ -178,7 +183,7 @@ export function FilterBar({
|
|
|
178
183
|
const [draft, setDraft] = useState(search.q ?? '');
|
|
179
184
|
// The address may change from elsewhere (a link, a saved view, back): follow it.
|
|
180
185
|
useEffect(() => setDraft((current) => (current.trim() === (search.q ?? '').trim() ? current : (search.q ?? ''))), [search.q]);
|
|
181
|
-
const params: Param[] = ['artifactTag', 'type', 'label', 'env', 'branch', 'outcome', ...(tab === 'artifacts' ? (['domain'] as const) : []), 'producer'];
|
|
186
|
+
const params: Param[] = ['artifactTag', 'type', 'label', 'env', 'branch', 'outcome', ...(tab === 'runs' ? (['recordings'] as const) : []), ...(tab === 'artifacts' ? (['domain'] as const) : []), 'producer'];
|
|
182
187
|
const chips = [
|
|
183
188
|
...params.flatMap((param) =>
|
|
184
189
|
values(search[param]).map((value) => ({ label: `${NOUN[param]}: ${shown(param, value)}`, clear: { [param]: toggled(search[param], value) } as never }))
|
|
@@ -233,6 +238,9 @@ export function FilterBar({
|
|
|
233
238
|
<Facet param="env" facets={facets.environments} search={search} />
|
|
234
239
|
<Facet param="branch" facets={facets.branches} search={search} />
|
|
235
240
|
<Facet param="outcome" facets={facets.outcomes} search={search} />
|
|
241
|
+
{tab === 'runs' && (facets.recordings || values(search.recordings).length > 0) && (
|
|
242
|
+
<Facet param="recordings" facets={facets.recordings ?? []} search={search} />
|
|
243
|
+
)}
|
|
236
244
|
{tab === 'artifacts' && facets.domains && <Facet param="domain" facets={facets.domains} search={search} />}
|
|
237
245
|
{facets.producers.length > 1 && <Facet param="producer" facets={facets.producers} search={search} />}
|
|
238
246
|
<TextInput type="date" label="From" value={search.since ?? ''} onChange={(event) => go({ since: event.target.value || undefined })} />
|
|
@@ -34,3 +34,20 @@ export function useExplorerNavigate() {
|
|
|
34
34
|
export function patched(patch: ExplorerPatch) {
|
|
35
35
|
return ((prev: Record<string, unknown>) => ({ ...prev, ...patch })) as never;
|
|
36
36
|
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The open watch playlist as the address holds it, so a link restores it: `watching` is true while the address says it is
|
|
40
|
+
* open and `clip` is the entry it is on. `open` pushes a history entry (Back closes it), `select` and `close` replace the
|
|
41
|
+
* current one, so stepping through clips does not fill the history. `open(clip)` starts on that entry.
|
|
42
|
+
*/
|
|
43
|
+
export function useWatchAddress() {
|
|
44
|
+
const search = useExplorerSearch();
|
|
45
|
+
const go = useExplorerNavigate();
|
|
46
|
+
return {
|
|
47
|
+
watching: search.watch === true,
|
|
48
|
+
clip: search.watchClip,
|
|
49
|
+
open: (clip?: string) => go({ watch: true, watchClip: clip }),
|
|
50
|
+
select: (clip: string) => go({ watchClip: clip }, { replace: true }),
|
|
51
|
+
close: () => go({ watch: undefined, watchClip: undefined }, { replace: true })
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { VideoRef } from '@crvouga/atlas/reports/evidence-types';
|
|
2
2
|
import { expect, test } from 'bun:test';
|
|
3
3
|
|
|
4
|
-
import { entriesFromRun, playlistCounts, recordingState, stepFrom, type PlaylistEntry } from './playlist-entries';
|
|
4
|
+
import { clipHints, entriesFromRun, playlistCounts, recordingState, stepFrom, type PlaylistEntry } from './playlist-entries';
|
|
5
5
|
|
|
6
6
|
const video = (recording?: string): VideoRef => ({ name: 'a.webm', owner: 'check', available: recording !== undefined, ...(recording ? { recording } : {}) });
|
|
7
7
|
const entry = (id: string, extra: Partial<PlaylistEntry> = {}): PlaylistEntry => ({
|
|
@@ -49,3 +49,12 @@ test('a run result gets its backend from the layers of the run, and none when th
|
|
|
49
49
|
expect(entriesFromRun([result], undefined)[0]!.backend).toBeNull();
|
|
50
50
|
expect(recordingState(entriesFromRun([result], undefined)[0]!)).toBe('unplayable');
|
|
51
51
|
});
|
|
52
|
+
|
|
53
|
+
test('clip hints are the poster and length the catalog already named for the clip that would play, and nothing otherwise', () => {
|
|
54
|
+
const hinted: VideoRef = { ...video('sha256:1'), poster: 'attachments/p', durationMs: 4_000 };
|
|
55
|
+
expect(clipHints(entry('a', { videos: [hinted] }))).toEqual({ poster: 'attachments/p', durationMs: 4_000 });
|
|
56
|
+
expect(clipHints(entry('a', { videos: [video('sha256:1')] }))).toEqual({});
|
|
57
|
+
// A reference that names no bytes is not the clip that would play.
|
|
58
|
+
expect(clipHints(entry('a', { videos: [{ ...video(), durationMs: 9 }] }))).toEqual({});
|
|
59
|
+
expect(clipHints(entry('a'))).toEqual({});
|
|
60
|
+
});
|
|
@@ -68,6 +68,16 @@ export function playableVideo(entry: Pick<PlaylistEntry, 'videos'>): VideoRef |
|
|
|
68
68
|
return entry.videos.find((video) => video.available && video.recording !== undefined);
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* What the catalog already says about the clip an entry would play, before anything is read or downloaded: the bundle key
|
|
73
|
+
* of its poster image and its declared length. Each is absent when the producer declared none (or the server predates the
|
|
74
|
+
* hints); they are hints to recognise a clip, never a reading of the recording and never a latency.
|
|
75
|
+
*/
|
|
76
|
+
export function clipHints(entry: Pick<PlaylistEntry, 'videos'>): { readonly poster?: string; readonly durationMs?: number } {
|
|
77
|
+
const video = playableVideo(entry);
|
|
78
|
+
return { ...(video?.poster ? { poster: video.poster } : {}), ...(video?.durationMs ? { durationMs: video.durationMs } : {}) };
|
|
79
|
+
}
|
|
80
|
+
|
|
71
81
|
/** The recorded results of the Artifacts tab as playlist entries, in their order. */
|
|
72
82
|
export function entriesFromArtifacts(results: readonly ArtifactResult[]): PlaylistEntry[] {
|
|
73
83
|
return results.map((result) => ({
|