@crvouga/atlas 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/reports/src/evidence-detail.d.ts +14 -3
- package/dist/reports/src/evidence-detail.js +33 -19
- package/dist/reports/src/evidence-scope.d.ts +83 -0
- package/dist/reports/src/evidence-scope.js +129 -0
- package/dist/reports/src/evidence-types.d.ts +3 -1
- package/dist/reports/src/explorer-types.d.ts +1 -1
- package/dist/reports/src/layer-merge.js +3 -2
- package/dist/reports/src/layer-types.d.ts +1 -1
- package/dist/reports/src/publish-checks.js +1 -0
- package/dist/reports/src/reporter.d.ts +1 -1
- package/dist/reports/src/reports-explorer-route.d.ts +276 -0
- package/dist/reports/src/reports-route.d.ts +276 -0
- package/dist/reports/src/test-catalog.d.ts +7 -1
- package/dist/reports/src/test-catalog.js +15 -3
- package/dist/schema/src/api.d.ts +520 -0
- package/dist/schema/src/api.js +7 -4
- package/dist/schema/src/layer.d.ts +981 -0
- package/dist/schema/src/layer.js +43 -4
- package/dist/server/src/openapi.js +1 -0
- package/docs/extending.md +1 -0
- package/docs/reference/http-api.md +2 -0
- package/docs/reference/reports.md +179 -7
- package/docs/reference/schema.md +23 -0
- package/docs/rich-evidence.md +47 -0
- package/llms-full.txt +50 -0
- package/package.json +1 -1
- package/packages/schema/schemas/check-detail.schema.json +673 -1
- package/packages/schema/schemas/effective-delta.schema.json +192 -0
- package/packages/schema/schemas/effective-evidence.schema.json +192 -0
- package/packages/schema/schemas/evidence-page.schema.json +96 -0
- package/packages/schema/schemas/layer-rows.schema.json +96 -0
- package/packages/schema/schemas/layer.schema.json +1598 -62
- package/packages/schema/schemas/publish-checks.schema.json +768 -0
- package/packages/schema/schemas/report-run.schema.json +96 -0
- package/packages/schema/schemas/test-catalog.schema.json +128 -0
- package/packages/schema/schemas/test-history.schema.json +202 -0
- package/packages/server/openapi.json +2581 -23
- package/packages/visualizer/src/evidence/artifacts.tsx +24 -18
- package/packages/visualizer/src/evidence/evidence-scope.tsx +106 -0
- package/packages/visualizer/src/evidence/format.ts +11 -0
- package/packages/visualizer/src/explorer/Explorer.tsx +19 -0
- package/packages/visualizer/src/explorer/search.ts +5 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog: @crvouga/atlas
|
|
2
2
|
|
|
3
|
+
## 1.3.0 (2026-10-28)
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
|
|
7
|
+
- expose readable evidence scope for synthetic, scripted, simulator and physical runs ([fae5d02](https://github.com/crvouga/atlas/commit/fae5d02c7bdaecdfdf4ec113b23f6b11f9e3e012))
|
|
8
|
+
|
|
3
9
|
## 1.2.0 (2026-10-28)
|
|
4
10
|
|
|
5
11
|
### Features
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* @see docs/extending.md
|
|
10
10
|
* @see docs/report-contract.md (evidence)
|
|
11
11
|
*/
|
|
12
|
-
import type { Annotation, Attachment, Attempt, Check, Step } from '../../schema/src/layer.js';
|
|
12
|
+
import type { Annotation, Attachment, Attempt, Check, EvidenceScope, Step } from '../../schema/src/layer.js';
|
|
13
13
|
import type { EvidenceSummary, VideoCounts, VideoRef } from './evidence-types.js';
|
|
14
14
|
/** Anything that may carry evidence: a layer, a part or a check. */
|
|
15
15
|
export type EvidenceHolder = {
|
|
@@ -25,6 +25,8 @@ export type EvidenceHolder = {
|
|
|
25
25
|
readonly steps?: readonly Step[];
|
|
26
26
|
readonly attempts?: readonly Attempt[];
|
|
27
27
|
readonly extensions?: Readonly<Record<string, unknown>>;
|
|
28
|
+
/** What its evidence shows, as its producer declared it (see `EvidenceScope`). */
|
|
29
|
+
readonly scope?: EvidenceScope;
|
|
28
30
|
};
|
|
29
31
|
/**
|
|
30
32
|
* What a result carries beyond its outcome, as counts and short facts. It rides in every row of the
|
|
@@ -35,9 +37,15 @@ export type EvidenceHolder = {
|
|
|
35
37
|
* attempts, `steps` counts only each holder's own top-level steps, tags are de-duplicated, and for a repeated
|
|
36
38
|
* label the later holder wins. `artifactTags` are the attachments' own normalized tags and are never inherited. `roles` are the attachments' roles (or, without one, what their media type or file
|
|
37
39
|
* name implies), sorted and distinct. `videos` lists every attachment whose effective media type is video, with the
|
|
38
|
-
* check, retry attempt or step that holds it (see {@link isVideoAttachment}).
|
|
40
|
+
* check, retry attempt or step that holds it (see {@link isVideoAttachment}), each with the scope that applies to it.
|
|
39
41
|
*/
|
|
40
42
|
export declare function summarizeEvidence(...holders: readonly EvidenceHolder[]): EvidenceSummary | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Like {@link summarizeEvidence} for the checks of a part: a video that declares no scope, and sits in a holder that
|
|
45
|
+
* declares none, takes the scope `inherited` from its part or layer (the nearest declaration of the two, see
|
|
46
|
+
* `effectiveScope`). Without `inherited` it is the same as `summarizeEvidence`.
|
|
47
|
+
*/
|
|
48
|
+
export declare function summarizeEvidenceIn(inherited: EvidenceScope | undefined, ...holders: readonly EvidenceHolder[]): EvidenceSummary | undefined;
|
|
41
49
|
/**
|
|
42
50
|
* Like {@link summarizeEvidence} for a layer and its parts: the same summary, but a video attached to the layer or a
|
|
43
51
|
* part is owned by the `layer`, not by a check.
|
|
@@ -66,6 +74,8 @@ export type CheckDetail = {
|
|
|
66
74
|
readonly check: Check;
|
|
67
75
|
/** What the run as a whole carries: its logs, its links, its labels. */
|
|
68
76
|
readonly run: EvidenceHolder;
|
|
77
|
+
/** The scope the check inherits: its part's declaration, else its layer's. Absent when neither declared one, which is unknown, not live or physical. */
|
|
78
|
+
readonly inheritedScope?: EvidenceScope;
|
|
69
79
|
};
|
|
70
80
|
/** One result's evidence, from a layer: the first check with id `checkId` (in a part of kind `kind`, when given) plus the layer's own evidence. Null when the layer holds no such check. */
|
|
71
81
|
export declare function checkDetailOf(layer: {
|
|
@@ -73,9 +83,10 @@ export declare function checkDetailOf(layer: {
|
|
|
73
83
|
readonly parts: readonly {
|
|
74
84
|
readonly kind: string;
|
|
75
85
|
readonly checks: readonly Check[];
|
|
86
|
+
readonly scope?: EvidenceScope;
|
|
76
87
|
}[];
|
|
77
88
|
} & EvidenceHolder, checkId: string, kind?: string): CheckDetail | null;
|
|
78
|
-
/** What a layer carries as a whole (its tags, labels, links, attachments, annotations and extensions; not its parts' checks). Absent fields stay absent. */
|
|
89
|
+
/** What a layer carries as a whole (its scope, tags, labels, links, attachments, annotations and extensions; not its parts' checks). Absent fields stay absent. */
|
|
79
90
|
export declare function runEvidence(layer: EvidenceHolder): EvidenceHolder;
|
|
80
91
|
/** Every distinct bundle key (`bundleKey`) that the attachments of a holder, its steps and its attempts point to, so a layer's files can be copied. Attachments held by `uri` or inline `text` have none. */
|
|
81
92
|
export declare function bundleKeysOf(holder: EvidenceHolder): string[];
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { normalizeTags } from './artifact-tags.js';
|
|
2
|
+
import { effectiveScope } from './evidence-scope.js';
|
|
2
3
|
import { contentTypeOf } from './media-types.js';
|
|
3
4
|
function count(holder) {
|
|
4
5
|
return (holder.attachments?.length ?? 0) + (holder.steps?.length ?? 0) + (holder.annotations?.length ?? 0) + (holder.attempts?.length ?? 0) + (holder.tags?.length ?? 0) + (holder.links?.length ?? 0) + Object.keys(holder.labels ?? {}).length + Object.keys(holder.extensions ?? {}).length;
|
|
@@ -12,19 +13,28 @@ function count(holder) {
|
|
|
12
13
|
* attempts, `steps` counts only each holder's own top-level steps, tags are de-duplicated, and for a repeated
|
|
13
14
|
* label the later holder wins. `artifactTags` are the attachments' own normalized tags and are never inherited. `roles` are the attachments' roles (or, without one, what their media type or file
|
|
14
15
|
* name implies), sorted and distinct. `videos` lists every attachment whose effective media type is video, with the
|
|
15
|
-
* check, retry attempt or step that holds it (see {@link isVideoAttachment}).
|
|
16
|
+
* check, retry attempt or step that holds it (see {@link isVideoAttachment}), each with the scope that applies to it.
|
|
16
17
|
*/
|
|
17
18
|
export function summarizeEvidence(...holders) {
|
|
18
19
|
return summarize(holders, 'check');
|
|
19
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* Like {@link summarizeEvidence} for the checks of a part: a video that declares no scope, and sits in a holder that
|
|
23
|
+
* declares none, takes the scope `inherited` from its part or layer (the nearest declaration of the two, see
|
|
24
|
+
* `effectiveScope`). Without `inherited` it is the same as `summarizeEvidence`.
|
|
25
|
+
*/
|
|
26
|
+
export function summarizeEvidenceIn(inherited, ...holders) {
|
|
27
|
+
return summarize(holders, 'check', inherited);
|
|
28
|
+
}
|
|
20
29
|
/**
|
|
21
30
|
* Like {@link summarizeEvidence} for a layer and its parts: the same summary, but a video attached to the layer or a
|
|
22
31
|
* part is owned by the `layer`, not by a check.
|
|
23
32
|
*/
|
|
24
33
|
export function summarizeLayerEvidence(...holders) {
|
|
25
|
-
|
|
34
|
+
// The first holder is the layer: its scope reaches the parts after it, which may declare their own.
|
|
35
|
+
return summarize(holders, 'layer', undefined, true);
|
|
26
36
|
}
|
|
27
|
-
function summarize(holders, owner) {
|
|
37
|
+
function summarize(holders, owner, inherited, firstIsParent = false) {
|
|
28
38
|
if (!holders.some(holder => count(holder) > 0))
|
|
29
39
|
return undefined;
|
|
30
40
|
const roles = new Set();
|
|
@@ -33,26 +43,28 @@ function summarize(holders, owner) {
|
|
|
33
43
|
const labels = {};
|
|
34
44
|
const videos = [];
|
|
35
45
|
let attachments = 0, steps = 0, annotations = 0, attempts = 0, links = 0;
|
|
36
|
-
const attachmentsOf = (list, by, where = {}) => {
|
|
46
|
+
const attachmentsOf = (list, by, scopes, where = {}) => {
|
|
37
47
|
for (const item of list ?? []) {
|
|
38
48
|
attachments += 1;
|
|
39
49
|
for (const tag of normalizeTags(item.tags))
|
|
40
50
|
artifactTags.add(tag);
|
|
41
51
|
roles.add(item.role ?? roleOfType(item.contentType ?? contentTypeOf(item.name)));
|
|
42
|
-
const video = videoOf(item, by, where);
|
|
52
|
+
const video = videoOf(item, by, where, effectiveScope(item.scope, ...scopes));
|
|
43
53
|
if (video)
|
|
44
54
|
videos.push(video);
|
|
45
55
|
}
|
|
46
56
|
};
|
|
47
|
-
const stepsOf = (list, attempt) => { for (const step of list ?? [])
|
|
48
|
-
attachmentsOf(step.attachments, 'step', { ...(attempt === undefined ? {} : { attempt }), step: step.name }); };
|
|
49
|
-
for (const holder of holders) {
|
|
50
|
-
|
|
57
|
+
const stepsOf = (list, scopes, attempt) => { for (const step of list ?? [])
|
|
58
|
+
attachmentsOf(step.attachments, 'step', scopes, { ...(attempt === undefined ? {} : { attempt }), step: step.name }); };
|
|
59
|
+
for (const [index, holder] of holders.entries()) {
|
|
60
|
+
const around = [holder.scope, inherited, ...(firstIsParent && index > 0 ? [holders[0]?.scope] : [])];
|
|
61
|
+
attachmentsOf(holder.attachments, owner, around);
|
|
51
62
|
// Attachments on steps and attempts are part of the check too.
|
|
52
|
-
stepsOf(holder.steps);
|
|
63
|
+
stepsOf(holder.steps, around);
|
|
53
64
|
for (const attempt of holder.attempts ?? []) {
|
|
54
|
-
|
|
55
|
-
|
|
65
|
+
const tried = [attempt.scope, ...around];
|
|
66
|
+
attachmentsOf(attempt.attachments, 'attempt', tried, { attempt: attempt.index });
|
|
67
|
+
stepsOf(attempt.steps, tried, attempt.index);
|
|
56
68
|
}
|
|
57
69
|
steps += holder.steps?.length ?? 0;
|
|
58
70
|
annotations += holder.annotations?.length ?? 0;
|
|
@@ -73,12 +85,12 @@ export function isVideoAttachment(attachment) {
|
|
|
73
85
|
const type = (attachment.contentType ?? contentTypeOf(attachment.name)).toLowerCase();
|
|
74
86
|
return type.startsWith('video/') || (attachment.role === 'video' && type.startsWith('application/octet-stream'));
|
|
75
87
|
}
|
|
76
|
-
function videoOf(item, owner, where) {
|
|
88
|
+
function videoOf(item, owner, where, scope) {
|
|
77
89
|
if (!isVideoAttachment(item))
|
|
78
90
|
return undefined;
|
|
79
91
|
const recording = item.sha256 ? `sha256:${item.sha256}` : item.bundleKey ?? item.uri ?? (item.text === undefined ? undefined : `inline:${item.name}`);
|
|
80
92
|
const tags = normalizeTags(item.tags);
|
|
81
|
-
return { name: item.name, owner, ...where, ...(tags.length ? { tags } : {}), ...(recording === undefined ? {} : { recording }), available: recording !== undefined };
|
|
93
|
+
return { name: item.name, owner, ...where, ...(tags.length ? { tags } : {}), ...(recording === undefined ? {} : { recording }), available: recording !== undefined, ...(scope ? { scope } : {}) };
|
|
82
94
|
}
|
|
83
95
|
/**
|
|
84
96
|
* Counts references, distinct playable recordings and unplayable references. Undefined when there is no reference, so
|
|
@@ -118,15 +130,17 @@ export function checkDetailOf(layer, checkId, kind) {
|
|
|
118
130
|
if (kind !== undefined && part.kind !== kind)
|
|
119
131
|
continue;
|
|
120
132
|
const check = part.checks.find(item => item.id === checkId);
|
|
121
|
-
if (check)
|
|
122
|
-
|
|
133
|
+
if (check) {
|
|
134
|
+
const inheritedScope = effectiveScope(part.scope, layer.scope);
|
|
135
|
+
return { layer: layer.id, kind: part.kind, check, run: runEvidence(layer), ...(inheritedScope ? { inheritedScope } : {}) };
|
|
136
|
+
}
|
|
123
137
|
}
|
|
124
138
|
return null;
|
|
125
139
|
}
|
|
126
|
-
/** What a layer carries as a whole (its tags, labels, links, attachments, annotations and extensions; not its parts' checks). Absent fields stay absent. */
|
|
140
|
+
/** What a layer carries as a whole (its scope, tags, labels, links, attachments, annotations and extensions; not its parts' checks). Absent fields stay absent. */
|
|
127
141
|
export function runEvidence(layer) {
|
|
128
|
-
const { tags, labels, links, attachments, annotations, extensions } = layer;
|
|
129
|
-
return { ...(tags ? { tags } : {}), ...(labels ? { labels } : {}), ...(links ? { links } : {}), ...(attachments ? { attachments } : {}), ...(annotations ? { annotations } : {}), ...(extensions ? { extensions } : {}) };
|
|
142
|
+
const { tags, labels, links, attachments, annotations, extensions, scope } = layer;
|
|
143
|
+
return { ...(scope ? { scope } : {}), ...(tags ? { tags } : {}), ...(labels ? { labels } : {}), ...(links ? { links } : {}), ...(attachments ? { attachments } : {}), ...(annotations ? { annotations } : {}), ...(extensions ? { extensions } : {}) };
|
|
130
144
|
}
|
|
131
145
|
/** Every distinct bundle key (`bundleKey`) that the attachments of a holder, its steps and its attempts point to, so a layer's files can be copied. Attachments held by `uri` or inline `text` have none. */
|
|
132
146
|
export function bundleKeysOf(holder) {
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Evidence scope
|
|
3
|
+
*
|
|
4
|
+
* Reads what a producer declared a piece of evidence shows (`EvidenceScope`): which declaration applies to an
|
|
5
|
+
* attachment, the short badges a reviewer sees before playing a recording, the facts split into declared and
|
|
6
|
+
* independently verified, and the filter tokens (`device:physical`, `audio:recorded`) a query narrows by. Pure
|
|
7
|
+
* functions, safe in a browser.
|
|
8
|
+
*
|
|
9
|
+
* The one rule: what is not declared is unknown. A missing scope, or a missing field, never reads as live, physical,
|
|
10
|
+
* audible or end to end, and a filter for a value never matches evidence that did not say it. A scope describes
|
|
11
|
+
* evidence; it never decides a status, a slot or a winner.
|
|
12
|
+
*
|
|
13
|
+
* @see docs/rich-evidence.md#evidence-scope
|
|
14
|
+
*/
|
|
15
|
+
import type { EvidenceScope } from '../../schema/src/layer.js';
|
|
16
|
+
/** The fields of a scope that say something about the evidence and that `verified` may name, in the order they are shown. */
|
|
17
|
+
export declare const SCOPE_DIMENSIONS: readonly ["surface", "deviceKind", "syntheticData", "dependencies", "audioCaptured", "pacing"];
|
|
18
|
+
/** One of {@link SCOPE_DIMENSIONS}. */
|
|
19
|
+
export type ScopeDimension = (typeof SCOPE_DIMENSIONS)[number];
|
|
20
|
+
/** The filter token of a recording that declares no scope at all. */
|
|
21
|
+
export declare const UNKNOWN_SCOPE = "unknown";
|
|
22
|
+
/**
|
|
23
|
+
* The declaration that applies to something: the nearest one wins as a whole, never field by field, so a recording
|
|
24
|
+
* never shows a limitation or a device that was declared for another source. Pass the candidates nearest first
|
|
25
|
+
* (attachment, attempt, check, part, layer); absent ones are skipped. Undefined when none declares a scope.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```ts
|
|
29
|
+
* effectiveScope(attachment.scope, attempt.scope, check.scope, layer.scope);
|
|
30
|
+
* ```
|
|
31
|
+
*/
|
|
32
|
+
export declare function effectiveScope(...candidates: readonly (EvidenceScope | undefined)[]): EvidenceScope | undefined;
|
|
33
|
+
/** A stable string that is equal for equal declarations (key order never matters), to tell one scope from another. */
|
|
34
|
+
export declare function scopeKey(scope: EvidenceScope | undefined): string;
|
|
35
|
+
/**
|
|
36
|
+
* The filter tokens one declaration states, normalized (letter case and spaces never matter): `surface:<value>`,
|
|
37
|
+
* `device:<value>`, `data:synthetic` or `data:real`, `dependencies:<value>`, `audio:recorded` or `audio:silent`,
|
|
38
|
+
* `pacing:<value>`. A field that is not declared adds no token. No scope at all is the single token
|
|
39
|
+
* {@link UNKNOWN_SCOPE}, so a filter can ask for the unknown on purpose; a scope that declares nothing readable is
|
|
40
|
+
* also unknown.
|
|
41
|
+
*/
|
|
42
|
+
export declare function scopeTokens(scope: EvidenceScope | undefined): string[];
|
|
43
|
+
/**
|
|
44
|
+
* Whether one recording's tokens satisfy a filter: every wanted token must be among them. Tokens compare normalized.
|
|
45
|
+
* An empty filter matches everything. The token {@link UNKNOWN_SCOPE} matches only a recording with no declared
|
|
46
|
+
* fields, so asking for `device:physical` never includes a recording that did not say what it ran on.
|
|
47
|
+
*/
|
|
48
|
+
export declare function matchesScope(tokens: readonly string[], wanted: readonly string[]): boolean;
|
|
49
|
+
/** How a fact is known: `verified` (independently checked), `declared` (the producer's word) or `undeclared` (unknown). */
|
|
50
|
+
export type ScopeBasis = 'verified' | 'declared' | 'undeclared';
|
|
51
|
+
/** One line of what a scope says: the dimension, a sentence for a person, and how well it is known. */
|
|
52
|
+
export type ScopeFact = {
|
|
53
|
+
readonly dimension: ScopeDimension;
|
|
54
|
+
/** A name for the dimension: `Device`, `Audio`. */
|
|
55
|
+
readonly label: string;
|
|
56
|
+
/** The value as words: `Physical device`, `Silent`, `Not declared`. */
|
|
57
|
+
readonly value: string;
|
|
58
|
+
readonly basis: ScopeBasis;
|
|
59
|
+
};
|
|
60
|
+
/** A short qualification to show beside a recording: `Device: Simulator`, `Scripted dependencies`, `Silent`. */
|
|
61
|
+
export type ScopeBadge = {
|
|
62
|
+
readonly dimension: ScopeDimension | 'unknown';
|
|
63
|
+
readonly text: string;
|
|
64
|
+
readonly basis: ScopeBasis;
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* Every dimension of a scope with its value and how it is known, in a fixed order, including the ones nobody declared
|
|
68
|
+
* (`Not declared`, basis `undeclared`). A dimension is `verified` only when the scope's `verified` names it and it has
|
|
69
|
+
* a value; everything else a producer wrote is `declared`. With no scope every dimension is undeclared.
|
|
70
|
+
*/
|
|
71
|
+
export declare function scopeFacts(scope: EvidenceScope | undefined): ScopeFact[];
|
|
72
|
+
/**
|
|
73
|
+
* The concise badges for a scope: one per declared dimension, in a fixed order (`Surface: Browser`, `Device: Simulator`, `Synthetic
|
|
74
|
+
* data`, `Scripted dependencies`, `Silent`, `Paced`). With no declared dimension the one badge `Scope unknown`, so
|
|
75
|
+
* a reviewer is told nothing was claimed rather than shown nothing.
|
|
76
|
+
*/
|
|
77
|
+
export declare function scopeBadges(scope: EvidenceScope | undefined): ScopeBadge[];
|
|
78
|
+
/**
|
|
79
|
+
* What a reviewer must not read into evidence: a sentence per thing that is not shown. It names the assertions the
|
|
80
|
+
* result stands on when there are some (so activity visible beyond them is not verified), and every kind of reality
|
|
81
|
+
* that was not declared. Empty only for a scope that declares and verifies everything and lists no limitation.
|
|
82
|
+
*/
|
|
83
|
+
export declare function scopeCaveats(scope: EvidenceScope | undefined): string[];
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/** The fields of a scope that say something about the evidence and that `verified` may name, in the order they are shown. */
|
|
2
|
+
export const SCOPE_DIMENSIONS = ['surface', 'deviceKind', 'syntheticData', 'dependencies', 'audioCaptured', 'pacing'];
|
|
3
|
+
/** The filter token of a recording that declares no scope at all. */
|
|
4
|
+
export const UNKNOWN_SCOPE = 'unknown';
|
|
5
|
+
/**
|
|
6
|
+
* The declaration that applies to something: the nearest one wins as a whole, never field by field, so a recording
|
|
7
|
+
* never shows a limitation or a device that was declared for another source. Pass the candidates nearest first
|
|
8
|
+
* (attachment, attempt, check, part, layer); absent ones are skipped. Undefined when none declares a scope.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* effectiveScope(attachment.scope, attempt.scope, check.scope, layer.scope);
|
|
13
|
+
* ```
|
|
14
|
+
*/
|
|
15
|
+
export function effectiveScope(...candidates) {
|
|
16
|
+
return candidates.find((candidate) => candidate !== undefined);
|
|
17
|
+
}
|
|
18
|
+
const word = (value) => value.normalize('NFC').trim().toLowerCase().replace(/\s+/g, '-');
|
|
19
|
+
/** A stable string that is equal for equal declarations (key order never matters), to tell one scope from another. */
|
|
20
|
+
export function scopeKey(scope) {
|
|
21
|
+
if (!scope)
|
|
22
|
+
return UNKNOWN_SCOPE;
|
|
23
|
+
const sort = (value) => Array.isArray(value) ? value.map(sort) : value && typeof value === 'object' ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([key, item]) => [key, sort(item)])) : value;
|
|
24
|
+
return JSON.stringify(sort(scope));
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The filter tokens one declaration states, normalized (letter case and spaces never matter): `surface:<value>`,
|
|
28
|
+
* `device:<value>`, `data:synthetic` or `data:real`, `dependencies:<value>`, `audio:recorded` or `audio:silent`,
|
|
29
|
+
* `pacing:<value>`. A field that is not declared adds no token. No scope at all is the single token
|
|
30
|
+
* {@link UNKNOWN_SCOPE}, so a filter can ask for the unknown on purpose; a scope that declares nothing readable is
|
|
31
|
+
* also unknown.
|
|
32
|
+
*/
|
|
33
|
+
export function scopeTokens(scope) {
|
|
34
|
+
const tokens = [];
|
|
35
|
+
if (scope?.surface)
|
|
36
|
+
tokens.push(`surface:${word(scope.surface)}`);
|
|
37
|
+
if (scope?.deviceKind)
|
|
38
|
+
tokens.push(`device:${word(scope.deviceKind)}`);
|
|
39
|
+
if (scope?.syntheticData !== undefined)
|
|
40
|
+
tokens.push(scope.syntheticData ? 'data:synthetic' : 'data:real');
|
|
41
|
+
if (scope?.dependencies)
|
|
42
|
+
tokens.push(`dependencies:${word(scope.dependencies)}`);
|
|
43
|
+
if (scope?.audioCaptured !== undefined)
|
|
44
|
+
tokens.push(scope.audioCaptured ? 'audio:recorded' : 'audio:silent');
|
|
45
|
+
if (scope?.pacing)
|
|
46
|
+
tokens.push(`pacing:${word(scope.pacing)}`);
|
|
47
|
+
return tokens.length ? tokens : [UNKNOWN_SCOPE];
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Whether one recording's tokens satisfy a filter: every wanted token must be among them. Tokens compare normalized.
|
|
51
|
+
* An empty filter matches everything. The token {@link UNKNOWN_SCOPE} matches only a recording with no declared
|
|
52
|
+
* fields, so asking for `device:physical` never includes a recording that did not say what it ran on.
|
|
53
|
+
*/
|
|
54
|
+
export function matchesScope(tokens, wanted) {
|
|
55
|
+
const have = new Set(tokens.map(word));
|
|
56
|
+
return wanted.every(token => have.has(word(token)));
|
|
57
|
+
}
|
|
58
|
+
const title = (value) => { const text = value.replace(/[-_]+/g, ' ').trim(); return text.charAt(0).toUpperCase() + text.slice(1); };
|
|
59
|
+
const LABELS = { surface: 'Surface', deviceKind: 'Device', syntheticData: 'Data', dependencies: 'Dependencies', audioCaptured: 'Audio', pacing: 'Pacing' };
|
|
60
|
+
function valueOf(scope, dimension) {
|
|
61
|
+
switch (dimension) {
|
|
62
|
+
case 'surface': return scope.surface ? title(scope.surface) : undefined;
|
|
63
|
+
case 'deviceKind': return scope.deviceKind ? title(scope.deviceKind) : undefined;
|
|
64
|
+
case 'syntheticData': return scope.syntheticData === undefined ? undefined : scope.syntheticData ? 'Synthetic' : 'Real';
|
|
65
|
+
case 'dependencies': return scope.dependencies ? title(scope.dependencies) : undefined;
|
|
66
|
+
case 'audioCaptured': return scope.audioCaptured === undefined ? undefined : scope.audioCaptured ? 'Recorded' : 'Silent';
|
|
67
|
+
case 'pacing': return scope.pacing ? title(scope.pacing) : undefined;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Every dimension of a scope with its value and how it is known, in a fixed order, including the ones nobody declared
|
|
72
|
+
* (`Not declared`, basis `undeclared`). A dimension is `verified` only when the scope's `verified` names it and it has
|
|
73
|
+
* a value; everything else a producer wrote is `declared`. With no scope every dimension is undeclared.
|
|
74
|
+
*/
|
|
75
|
+
export function scopeFacts(scope) {
|
|
76
|
+
const verified = new Set(scope?.verified ?? []);
|
|
77
|
+
return SCOPE_DIMENSIONS.map(dimension => {
|
|
78
|
+
const value = scope ? valueOf(scope, dimension) : undefined;
|
|
79
|
+
return { dimension, label: LABELS[dimension], value: value ?? 'Not declared', basis: value === undefined ? 'undeclared' : verified.has(dimension) ? 'verified' : 'declared' };
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* The concise badges for a scope: one per declared dimension, in a fixed order (`Surface: Browser`, `Device: Simulator`, `Synthetic
|
|
84
|
+
* data`, `Scripted dependencies`, `Silent`, `Paced`). With no declared dimension the one badge `Scope unknown`, so
|
|
85
|
+
* a reviewer is told nothing was claimed rather than shown nothing.
|
|
86
|
+
*/
|
|
87
|
+
export function scopeBadges(scope) {
|
|
88
|
+
const badges = [];
|
|
89
|
+
for (const fact of scopeFacts(scope)) {
|
|
90
|
+
if (fact.basis === 'undeclared')
|
|
91
|
+
continue;
|
|
92
|
+
const text = fact.dimension === 'surface' || fact.dimension === 'deviceKind' ? `${fact.label}: ${fact.value}`
|
|
93
|
+
: fact.dimension === 'syntheticData' ? `${fact.value} data`
|
|
94
|
+
: fact.dimension === 'dependencies' ? `${fact.value} dependencies`
|
|
95
|
+
: fact.dimension === 'audioCaptured' ? (fact.value === 'Silent' ? 'Silent' : 'Audio recorded')
|
|
96
|
+
: fact.value;
|
|
97
|
+
badges.push({ dimension: fact.dimension, text, basis: fact.basis });
|
|
98
|
+
}
|
|
99
|
+
return badges.length ? badges : [{ dimension: 'unknown', text: 'Scope unknown', basis: 'undeclared' }];
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* What a reviewer must not read into evidence: a sentence per thing that is not shown. It names the assertions the
|
|
103
|
+
* result stands on when there are some (so activity visible beyond them is not verified), and every kind of reality
|
|
104
|
+
* that was not declared. Empty only for a scope that declares and verifies everything and lists no limitation.
|
|
105
|
+
*/
|
|
106
|
+
export function scopeCaveats(scope) {
|
|
107
|
+
const out = [];
|
|
108
|
+
if (!scope || scopeTokens(scope)[0] === UNKNOWN_SCOPE)
|
|
109
|
+
return ['Scope not declared: do not assume this ran live, on a physical device, with real audio or end to end.'];
|
|
110
|
+
if (scope.asserted?.length)
|
|
111
|
+
out.push(`Only these were asserted: ${scope.asserted.join('; ')}. Other activity visible in the recording is not verified.`);
|
|
112
|
+
else
|
|
113
|
+
out.push('No assertion boundary was declared: activity visible in the recording is not verified behavior.');
|
|
114
|
+
if (scope.syntheticData === true)
|
|
115
|
+
out.push('Uses synthetic data: not a statement about production data.');
|
|
116
|
+
if (scope.dependencies === undefined)
|
|
117
|
+
out.push('Whether dependencies were scripted or live is not declared.');
|
|
118
|
+
else if (word(scope.dependencies) !== 'live')
|
|
119
|
+
out.push(`Dependencies were ${word(scope.dependencies)}, not live.`);
|
|
120
|
+
if (scope.audioCaptured === undefined)
|
|
121
|
+
out.push('Whether audio was captured is not declared.');
|
|
122
|
+
else if (!scope.audioCaptured)
|
|
123
|
+
out.push('Silent: nothing audible was recorded.');
|
|
124
|
+
if (scope.deviceKind === undefined)
|
|
125
|
+
out.push('The kind of device is not declared.');
|
|
126
|
+
for (const limit of scope.limitations ?? [])
|
|
127
|
+
out.push(limit);
|
|
128
|
+
return out;
|
|
129
|
+
}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* @see docs/report-contract.md
|
|
10
10
|
*/
|
|
11
11
|
import type { EvidenceMeasurement, EvidenceStatus } from './explorer-types.js';
|
|
12
|
-
import type { LayerRun } from '../../schema/src/layer.js';
|
|
12
|
+
import type { EvidenceScope, LayerRun } from '../../schema/src/layer.js';
|
|
13
13
|
import type { LayerParameters } from './layer-types.js';
|
|
14
14
|
/** Where a recorded test's timeline and captured files are read from. */
|
|
15
15
|
export type RecordingAddress = {
|
|
@@ -57,6 +57,8 @@ export type VideoRef = {
|
|
|
57
57
|
readonly recording?: string;
|
|
58
58
|
/** False when the reference names no bytes at all (no bundle key, address or inline text), so it can never be played. */
|
|
59
59
|
readonly available: boolean;
|
|
60
|
+
/** What this recording shows, as declared for it: its own `scope`, else its attempt's, check's, part's or layer's. Absent when none declared one: its scope is unknown, not live, physical or audible. */
|
|
61
|
+
readonly scope?: EvidenceScope;
|
|
60
62
|
};
|
|
61
63
|
/** Video availability as counts: references, distinct playable recordings (deduplicated), and references that cannot be played. */
|
|
62
64
|
export type VideoCounts = {
|
|
@@ -36,7 +36,7 @@ export type EvidenceMeasurement = {
|
|
|
36
36
|
readonly limit?: number;
|
|
37
37
|
};
|
|
38
38
|
/** What a check may carry beyond its outcome. All optional and open: see docs/extending.md. */
|
|
39
|
-
export type EvidenceExtras = Partial<Pick<import('../../schema/src/layer.js').Check, 'tags' | 'labels' | 'links' | 'steps' | 'attachments' | 'annotations' | 'attempts' | 'extensions'>>;
|
|
39
|
+
export type EvidenceExtras = Partial<Pick<import('../../schema/src/layer.js').Check, 'tags' | 'labels' | 'links' | 'steps' | 'attachments' | 'annotations' | 'attempts' | 'extensions' | 'scope'>>;
|
|
40
40
|
/** One check of a part, as recorded. */
|
|
41
41
|
export type EvidenceCheck = EvidenceExtras & {
|
|
42
42
|
/** Stable across runs; the newer result replaces the older one only if both used the same id. */
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { summarizeEvidenceIn, summarizeLayerEvidence } from './evidence-detail.js';
|
|
2
|
+
import { effectiveScope } from './evidence-scope.js';
|
|
2
3
|
import { layerPlace } from './layer-axes.js';
|
|
3
4
|
/**
|
|
4
5
|
* A short, deterministic fingerprint of a spec's text, shaped `<length>:<hash>:<hash>` (the length in UTF-16 code
|
|
@@ -79,7 +80,7 @@ export function layerEvidence(summary, backend) {
|
|
|
79
80
|
for (const part of summary.parts) {
|
|
80
81
|
for (const check of part.checks) {
|
|
81
82
|
const test = tests.get(check.id);
|
|
82
|
-
const evidence =
|
|
83
|
+
const evidence = summarizeEvidenceIn(effectiveScope(part.scope, summary.scope), check);
|
|
83
84
|
const item = expectations?.get(check.id);
|
|
84
85
|
rows.push({
|
|
85
86
|
kind: part.kind,
|
|
@@ -15,7 +15,7 @@ import type { SourceRecord } from './protocol.js';
|
|
|
15
15
|
/** String choices (such as `{ device: 'iphone-15' }`) that make a result its own slot. Compared as sorted key/value pairs. */
|
|
16
16
|
export type LayerParameters = Readonly<Record<string, string>>;
|
|
17
17
|
/** What a layer may carry as a whole. All optional and open: see docs/extending.md. */
|
|
18
|
-
export type LayerExtras = Partial<Pick<import('../../schema/src/layer.js').LayerDocument, 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions' | 'recordings'>>;
|
|
18
|
+
export type LayerExtras = Partial<Pick<import('../../schema/src/layer.js').LayerDocument, 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions' | 'recordings' | 'scope'>>;
|
|
19
19
|
/** A layer without its captured files: what composition and the viewer's lists read. */
|
|
20
20
|
export type LayerSummary = LayerExtras & {
|
|
21
21
|
/** `modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout. Advisory; never changes a status. */
|
|
@@ -123,6 +123,7 @@ export async function publishChecks(store, input, options = {}) {
|
|
|
123
123
|
...(args.links?.length ? { links: args.links } : {}),
|
|
124
124
|
...(layerEvidence.attachments?.length ? { attachments: layerEvidence.attachments } : {}),
|
|
125
125
|
...(args.annotations?.length ? { annotations: args.annotations } : {}),
|
|
126
|
+
...(args.scope ? { scope: args.scope } : {}),
|
|
126
127
|
...(args.extensions && Object.keys(args.extensions).length ? { extensions: args.extensions } : {})
|
|
127
128
|
};
|
|
128
129
|
const bundle = entries.size ? await writeBundle(store, (async function* () { for (const entry of [...entries.values()].sort((a, b) => (a.key < b.key ? -1 : 1)))
|
|
@@ -13,7 +13,7 @@ import { type PublishChecksInput } from '../../schema/src/layer.js';
|
|
|
13
13
|
import type { ObjectStore } from './object-store.js';
|
|
14
14
|
type Check = PublishChecksInput['checks'][number];
|
|
15
15
|
/** What every layer of one stream shares: who is reporting, from where. Whatever `publishChecks` takes, except the checks and the time. */
|
|
16
|
-
export type ReporterIdentity = Pick<PublishChecksInput, 'producer' | 'kind' | 'title' | 'sha' | 'branch' | 'runUrl' | 'run' | 'environment' | 'parameters' | 'freshness' | 'specs' | 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions'>;
|
|
16
|
+
export type ReporterIdentity = Pick<PublishChecksInput, 'producer' | 'kind' | 'title' | 'sha' | 'branch' | 'runUrl' | 'run' | 'environment' | 'parameters' | 'freshness' | 'specs' | 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'scope' | 'extensions'>;
|
|
17
17
|
/** How a reporter sends: the `store` (a folder, S3, or `httpStore` for a server), how often (`flushMs`, `batchSize`), how hard it tries (`attempts`, `backoffMs`), what it tells (`onError`, `onPublished`), and the clock and sleep for tests. */
|
|
18
18
|
export type ReporterOptions = {
|
|
19
19
|
/** Where layers go: a directory, S3, or `httpStore` for an Atlas server. */
|