@crvouga/atlas 1.16.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -1
- package/dist/reports/src/test-catalog.d.ts +13 -0
- package/dist/reports/src/test-catalog.js +21 -5
- package/dist/schema/src/api.d.ts +29 -0
- package/dist/schema/src/api.js +6 -1
- package/docs/extending.md +1 -1
- package/docs/reference/reports.md +17 -0
- package/docs/ui.md +8 -0
- package/llms-full.txt +9 -1
- package/package.json +1 -1
- package/packages/schema/schemas/test-catalog.schema.json +26 -0
- package/packages/schema/schemas/test-history.schema.json +40 -0
- package/packages/server/openapi.json +66 -0
- package/packages/visualizer/src/evidence/applicability-view.tsx +46 -20
- package/packages/visualizer/src/evidence/applicability.test.ts +68 -1
- package/packages/visualizer/src/evidence/applicability.ts +121 -26
- package/packages/visualizer/src/explorer/Explorer.tsx +96 -8
- package/packages/visualizer/src/explorer/TestDetail.tsx +14 -1
- package/packages/visualizer/src/explorer/applicability.ts +43 -0
- package/packages/visualizer/src/explorer/search.ts +14 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
# Changelog: @crvouga/atlas
|
|
2
2
|
|
|
3
|
-
## 1.
|
|
3
|
+
## 1.17.0 (2026-10-29)
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
|
|
7
|
+
- read evidence applicability in the test explorer's list, sort and detail ([14181aa](https://github.com/crvouga/atlas/commit/14181aa391fb824c85364c4f5d93e1a6ccede176))
|
|
8
|
+
|
|
9
|
+
## 1.16.0 (2026-10-28)
|
|
4
10
|
|
|
5
11
|
### Features
|
|
6
12
|
|
|
@@ -6,6 +6,13 @@ import { RecordedSpecs } from './recorded-specs.js';
|
|
|
6
6
|
export declare const TEST_CATALOG_VERSION = 1;
|
|
7
7
|
/** Recorded results kept per test, newest first. Older ones are counted in no statistic. */
|
|
8
8
|
export declare const HISTORY_PER_TEST = 100;
|
|
9
|
+
/**
|
|
10
|
+
* The open `labels` (see docs/extending.md) that say what a result's evidence ran against and who vouches for it: `build`,
|
|
11
|
+
* `platform`, `applies-to` (builds or commits it is asserted to still hold for), `applies-to-by` (the policy or person that
|
|
12
|
+
* asserts it), `published-at` and `enriched-at` (when the evidence was published or enriched, epoch milliseconds or an ISO
|
|
13
|
+
* date: never when the checks ran). A test's current results carry them as `provenance`.
|
|
14
|
+
*/
|
|
15
|
+
export declare const APPLICABILITY_LABELS: readonly ["build", "platform", "applies-to", "applies-to-by", "published-at", "enriched-at"];
|
|
9
16
|
/** One recorded result of a test: when, where, how it went and how long it took, and the layer and run it came from. */
|
|
10
17
|
export type TestResultEntry = {
|
|
11
18
|
/** When it was recorded, epoch ms. */
|
|
@@ -32,6 +39,10 @@ export type TestResultEntry = {
|
|
|
32
39
|
readonly detail: string;
|
|
33
40
|
/** The fingerprint of its spec text the layer recorded, when it recorded one: results with different revisions were recorded against different promises. Absent means not recorded, never "the same". */
|
|
34
41
|
readonly specRevision?: string;
|
|
42
|
+
/** `modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, and on results other than a test's current ones. Advisory; never changes a status. */
|
|
43
|
+
readonly freshness?: 'unknown' | 'modified';
|
|
44
|
+
/** The labels of its layer and check that say what it ran against and who vouches for it: only those named in {@link APPLICABILITY_LABELS}. Present on a test's current results when it has any. Read by the viewer's evidence applicability; never decides a result. */
|
|
45
|
+
readonly provenance?: Readonly<Record<string, string>>;
|
|
35
46
|
/** What it carried beyond its outcome (attachments, steps, retries, tags, labels), when anything. */
|
|
36
47
|
readonly evidence?: EvidenceSummary;
|
|
37
48
|
/** Its product-latency measurements (`kind: product-latency`), when it recorded any. Separate from `durationMs`, which is how long the test took (setup, waits and pacing included). */
|
|
@@ -82,6 +93,8 @@ export type CatalogSpec = {
|
|
|
82
93
|
/** The group it belongs to: the spec's own domain, else the folder under `specs/`. */
|
|
83
94
|
readonly domain: string;
|
|
84
95
|
readonly kind: string;
|
|
96
|
+
/** The fingerprint of the spec text now (the promise), when the spec set or a layer recorded one. A result's `specRevision` is compared with it: different means the promise changed after the result was recorded. Absent: unknown, never "the same". */
|
|
97
|
+
readonly fingerprint?: string;
|
|
85
98
|
};
|
|
86
99
|
/** One test: what it is, its current status per environment, and how it has gone. */
|
|
87
100
|
export type TestSummary = {
|
|
@@ -34,6 +34,13 @@ import { RecordedSpecs } from './recorded-specs.js';
|
|
|
34
34
|
export const TEST_CATALOG_VERSION = 1;
|
|
35
35
|
/** Recorded results kept per test, newest first. Older ones are counted in no statistic. */
|
|
36
36
|
export const HISTORY_PER_TEST = 100;
|
|
37
|
+
/**
|
|
38
|
+
* The open `labels` (see docs/extending.md) that say what a result's evidence ran against and who vouches for it: `build`,
|
|
39
|
+
* `platform`, `applies-to` (builds or commits it is asserted to still hold for), `applies-to-by` (the policy or person that
|
|
40
|
+
* asserts it), `published-at` and `enriched-at` (when the evidence was published or enriched, epoch milliseconds or an ISO
|
|
41
|
+
* date: never when the checks ran). A test's current results carry them as `provenance`.
|
|
42
|
+
*/
|
|
43
|
+
export const APPLICABILITY_LABELS = ['build', 'platform', 'applies-to', 'applies-to-by', 'published-at', 'enriched-at'];
|
|
37
44
|
const FAILURE_RANK = { passed: 0, 'not-run': 1, running: 2, failed: 3 };
|
|
38
45
|
/** How tests are ordered. */
|
|
39
46
|
export const TEST_SORTS = ['spec', 'title', 'recent', 'slowest', 'failures', 'flaky', 'runs'];
|
|
@@ -409,6 +416,7 @@ export function catalogSpec(path, known) {
|
|
|
409
416
|
title: entry?.title || (base ? base[0].toUpperCase() + base.slice(1) : path),
|
|
410
417
|
domain: entry?.domain || folder,
|
|
411
418
|
kind: entry?.kind || (/\.([^./]+)$/.exec(path)?.[1] ?? 'unknown'),
|
|
419
|
+
...(entry?.fingerprint ? { fingerprint: entry.fingerprint } : {}),
|
|
412
420
|
};
|
|
413
421
|
}
|
|
414
422
|
function statsOf(history, outcomes, tags, labels) {
|
|
@@ -464,11 +472,18 @@ function statsOf(history, outcomes, tags, labels) {
|
|
|
464
472
|
firstSeenMs, lastSeenMs, lastPassedAtMs, lastFailedAtMs, runs: runs.size, branches,
|
|
465
473
|
};
|
|
466
474
|
}
|
|
467
|
-
|
|
468
|
-
|
|
475
|
+
/** A current result with what the layer recorded about its spec and about what it ran against (see {@link APPLICABILITY_LABELS}). */
|
|
476
|
+
function outcomeOf(result, fingerprints = {}) {
|
|
477
|
+
const labels = { ...(result.layer.evidence?.labels ?? {}), ...(result.row.evidence?.labels ?? {}) };
|
|
478
|
+
const provenance = Object.fromEntries(APPLICABILITY_LABELS.filter(name => typeof labels[name] === 'string').map(name => [name, labels[name]]));
|
|
479
|
+
return {
|
|
480
|
+
...entryOf(result.layer, result.row, result.layer.backend, fingerprints),
|
|
481
|
+
...(result.layer.freshness ? { freshness: result.layer.freshness } : {}),
|
|
482
|
+
...(Object.keys(provenance).length ? { provenance } : {}),
|
|
483
|
+
};
|
|
469
484
|
}
|
|
470
|
-
function summaryOf(slot, history, known) {
|
|
471
|
-
const outcomes = slot.outcomes.map(outcomeOf);
|
|
485
|
+
function summaryOf(slot, history, known, fingerprintsOf) {
|
|
486
|
+
const outcomes = slot.outcomes.map(result => outcomeOf(result, fingerprintsOf(result.layer.id)));
|
|
472
487
|
const tags = new Set();
|
|
473
488
|
const artifactTags = new Set();
|
|
474
489
|
const inheritedTags = new Set();
|
|
@@ -586,7 +601,8 @@ export function catalogModel(layers, sources = {}) {
|
|
|
586
601
|
const known = (path) => recorded.get(path);
|
|
587
602
|
const combined = combineEvidence({ specs: [], sha: null, changedFiles: [] }, layers);
|
|
588
603
|
const slots = checkSlots(combined.unassigned);
|
|
589
|
-
const
|
|
604
|
+
const fingerprints = new Map(layers.map(entry => [entry.layer.id, entry.fingerprints]));
|
|
605
|
+
const tests = slots.map(slot => summaryOf(slot, history, known, id => fingerprints.get(id)));
|
|
590
606
|
const byKey = new Map(tests.map(test => [test.key, test]));
|
|
591
607
|
const haystacks = new Map(tests.map(test => [test.key, haystackOf(test)]));
|
|
592
608
|
const counts = emptyCounts();
|
package/dist/schema/src/api.d.ts
CHANGED
|
@@ -2255,6 +2255,11 @@ export declare const TestResultEntrySchema: z.ZodObject<{
|
|
|
2255
2255
|
backend: z.ZodString;
|
|
2256
2256
|
detail: z.ZodString;
|
|
2257
2257
|
specRevision: z.ZodOptional<z.ZodString>;
|
|
2258
|
+
freshness: z.ZodOptional<z.ZodEnum<{
|
|
2259
|
+
unknown: "unknown";
|
|
2260
|
+
modified: "modified";
|
|
2261
|
+
}>>;
|
|
2262
|
+
provenance: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
2258
2263
|
evidence: z.ZodOptional<z.ZodObject<{
|
|
2259
2264
|
attachments: z.ZodNumber;
|
|
2260
2265
|
steps: z.ZodNumber;
|
|
@@ -2329,6 +2334,7 @@ export declare const TestSummarySchema: z.ZodObject<{
|
|
|
2329
2334
|
title: z.ZodString;
|
|
2330
2335
|
domain: z.ZodString;
|
|
2331
2336
|
kind: z.ZodString;
|
|
2337
|
+
fingerprint: z.ZodOptional<z.ZodString>;
|
|
2332
2338
|
}, z.core.$loose>>;
|
|
2333
2339
|
status: z.ZodEnum<{
|
|
2334
2340
|
passed: "passed";
|
|
@@ -2359,6 +2365,11 @@ export declare const TestSummarySchema: z.ZodObject<{
|
|
|
2359
2365
|
backend: z.ZodString;
|
|
2360
2366
|
detail: z.ZodString;
|
|
2361
2367
|
specRevision: z.ZodOptional<z.ZodString>;
|
|
2368
|
+
freshness: z.ZodOptional<z.ZodEnum<{
|
|
2369
|
+
unknown: "unknown";
|
|
2370
|
+
modified: "modified";
|
|
2371
|
+
}>>;
|
|
2372
|
+
provenance: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
2362
2373
|
evidence: z.ZodOptional<z.ZodObject<{
|
|
2363
2374
|
attachments: z.ZodNumber;
|
|
2364
2375
|
steps: z.ZodNumber;
|
|
@@ -2586,6 +2597,7 @@ export declare const TestCatalogSchema: z.ZodObject<{
|
|
|
2586
2597
|
title: z.ZodString;
|
|
2587
2598
|
domain: z.ZodString;
|
|
2588
2599
|
kind: z.ZodString;
|
|
2600
|
+
fingerprint: z.ZodOptional<z.ZodString>;
|
|
2589
2601
|
tests: z.ZodNumber;
|
|
2590
2602
|
counts: z.ZodRecord<z.ZodEnum<{
|
|
2591
2603
|
passed: "passed";
|
|
@@ -2628,6 +2640,7 @@ export declare const TestCatalogSchema: z.ZodObject<{
|
|
|
2628
2640
|
title: z.ZodString;
|
|
2629
2641
|
domain: z.ZodString;
|
|
2630
2642
|
kind: z.ZodString;
|
|
2643
|
+
fingerprint: z.ZodOptional<z.ZodString>;
|
|
2631
2644
|
}, z.core.$loose>>;
|
|
2632
2645
|
status: z.ZodEnum<{
|
|
2633
2646
|
passed: "passed";
|
|
@@ -2658,6 +2671,11 @@ export declare const TestCatalogSchema: z.ZodObject<{
|
|
|
2658
2671
|
backend: z.ZodString;
|
|
2659
2672
|
detail: z.ZodString;
|
|
2660
2673
|
specRevision: z.ZodOptional<z.ZodString>;
|
|
2674
|
+
freshness: z.ZodOptional<z.ZodEnum<{
|
|
2675
|
+
unknown: "unknown";
|
|
2676
|
+
modified: "modified";
|
|
2677
|
+
}>>;
|
|
2678
|
+
provenance: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
2661
2679
|
evidence: z.ZodOptional<z.ZodObject<{
|
|
2662
2680
|
attachments: z.ZodNumber;
|
|
2663
2681
|
steps: z.ZodNumber;
|
|
@@ -2860,6 +2878,7 @@ export declare const TestHistorySchema: z.ZodObject<{
|
|
|
2860
2878
|
title: z.ZodString;
|
|
2861
2879
|
domain: z.ZodString;
|
|
2862
2880
|
kind: z.ZodString;
|
|
2881
|
+
fingerprint: z.ZodOptional<z.ZodString>;
|
|
2863
2882
|
}, z.core.$loose>>;
|
|
2864
2883
|
status: z.ZodEnum<{
|
|
2865
2884
|
passed: "passed";
|
|
@@ -2890,6 +2909,11 @@ export declare const TestHistorySchema: z.ZodObject<{
|
|
|
2890
2909
|
backend: z.ZodString;
|
|
2891
2910
|
detail: z.ZodString;
|
|
2892
2911
|
specRevision: z.ZodOptional<z.ZodString>;
|
|
2912
|
+
freshness: z.ZodOptional<z.ZodEnum<{
|
|
2913
|
+
unknown: "unknown";
|
|
2914
|
+
modified: "modified";
|
|
2915
|
+
}>>;
|
|
2916
|
+
provenance: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
2893
2917
|
evidence: z.ZodOptional<z.ZodObject<{
|
|
2894
2918
|
attachments: z.ZodNumber;
|
|
2895
2919
|
steps: z.ZodNumber;
|
|
@@ -3013,6 +3037,11 @@ export declare const TestHistorySchema: z.ZodObject<{
|
|
|
3013
3037
|
backend: z.ZodString;
|
|
3014
3038
|
detail: z.ZodString;
|
|
3015
3039
|
specRevision: z.ZodOptional<z.ZodString>;
|
|
3040
|
+
freshness: z.ZodOptional<z.ZodEnum<{
|
|
3041
|
+
unknown: "unknown";
|
|
3042
|
+
modified: "modified";
|
|
3043
|
+
}>>;
|
|
3044
|
+
provenance: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
3016
3045
|
evidence: z.ZodOptional<z.ZodObject<{
|
|
3017
3046
|
attachments: z.ZodNumber;
|
|
3018
3047
|
steps: z.ZodNumber;
|
package/dist/schema/src/api.js
CHANGED
|
@@ -432,13 +432,18 @@ export const TestResultEntrySchema = z.looseObject({
|
|
|
432
432
|
backend: z.string(),
|
|
433
433
|
detail: z.string().describe('Why it has its status, at most 500 characters'),
|
|
434
434
|
specRevision: z.string().optional().describe('The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match'),
|
|
435
|
+
freshness: z.enum(['modified', 'unknown']).optional().describe('`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, on results other than a test\'s current ones, and from an older server. Advisory; never changes a status'),
|
|
436
|
+
provenance: z.record(z.string(), z.string()).optional().describe('The labels of its layer and check that say what it ran against and who vouches for it: `build`, `platform`, `applies-to`, `applies-to-by`, `published-at`, `enriched-at`. Present on a test\'s current results when it has any, and absent from an older server. Never decides a result'),
|
|
435
437
|
evidence: EvidenceSummarySchema.optional(),
|
|
436
438
|
latency: z.array(MeasurementSchema).optional().describe('Its product-latency measurements (`kind: product-latency`), when it recorded any. Separate from `durationMs`, which is how long the test took')
|
|
437
439
|
});
|
|
438
440
|
const DurationStatsSchema = z.looseObject({
|
|
439
441
|
count: z.number().int().nonnegative(), totalMs: z.number(), avgMs: z.number(), p50Ms: z.number(), p95Ms: z.number(), minMs: z.number(), maxMs: z.number(), lastMs: z.number()
|
|
440
442
|
});
|
|
441
|
-
const CatalogSpecSchema = z.looseObject({
|
|
443
|
+
const CatalogSpecSchema = z.looseObject({
|
|
444
|
+
path: z.string(), title: z.string(), domain: z.string(), kind: z.string(),
|
|
445
|
+
fingerprint: z.string().optional().describe('The fingerprint of the spec text now (the promise). Compare it with a result\'s `specRevision`: different means the promise changed after the result was recorded. Absent: unknown, never "the same". Absent from an older server')
|
|
446
|
+
});
|
|
442
447
|
/** One test of the catalog: what it is, its current result per environment, and how it has gone over its recorded history. */
|
|
443
448
|
export const TestSummarySchema = z.looseObject({
|
|
444
449
|
key: z.string().describe('The test\'s identity, `JSON.stringify([specPath, id])`. Pass it to `/api/reports/tests/history?key=`'),
|
package/docs/extending.md
CHANGED
|
@@ -30,7 +30,7 @@ carry any of these. Every one is optional.
|
|
|
30
30
|
| `annotations[]` | Notes about the result: a warning, a deprecation, an accessibility finding, a coverage figure. |
|
|
31
31
|
| `attempts[]` | Every try, when the check was retried. |
|
|
32
32
|
| `tags[]` | Free labels to filter by: `smoke`, `payments`. |
|
|
33
|
-
| `labels{}` | Facts about the result that are not its identity: `owner`, `severity`, `browser`, `build`. The viewer reads `build`, `platform`, `applies-to` and `applies-to-by` to tell whether evidence applies to a release (see [the viewer conventions](ui.md#behaviors-read-watch-inspect)). `behavior-id` and `example-id` declare a stable id for the behavior and for one example of an outline, and `outline` names the outline; the [catalog health](server.md#reading-the-report-as-an-agent) report counts them and names an id shared by several checks. They never change which check a result belongs to. |
|
|
33
|
+
| `labels{}` | Facts about the result that are not its identity: `owner`, `severity`, `browser`, `build`. The viewer reads `build`, `platform`, `applies-to` and `applies-to-by` to tell whether evidence applies to a release, and `published-at` and `enriched-at` (epoch milliseconds or an ISO date) to show when it was published or enriched apart from when the checks ran (see [the viewer conventions](ui.md#behaviors-read-watch-inspect)). `behavior-id` and `example-id` declare a stable id for the behavior and for one example of an outline, and `outline` names the outline; the [catalog health](server.md#reading-the-report-as-an-agent) report counts them and names an id shared by several checks. They never change which check a result belongs to. |
|
|
34
34
|
| `scope{}` | What the evidence does and does not show: surface, device kind, synthetic data, scripted dependencies, audio, pacing, what was asserted and its limits. Unknown when absent. See [Evidence scope](rich-evidence.md#evidence-scope). |
|
|
35
35
|
| `links[]` | Where to read more: the CI run, the issue, the trace client. |
|
|
36
36
|
| `extensions{}` | Structured data of your own or a tool's, under a name you own. |
|
|
@@ -9578,6 +9578,17 @@ export const HISTORY_PER_TEST = …
|
|
|
9578
9578
|
|
|
9579
9579
|
Recorded results kept per test, newest first. Older ones are counted in no statistic.
|
|
9580
9580
|
|
|
9581
|
+
#### `APPLICABILITY_LABELS`
|
|
9582
|
+
|
|
9583
|
+
```ts
|
|
9584
|
+
export const APPLICABILITY_LABELS = …
|
|
9585
|
+
```
|
|
9586
|
+
|
|
9587
|
+
The open `labels` (see docs/extending.md) that say what a result's evidence ran against and who vouches for it: `build`,
|
|
9588
|
+
`platform`, `applies-to` (builds or commits it is asserted to still hold for), `applies-to-by` (the policy or person that
|
|
9589
|
+
asserts it), `published-at` and `enriched-at` (when the evidence was published or enriched, epoch milliseconds or an ISO
|
|
9590
|
+
date: never when the checks ran). A test's current results carry them as `provenance`.
|
|
9591
|
+
|
|
9581
9592
|
#### `TestResultEntry`
|
|
9582
9593
|
|
|
9583
9594
|
```ts
|
|
@@ -9606,6 +9617,10 @@ export type TestResultEntry = {
|
|
|
9606
9617
|
readonly detail: string;
|
|
9607
9618
|
/** The fingerprint of its spec text the layer recorded, when it recorded one: results with different revisions were recorded against different promises. Absent means not recorded, never "the same". */
|
|
9608
9619
|
readonly specRevision?: string;
|
|
9620
|
+
/** `modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, and on results other than a test's current ones. Advisory; never changes a status. */
|
|
9621
|
+
readonly freshness?: 'unknown' | 'modified';
|
|
9622
|
+
/** The labels of its layer and check that say what it ran against and who vouches for it: only those named in {@link APPLICABILITY_LABELS}. Present on a test's current results when it has any. Read by the viewer's evidence applicability; never decides a result. */
|
|
9623
|
+
readonly provenance?: Readonly<Record<string, string>>;
|
|
9609
9624
|
/** What it carried beyond its outcome (attachments, steps, retries, tags, labels), when anything. */
|
|
9610
9625
|
readonly evidence?: EvidenceSummary;
|
|
9611
9626
|
/** Its product-latency measurements (`kind: product-latency`), when it recorded any. Separate from `durationMs`, which is how long the test took (setup, waits and pacing included). */
|
|
@@ -9680,6 +9695,8 @@ export type CatalogSpec = {
|
|
|
9680
9695
|
/** The group it belongs to: the spec's own domain, else the folder under `specs/`. */
|
|
9681
9696
|
readonly domain: string;
|
|
9682
9697
|
readonly kind: string;
|
|
9698
|
+
/** The fingerprint of the spec text now (the promise), when the spec set or a layer recorded one. A result's `specRevision` is compared with it: different means the promise changed after the result was recorded. Absent: unknown, never "the same". */
|
|
9699
|
+
readonly fingerprint?: string;
|
|
9683
9700
|
};
|
|
9684
9701
|
```
|
|
9685
9702
|
|
package/docs/ui.md
CHANGED
|
@@ -91,6 +91,14 @@ when Watch is on screen.
|
|
|
91
91
|
what they know with labels: `build`, `platform`, and `applies-to` with `applies-to-by` (the policy that asserts the
|
|
92
92
|
evidence still holds for those builds; without it the assertion is refused). The "Stale or unverified" filter
|
|
93
93
|
(`evidence=stale`) lists every behavior whose evidence does not apply, including those with none.
|
|
94
|
+
- The test explorer makes the same reading of every test from the catalog (`explorer/applicability.ts`): the catalog
|
|
95
|
+
carries the spec's fingerprint now (`spec.fingerprint`) and, on each current result, `specRevision`, `freshness` and
|
|
96
|
+
`provenance` (the labels above), all optional. The same address names (`release`, `releaseSha`, `releasePlatform`,
|
|
97
|
+
`evidence=stale`) give the "Stale or unverified" button, the "Stale evidence first" sort, a verdict badge on rows
|
|
98
|
+
(stale always; unknown or none when a release is named or evidence is filtered or sorted), and the summary in test
|
|
99
|
+
detail. Layers carry no publication time, because a layer's id is the digest of its content and a time of upload in
|
|
100
|
+
it would make a retried publish a different layer: a producer that knows states `published-at` and `enriched-at`
|
|
101
|
+
labels, shown apart from "Checks ran", which alone is the age.
|
|
94
102
|
- A recording is always labelled illustrative: it shows one run and does not approve a behavior for production.
|
|
95
103
|
`demo`/`representative` tags only add a "Representative demo" label.
|
|
96
104
|
- The spec index searches titles, paths and declared domains, and can group by the domain from spec metadata (specs with
|
package/llms-full.txt
CHANGED
|
@@ -3204,7 +3204,7 @@ carry any of these. Every one is optional.
|
|
|
3204
3204
|
| `annotations[]` | Notes about the result: a warning, a deprecation, an accessibility finding, a coverage figure. |
|
|
3205
3205
|
| `attempts[]` | Every try, when the check was retried. |
|
|
3206
3206
|
| `tags[]` | Free labels to filter by: `smoke`, `payments`. |
|
|
3207
|
-
| `labels{}` | Facts about the result that are not its identity: `owner`, `severity`, `browser`, `build`. The viewer reads `build`, `platform`, `applies-to` and `applies-to-by` to tell whether evidence applies to a release (see [the viewer conventions](ui.md#behaviors-read-watch-inspect)). `behavior-id` and `example-id` declare a stable id for the behavior and for one example of an outline, and `outline` names the outline; the [catalog health](server.md#reading-the-report-as-an-agent) report counts them and names an id shared by several checks. They never change which check a result belongs to. |
|
|
3207
|
+
| `labels{}` | Facts about the result that are not its identity: `owner`, `severity`, `browser`, `build`. The viewer reads `build`, `platform`, `applies-to` and `applies-to-by` to tell whether evidence applies to a release, and `published-at` and `enriched-at` (epoch milliseconds or an ISO date) to show when it was published or enriched apart from when the checks ran (see [the viewer conventions](ui.md#behaviors-read-watch-inspect)). `behavior-id` and `example-id` declare a stable id for the behavior and for one example of an outline, and `outline` names the outline; the [catalog health](server.md#reading-the-report-as-an-agent) report counts them and names an id shared by several checks. They never change which check a result belongs to. |
|
|
3208
3208
|
| `scope{}` | What the evidence does and does not show: surface, device kind, synthetic data, scripted dependencies, audio, pacing, what was asserted and its limits. Unknown when absent. See [Evidence scope](rich-evidence.md#evidence-scope). |
|
|
3209
3209
|
| `links[]` | Where to read more: the CI run, the issue, the trace client. |
|
|
3210
3210
|
| `extensions{}` | Structured data of your own or a tool's, under a name you own. |
|
|
@@ -4519,6 +4519,14 @@ when Watch is on screen.
|
|
|
4519
4519
|
what they know with labels: `build`, `platform`, and `applies-to` with `applies-to-by` (the policy that asserts the
|
|
4520
4520
|
evidence still holds for those builds; without it the assertion is refused). The "Stale or unverified" filter
|
|
4521
4521
|
(`evidence=stale`) lists every behavior whose evidence does not apply, including those with none.
|
|
4522
|
+
- The test explorer makes the same reading of every test from the catalog (`explorer/applicability.ts`): the catalog
|
|
4523
|
+
carries the spec's fingerprint now (`spec.fingerprint`) and, on each current result, `specRevision`, `freshness` and
|
|
4524
|
+
`provenance` (the labels above), all optional. The same address names (`release`, `releaseSha`, `releasePlatform`,
|
|
4525
|
+
`evidence=stale`) give the "Stale or unverified" button, the "Stale evidence first" sort, a verdict badge on rows
|
|
4526
|
+
(stale always; unknown or none when a release is named or evidence is filtered or sorted), and the summary in test
|
|
4527
|
+
detail. Layers carry no publication time, because a layer's id is the digest of its content and a time of upload in
|
|
4528
|
+
it would make a retried publish a different layer: a producer that knows states `published-at` and `enriched-at`
|
|
4529
|
+
labels, shown apart from "Checks ran", which alone is the age.
|
|
4522
4530
|
- A recording is always labelled illustrative: it shows one run and does not approve a behavior for production.
|
|
4523
4531
|
`demo`/`representative` tags only add a "Representative demo" label.
|
|
4524
4532
|
- The spec index searches titles, paths and declared domains, and can group by the domain from spec metadata (specs with
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crvouga/atlas",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.17.0",
|
|
4
4
|
"description": "Atlas: specs of any kind, reported as layered evidence. Publish checks from any runner (JUnit, CTRF, a script), combine them across nodes, branches, environments and backends, gate CI on the result, and run statecharts against the real app.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -529,6 +529,10 @@
|
|
|
529
529
|
"kind": {
|
|
530
530
|
"type": "string"
|
|
531
531
|
},
|
|
532
|
+
"fingerprint": {
|
|
533
|
+
"description": "The fingerprint of the spec text now (the promise). Compare it with a result's `specRevision`: different means the promise changed after the result was recorded. Absent: unknown, never \"the same\". Absent from an older server",
|
|
534
|
+
"type": "string"
|
|
535
|
+
},
|
|
532
536
|
"tests": {
|
|
533
537
|
"type": "integer",
|
|
534
538
|
"minimum": 0,
|
|
@@ -747,6 +751,10 @@
|
|
|
747
751
|
},
|
|
748
752
|
"kind": {
|
|
749
753
|
"type": "string"
|
|
754
|
+
},
|
|
755
|
+
"fingerprint": {
|
|
756
|
+
"description": "The fingerprint of the spec text now (the promise). Compare it with a result's `specRevision`: different means the promise changed after the result was recorded. Absent: unknown, never \"the same\". Absent from an older server",
|
|
757
|
+
"type": "string"
|
|
750
758
|
}
|
|
751
759
|
},
|
|
752
760
|
"required": [
|
|
@@ -869,6 +877,24 @@
|
|
|
869
877
|
"description": "The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match",
|
|
870
878
|
"type": "string"
|
|
871
879
|
},
|
|
880
|
+
"freshness": {
|
|
881
|
+
"description": "`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, on results other than a test's current ones, and from an older server. Advisory; never changes a status",
|
|
882
|
+
"type": "string",
|
|
883
|
+
"enum": [
|
|
884
|
+
"modified",
|
|
885
|
+
"unknown"
|
|
886
|
+
]
|
|
887
|
+
},
|
|
888
|
+
"provenance": {
|
|
889
|
+
"description": "The labels of its layer and check that say what it ran against and who vouches for it: `build`, `platform`, `applies-to`, `applies-to-by`, `published-at`, `enriched-at`. Present on a test's current results when it has any, and absent from an older server. Never decides a result",
|
|
890
|
+
"type": "object",
|
|
891
|
+
"propertyNames": {
|
|
892
|
+
"type": "string"
|
|
893
|
+
},
|
|
894
|
+
"additionalProperties": {
|
|
895
|
+
"type": "string"
|
|
896
|
+
}
|
|
897
|
+
},
|
|
872
898
|
"evidence": {
|
|
873
899
|
"type": "object",
|
|
874
900
|
"properties": {
|
|
@@ -45,6 +45,10 @@
|
|
|
45
45
|
},
|
|
46
46
|
"kind": {
|
|
47
47
|
"type": "string"
|
|
48
|
+
},
|
|
49
|
+
"fingerprint": {
|
|
50
|
+
"description": "The fingerprint of the spec text now (the promise). Compare it with a result's `specRevision`: different means the promise changed after the result was recorded. Absent: unknown, never \"the same\". Absent from an older server",
|
|
51
|
+
"type": "string"
|
|
48
52
|
}
|
|
49
53
|
},
|
|
50
54
|
"required": [
|
|
@@ -167,6 +171,24 @@
|
|
|
167
171
|
"description": "The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match",
|
|
168
172
|
"type": "string"
|
|
169
173
|
},
|
|
174
|
+
"freshness": {
|
|
175
|
+
"description": "`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, on results other than a test's current ones, and from an older server. Advisory; never changes a status",
|
|
176
|
+
"type": "string",
|
|
177
|
+
"enum": [
|
|
178
|
+
"modified",
|
|
179
|
+
"unknown"
|
|
180
|
+
]
|
|
181
|
+
},
|
|
182
|
+
"provenance": {
|
|
183
|
+
"description": "The labels of its layer and check that say what it ran against and who vouches for it: `build`, `platform`, `applies-to`, `applies-to-by`, `published-at`, `enriched-at`. Present on a test's current results when it has any, and absent from an older server. Never decides a result",
|
|
184
|
+
"type": "object",
|
|
185
|
+
"propertyNames": {
|
|
186
|
+
"type": "string"
|
|
187
|
+
},
|
|
188
|
+
"additionalProperties": {
|
|
189
|
+
"type": "string"
|
|
190
|
+
}
|
|
191
|
+
},
|
|
170
192
|
"evidence": {
|
|
171
193
|
"type": "object",
|
|
172
194
|
"properties": {
|
|
@@ -820,6 +842,24 @@
|
|
|
820
842
|
"description": "The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match",
|
|
821
843
|
"type": "string"
|
|
822
844
|
},
|
|
845
|
+
"freshness": {
|
|
846
|
+
"description": "`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, on results other than a test's current ones, and from an older server. Advisory; never changes a status",
|
|
847
|
+
"type": "string",
|
|
848
|
+
"enum": [
|
|
849
|
+
"modified",
|
|
850
|
+
"unknown"
|
|
851
|
+
]
|
|
852
|
+
},
|
|
853
|
+
"provenance": {
|
|
854
|
+
"description": "The labels of its layer and check that say what it ran against and who vouches for it: `build`, `platform`, `applies-to`, `applies-to-by`, `published-at`, `enriched-at`. Present on a test's current results when it has any, and absent from an older server. Never decides a result",
|
|
855
|
+
"type": "object",
|
|
856
|
+
"propertyNames": {
|
|
857
|
+
"type": "string"
|
|
858
|
+
},
|
|
859
|
+
"additionalProperties": {
|
|
860
|
+
"type": "string"
|
|
861
|
+
}
|
|
862
|
+
},
|
|
823
863
|
"evidence": {
|
|
824
864
|
"type": "object",
|
|
825
865
|
"properties": {
|
|
@@ -20619,6 +20619,10 @@
|
|
|
20619
20619
|
"kind": {
|
|
20620
20620
|
"type": "string"
|
|
20621
20621
|
},
|
|
20622
|
+
"fingerprint": {
|
|
20623
|
+
"description": "The fingerprint of the spec text now (the promise). Compare it with a result's `specRevision`: different means the promise changed after the result was recorded. Absent: unknown, never \"the same\". Absent from an older server",
|
|
20624
|
+
"type": "string"
|
|
20625
|
+
},
|
|
20622
20626
|
"tests": {
|
|
20623
20627
|
"type": "integer",
|
|
20624
20628
|
"minimum": 0,
|
|
@@ -20837,6 +20841,10 @@
|
|
|
20837
20841
|
},
|
|
20838
20842
|
"kind": {
|
|
20839
20843
|
"type": "string"
|
|
20844
|
+
},
|
|
20845
|
+
"fingerprint": {
|
|
20846
|
+
"description": "The fingerprint of the spec text now (the promise). Compare it with a result's `specRevision`: different means the promise changed after the result was recorded. Absent: unknown, never \"the same\". Absent from an older server",
|
|
20847
|
+
"type": "string"
|
|
20840
20848
|
}
|
|
20841
20849
|
},
|
|
20842
20850
|
"required": [
|
|
@@ -20959,6 +20967,24 @@
|
|
|
20959
20967
|
"description": "The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match",
|
|
20960
20968
|
"type": "string"
|
|
20961
20969
|
},
|
|
20970
|
+
"freshness": {
|
|
20971
|
+
"description": "`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, on results other than a test's current ones, and from an older server. Advisory; never changes a status",
|
|
20972
|
+
"type": "string",
|
|
20973
|
+
"enum": [
|
|
20974
|
+
"modified",
|
|
20975
|
+
"unknown"
|
|
20976
|
+
]
|
|
20977
|
+
},
|
|
20978
|
+
"provenance": {
|
|
20979
|
+
"description": "The labels of its layer and check that say what it ran against and who vouches for it: `build`, `platform`, `applies-to`, `applies-to-by`, `published-at`, `enriched-at`. Present on a test's current results when it has any, and absent from an older server. Never decides a result",
|
|
20980
|
+
"type": "object",
|
|
20981
|
+
"propertyNames": {
|
|
20982
|
+
"type": "string"
|
|
20983
|
+
},
|
|
20984
|
+
"additionalProperties": {
|
|
20985
|
+
"type": "string"
|
|
20986
|
+
}
|
|
20987
|
+
},
|
|
20962
20988
|
"evidence": {
|
|
20963
20989
|
"type": "object",
|
|
20964
20990
|
"properties": {
|
|
@@ -22006,6 +22032,10 @@
|
|
|
22006
22032
|
},
|
|
22007
22033
|
"kind": {
|
|
22008
22034
|
"type": "string"
|
|
22035
|
+
},
|
|
22036
|
+
"fingerprint": {
|
|
22037
|
+
"description": "The fingerprint of the spec text now (the promise). Compare it with a result's `specRevision`: different means the promise changed after the result was recorded. Absent: unknown, never \"the same\". Absent from an older server",
|
|
22038
|
+
"type": "string"
|
|
22009
22039
|
}
|
|
22010
22040
|
},
|
|
22011
22041
|
"required": [
|
|
@@ -22128,6 +22158,24 @@
|
|
|
22128
22158
|
"description": "The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match",
|
|
22129
22159
|
"type": "string"
|
|
22130
22160
|
},
|
|
22161
|
+
"freshness": {
|
|
22162
|
+
"description": "`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, on results other than a test's current ones, and from an older server. Advisory; never changes a status",
|
|
22163
|
+
"type": "string",
|
|
22164
|
+
"enum": [
|
|
22165
|
+
"modified",
|
|
22166
|
+
"unknown"
|
|
22167
|
+
]
|
|
22168
|
+
},
|
|
22169
|
+
"provenance": {
|
|
22170
|
+
"description": "The labels of its layer and check that say what it ran against and who vouches for it: `build`, `platform`, `applies-to`, `applies-to-by`, `published-at`, `enriched-at`. Present on a test's current results when it has any, and absent from an older server. Never decides a result",
|
|
22171
|
+
"type": "object",
|
|
22172
|
+
"propertyNames": {
|
|
22173
|
+
"type": "string"
|
|
22174
|
+
},
|
|
22175
|
+
"additionalProperties": {
|
|
22176
|
+
"type": "string"
|
|
22177
|
+
}
|
|
22178
|
+
},
|
|
22131
22179
|
"evidence": {
|
|
22132
22180
|
"type": "object",
|
|
22133
22181
|
"properties": {
|
|
@@ -22781,6 +22829,24 @@
|
|
|
22781
22829
|
"description": "The fingerprint of its spec text that the layer recorded, when it recorded one. Two results with different revisions were recorded against different promises. Absent: not recorded, never assumed to match",
|
|
22782
22830
|
"type": "string"
|
|
22783
22831
|
},
|
|
22832
|
+
"freshness": {
|
|
22833
|
+
"description": "`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Absent for a clean checkout, on results other than a test's current ones, and from an older server. Advisory; never changes a status",
|
|
22834
|
+
"type": "string",
|
|
22835
|
+
"enum": [
|
|
22836
|
+
"modified",
|
|
22837
|
+
"unknown"
|
|
22838
|
+
]
|
|
22839
|
+
},
|
|
22840
|
+
"provenance": {
|
|
22841
|
+
"description": "The labels of its layer and check that say what it ran against and who vouches for it: `build`, `platform`, `applies-to`, `applies-to-by`, `published-at`, `enriched-at`. Present on a test's current results when it has any, and absent from an older server. Never decides a result",
|
|
22842
|
+
"type": "object",
|
|
22843
|
+
"propertyNames": {
|
|
22844
|
+
"type": "string"
|
|
22845
|
+
},
|
|
22846
|
+
"additionalProperties": {
|
|
22847
|
+
"type": "string"
|
|
22848
|
+
}
|
|
22849
|
+
},
|
|
22784
22850
|
"evidence": {
|
|
22785
22851
|
"type": "object",
|
|
22786
22852
|
"properties": {
|
|
@@ -111,6 +111,26 @@ export function ApplicabilitySummary({ assessment, release }: { assessment: Asse
|
|
|
111
111
|
</time>
|
|
112
112
|
</Text>
|
|
113
113
|
</Text>
|
|
114
|
+
{facts.publishedAtMs !== null && (
|
|
115
|
+
<Text span inherit data-published>
|
|
116
|
+
Published:{' '}
|
|
117
|
+
<Text span inherit fw={600}>
|
|
118
|
+
<time dateTime={new Date(facts.publishedAtMs).toISOString()} title={exactTime(facts.publishedAtMs)}>
|
|
119
|
+
{exactTime(facts.publishedAtMs)} ({ago(facts.publishedAtMs, now)})
|
|
120
|
+
</time>
|
|
121
|
+
</Text>
|
|
122
|
+
</Text>
|
|
123
|
+
)}
|
|
124
|
+
{facts.enrichedAtMs !== null && (
|
|
125
|
+
<Text span inherit data-enriched>
|
|
126
|
+
Enriched:{' '}
|
|
127
|
+
<Text span inherit fw={600}>
|
|
128
|
+
<time dateTime={new Date(facts.enrichedAtMs).toISOString()} title={exactTime(facts.enrichedAtMs)}>
|
|
129
|
+
{exactTime(facts.enrichedAtMs)} ({ago(facts.enrichedAtMs, now)})
|
|
130
|
+
</time>
|
|
131
|
+
</Text>
|
|
132
|
+
</Text>
|
|
133
|
+
)}
|
|
114
134
|
<Text span inherit>
|
|
115
135
|
Build:{' '}
|
|
116
136
|
<Text span inherit fw={600}>
|
|
@@ -167,13 +187,17 @@ export function ApplicabilitySummary({ assessment, release }: { assessment: Asse
|
|
|
167
187
|
*/
|
|
168
188
|
export function ApplicabilityControls({
|
|
169
189
|
state,
|
|
170
|
-
stale,
|
|
171
|
-
total
|
|
190
|
+
stale = 0,
|
|
191
|
+
total = 0,
|
|
192
|
+
filter = true
|
|
172
193
|
}: {
|
|
173
194
|
state: ApplicabilityState;
|
|
174
|
-
/** How many behaviors have stale or unverified evidence. */
|
|
175
|
-
stale
|
|
176
|
-
|
|
195
|
+
/** How many behaviors (or tests) have stale or unverified evidence. Only shown with the filter. */
|
|
196
|
+
stale?: number;
|
|
197
|
+
/** How many behaviors (or tests) there are. Only shown with the filter. */
|
|
198
|
+
total?: number;
|
|
199
|
+
/** Whether to offer the stale-or-unverified filter here; a caller that offers it elsewhere sets this false. */
|
|
200
|
+
filter?: boolean;
|
|
177
201
|
}) {
|
|
178
202
|
const { release, setRelease, onlyStale, setOnlyStale } = state;
|
|
179
203
|
return (
|
|
@@ -208,21 +232,23 @@ export function ApplicabilityControls({
|
|
|
208
232
|
A platform alone does not name a release. Add its build or commit.
|
|
209
233
|
</Text>
|
|
210
234
|
)}
|
|
211
|
-
|
|
212
|
-
<
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
235
|
+
{filter && (
|
|
236
|
+
<Group role="group" aria-label="Evidence filter" gap="sm">
|
|
237
|
+
<Button type="button" aria-pressed={!onlyStale} onClick={() => setOnlyStale(false)} variant={onlyStale ? 'subtle' : 'light'}>
|
|
238
|
+
All evidence{' '}
|
|
239
|
+
<Text span inherit>
|
|
240
|
+
{total}
|
|
241
|
+
</Text>
|
|
242
|
+
</Button>
|
|
243
|
+
<Button type="button" aria-pressed={onlyStale} onClick={() => setOnlyStale(true)} variant={onlyStale ? 'light' : 'subtle'}>
|
|
244
|
+
Stale or unverified{' '}
|
|
245
|
+
<Text span inherit>
|
|
246
|
+
{stale}
|
|
247
|
+
</Text>
|
|
248
|
+
</Button>
|
|
249
|
+
</Group>
|
|
250
|
+
)}
|
|
251
|
+
{filter && onlyStale && stale === 0 && (
|
|
226
252
|
<Text role="status" component="div" c="dimmed">
|
|
227
253
|
Every behavior has evidence that applies.
|
|
228
254
|
</Text>
|