@crvouga/atlas 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -1
- package/dist/reports/src/artifact-query.d.ts +14 -2
- package/dist/reports/src/artifact-query.js +13 -4
- package/dist/reports/src/evidence-detail.js +7 -4
- package/dist/reports/src/evidence-types.d.ts +4 -0
- package/dist/reports/src/recording-receipt.d.ts +23 -0
- package/dist/reports/src/recording-receipt.js +16 -0
- package/dist/reports/src/recording-verify.d.ts +74 -0
- package/dist/reports/src/recording-verify.js +142 -0
- package/dist/reports/src/report-cli.d.ts +2 -2
- package/dist/reports/src/report-cli.js +53 -3
- package/dist/reports/src/test-catalog.d.ts +7 -1
- package/dist/reports/src/test-catalog.js +10 -4
- package/dist/schema/src/api.d.ts +84 -0
- package/dist/schema/src/api.js +11 -2
- package/dist/schema/src/cli-results.d.ts +51 -1
- package/dist/schema/src/cli-results.js +25 -1
- package/dist/server/src/openapi.js +1 -0
- package/docs/agents.md +1 -0
- package/docs/reference/cli.md +11 -2
- package/docs/reference/http-api.md +2 -0
- package/docs/reference/reports.md +185 -9
- package/docs/reference/schema.md +19 -1
- package/docs/rich-evidence.md +32 -5
- package/docs/server.md +1 -1
- package/llms-full.txt +47 -8
- package/package.json +1 -1
- package/packages/schema/schemas/effective-delta.schema.json +8 -0
- package/packages/schema/schemas/effective-evidence.schema.json +8 -0
- package/packages/schema/schemas/evidence-page.schema.json +4 -0
- package/packages/schema/schemas/layer-rows.schema.json +4 -0
- package/packages/schema/schemas/report-run.schema.json +74 -0
- package/packages/schema/schemas/report-runs.schema.json +86 -0
- package/packages/schema/schemas/review-collection.schema.json +4 -0
- package/packages/schema/schemas/review-manifest.schema.json +4 -0
- package/packages/schema/schemas/test-catalog.schema.json +4 -0
- package/packages/schema/schemas/test-history.schema.json +8 -0
- package/packages/schema/schemas/verify-recordings-result.schema.json +97 -0
- package/packages/server/openapi.json +252 -0
- package/packages/visualizer/src/explorer/ArtifactsView.tsx +3 -2
- package/packages/visualizer/src/explorer/RunsView.tsx +28 -3
- package/packages/visualizer/src/explorer/filters.tsx +11 -3
- package/packages/visualizer/src/explorer/search.ts +6 -2
|
@@ -162,6 +162,12 @@
|
|
|
162
162
|
"minimum": 0,
|
|
163
163
|
"maximum": 9007199254740991,
|
|
164
164
|
"description": "References that name no bytes, or bytes the layer does not hold"
|
|
165
|
+
},
|
|
166
|
+
"missing": {
|
|
167
|
+
"description": "Of `unavailable`, those that name a file the layer's bundle does not hold (stored, then gone). Absent when there are none",
|
|
168
|
+
"type": "integer",
|
|
169
|
+
"exclusiveMinimum": 0,
|
|
170
|
+
"maximum": 9007199254740991
|
|
165
171
|
}
|
|
166
172
|
},
|
|
167
173
|
"required": [
|
|
@@ -171,6 +177,64 @@
|
|
|
171
177
|
],
|
|
172
178
|
"additionalProperties": {}
|
|
173
179
|
},
|
|
180
|
+
"recordings": {
|
|
181
|
+
"description": "Whether the recordings its layers expected were published, apart from its outcomes. Absent when no layer declared an expectation: its completeness is unknown, never complete. Matched by the `recordings` filter",
|
|
182
|
+
"type": "object",
|
|
183
|
+
"properties": {
|
|
184
|
+
"state": {
|
|
185
|
+
"type": "string",
|
|
186
|
+
"enum": [
|
|
187
|
+
"complete",
|
|
188
|
+
"incomplete",
|
|
189
|
+
"none-expected",
|
|
190
|
+
"unknown"
|
|
191
|
+
],
|
|
192
|
+
"description": "`incomplete` when an expected recording was skipped or unresolved; else `unknown` when a layer declared no expectations; else `complete` when something was expected and `none-expected` when nothing was"
|
|
193
|
+
},
|
|
194
|
+
"expected": {
|
|
195
|
+
"type": "integer",
|
|
196
|
+
"minimum": 0,
|
|
197
|
+
"maximum": 9007199254740991
|
|
198
|
+
},
|
|
199
|
+
"published": {
|
|
200
|
+
"type": "integer",
|
|
201
|
+
"minimum": 0,
|
|
202
|
+
"maximum": 9007199254740991,
|
|
203
|
+
"description": "Of the expected recordings, uploaded or referenced by address"
|
|
204
|
+
},
|
|
205
|
+
"skipped": {
|
|
206
|
+
"type": "integer",
|
|
207
|
+
"minimum": 0,
|
|
208
|
+
"maximum": 9007199254740991
|
|
209
|
+
},
|
|
210
|
+
"unresolved": {
|
|
211
|
+
"type": "integer",
|
|
212
|
+
"minimum": 0,
|
|
213
|
+
"maximum": 9007199254740991
|
|
214
|
+
},
|
|
215
|
+
"notApplicable": {
|
|
216
|
+
"type": "integer",
|
|
217
|
+
"minimum": 0,
|
|
218
|
+
"maximum": 9007199254740991
|
|
219
|
+
},
|
|
220
|
+
"layersWithoutReceipt": {
|
|
221
|
+
"type": "integer",
|
|
222
|
+
"minimum": 0,
|
|
223
|
+
"maximum": 9007199254740991,
|
|
224
|
+
"description": "Layers of the run that declared no recording expectations: nothing is claimed for them"
|
|
225
|
+
}
|
|
226
|
+
},
|
|
227
|
+
"required": [
|
|
228
|
+
"state",
|
|
229
|
+
"expected",
|
|
230
|
+
"published",
|
|
231
|
+
"skipped",
|
|
232
|
+
"unresolved",
|
|
233
|
+
"notApplicable",
|
|
234
|
+
"layersWithoutReceipt"
|
|
235
|
+
],
|
|
236
|
+
"additionalProperties": {}
|
|
237
|
+
},
|
|
174
238
|
"attachments": {
|
|
175
239
|
"description": "Files attached to the run itself, in its layers, not to a check. Absent when there are none",
|
|
176
240
|
"type": "integer",
|
|
@@ -316,6 +380,10 @@
|
|
|
316
380
|
"type": "boolean",
|
|
317
381
|
"description": "False when the reference names no bytes: no bundle key, address or inline text, or a bundle key that the layer's own bundle does not hold"
|
|
318
382
|
},
|
|
383
|
+
"reason": {
|
|
384
|
+
"description": "Why it is unavailable, when the report can tell: `not-in-bundle` is a file the layer names but does not hold (stored, then gone). An open vocabulary: treat an unknown reason as plain unavailable",
|
|
385
|
+
"type": "string"
|
|
386
|
+
},
|
|
319
387
|
"scope": {
|
|
320
388
|
"description": "What this recording shows as its producer declared it: its own `scope`, else its attempt's, check's, part's or layer's. Absent: the scope is unknown, never live, physical or audible",
|
|
321
389
|
"type": "object",
|
|
@@ -517,6 +585,12 @@
|
|
|
517
585
|
"minimum": 0,
|
|
518
586
|
"maximum": 9007199254740991,
|
|
519
587
|
"description": "References that name no bytes, or bytes the layer does not hold"
|
|
588
|
+
},
|
|
589
|
+
"missing": {
|
|
590
|
+
"description": "Of `unavailable`, those that name a file the layer's bundle does not hold (stored, then gone). Absent when there are none",
|
|
591
|
+
"type": "integer",
|
|
592
|
+
"exclusiveMinimum": 0,
|
|
593
|
+
"maximum": 9007199254740991
|
|
520
594
|
}
|
|
521
595
|
},
|
|
522
596
|
"required": [
|
|
@@ -170,6 +170,12 @@
|
|
|
170
170
|
"minimum": 0,
|
|
171
171
|
"maximum": 9007199254740991,
|
|
172
172
|
"description": "References that name no bytes, or bytes the layer does not hold"
|
|
173
|
+
},
|
|
174
|
+
"missing": {
|
|
175
|
+
"description": "Of `unavailable`, those that name a file the layer's bundle does not hold (stored, then gone). Absent when there are none",
|
|
176
|
+
"type": "integer",
|
|
177
|
+
"exclusiveMinimum": 0,
|
|
178
|
+
"maximum": 9007199254740991
|
|
173
179
|
}
|
|
174
180
|
},
|
|
175
181
|
"required": [
|
|
@@ -179,6 +185,64 @@
|
|
|
179
185
|
],
|
|
180
186
|
"additionalProperties": {}
|
|
181
187
|
},
|
|
188
|
+
"recordings": {
|
|
189
|
+
"description": "Whether the recordings its layers expected were published, apart from its outcomes. Absent when no layer declared an expectation: its completeness is unknown, never complete. Matched by the `recordings` filter",
|
|
190
|
+
"type": "object",
|
|
191
|
+
"properties": {
|
|
192
|
+
"state": {
|
|
193
|
+
"type": "string",
|
|
194
|
+
"enum": [
|
|
195
|
+
"complete",
|
|
196
|
+
"incomplete",
|
|
197
|
+
"none-expected",
|
|
198
|
+
"unknown"
|
|
199
|
+
],
|
|
200
|
+
"description": "`incomplete` when an expected recording was skipped or unresolved; else `unknown` when a layer declared no expectations; else `complete` when something was expected and `none-expected` when nothing was"
|
|
201
|
+
},
|
|
202
|
+
"expected": {
|
|
203
|
+
"type": "integer",
|
|
204
|
+
"minimum": 0,
|
|
205
|
+
"maximum": 9007199254740991
|
|
206
|
+
},
|
|
207
|
+
"published": {
|
|
208
|
+
"type": "integer",
|
|
209
|
+
"minimum": 0,
|
|
210
|
+
"maximum": 9007199254740991,
|
|
211
|
+
"description": "Of the expected recordings, uploaded or referenced by address"
|
|
212
|
+
},
|
|
213
|
+
"skipped": {
|
|
214
|
+
"type": "integer",
|
|
215
|
+
"minimum": 0,
|
|
216
|
+
"maximum": 9007199254740991
|
|
217
|
+
},
|
|
218
|
+
"unresolved": {
|
|
219
|
+
"type": "integer",
|
|
220
|
+
"minimum": 0,
|
|
221
|
+
"maximum": 9007199254740991
|
|
222
|
+
},
|
|
223
|
+
"notApplicable": {
|
|
224
|
+
"type": "integer",
|
|
225
|
+
"minimum": 0,
|
|
226
|
+
"maximum": 9007199254740991
|
|
227
|
+
},
|
|
228
|
+
"layersWithoutReceipt": {
|
|
229
|
+
"type": "integer",
|
|
230
|
+
"minimum": 0,
|
|
231
|
+
"maximum": 9007199254740991,
|
|
232
|
+
"description": "Layers of the run that declared no recording expectations: nothing is claimed for them"
|
|
233
|
+
}
|
|
234
|
+
},
|
|
235
|
+
"required": [
|
|
236
|
+
"state",
|
|
237
|
+
"expected",
|
|
238
|
+
"published",
|
|
239
|
+
"skipped",
|
|
240
|
+
"unresolved",
|
|
241
|
+
"notApplicable",
|
|
242
|
+
"layersWithoutReceipt"
|
|
243
|
+
],
|
|
244
|
+
"additionalProperties": {}
|
|
245
|
+
},
|
|
182
246
|
"attachments": {
|
|
183
247
|
"description": "Files attached to the run itself, in its layers, not to a check. Absent when there are none",
|
|
184
248
|
"type": "integer",
|
|
@@ -419,6 +483,28 @@
|
|
|
419
483
|
],
|
|
420
484
|
"additionalProperties": {}
|
|
421
485
|
}
|
|
486
|
+
},
|
|
487
|
+
"recordings": {
|
|
488
|
+
"description": "Only for runs: `complete`, `incomplete`, `none-expected` and `unknown` with how many runs have each. Absent on servers that predate the filter",
|
|
489
|
+
"type": "array",
|
|
490
|
+
"items": {
|
|
491
|
+
"type": "object",
|
|
492
|
+
"properties": {
|
|
493
|
+
"value": {
|
|
494
|
+
"type": "string"
|
|
495
|
+
},
|
|
496
|
+
"count": {
|
|
497
|
+
"type": "integer",
|
|
498
|
+
"minimum": 0,
|
|
499
|
+
"maximum": 9007199254740991
|
|
500
|
+
}
|
|
501
|
+
},
|
|
502
|
+
"required": [
|
|
503
|
+
"value",
|
|
504
|
+
"count"
|
|
505
|
+
],
|
|
506
|
+
"additionalProperties": {}
|
|
507
|
+
}
|
|
422
508
|
}
|
|
423
509
|
},
|
|
424
510
|
"required": [
|
|
@@ -196,6 +196,10 @@
|
|
|
196
196
|
"type": "boolean",
|
|
197
197
|
"description": "False when the reference names no bytes: no bundle key, address or inline text, or a bundle key that the layer's own bundle does not hold"
|
|
198
198
|
},
|
|
199
|
+
"reason": {
|
|
200
|
+
"description": "Why it is unavailable, when the report can tell: `not-in-bundle` is a file the layer names but does not hold (stored, then gone). An open vocabulary: treat an unknown reason as plain unavailable",
|
|
201
|
+
"type": "string"
|
|
202
|
+
},
|
|
199
203
|
"scope": {
|
|
200
204
|
"description": "What this recording shows as its producer declared it: its own `scope`, else its attempt's, check's, part's or layer's. Absent: the scope is unknown, never live, physical or audible",
|
|
201
205
|
"type": "object",
|
|
@@ -221,6 +221,10 @@
|
|
|
221
221
|
"type": "boolean",
|
|
222
222
|
"description": "False when the reference names no bytes: no bundle key, address or inline text, or a bundle key that the layer's own bundle does not hold"
|
|
223
223
|
},
|
|
224
|
+
"reason": {
|
|
225
|
+
"description": "Why it is unavailable, when the report can tell: `not-in-bundle` is a file the layer names but does not hold (stored, then gone). An open vocabulary: treat an unknown reason as plain unavailable",
|
|
226
|
+
"type": "string"
|
|
227
|
+
},
|
|
224
228
|
"scope": {
|
|
225
229
|
"description": "What this recording shows as its producer declared it: its own `scope`, else its attempt's, check's, part's or layer's. Absent: the scope is unknown, never live, physical or audible",
|
|
226
230
|
"type": "object",
|
|
@@ -1003,6 +1003,10 @@
|
|
|
1003
1003
|
"type": "boolean",
|
|
1004
1004
|
"description": "False when the reference names no bytes: no bundle key, address or inline text, or a bundle key that the layer's own bundle does not hold"
|
|
1005
1005
|
},
|
|
1006
|
+
"reason": {
|
|
1007
|
+
"description": "Why it is unavailable, when the report can tell: `not-in-bundle` is a file the layer names but does not hold (stored, then gone). An open vocabulary: treat an unknown reason as plain unavailable",
|
|
1008
|
+
"type": "string"
|
|
1009
|
+
},
|
|
1006
1010
|
"scope": {
|
|
1007
1011
|
"description": "What this recording shows as its producer declared it: its own `scope`, else its attempt's, check's, part's or layer's. Absent: the scope is unknown, never live, physical or audible",
|
|
1008
1012
|
"type": "object",
|
|
@@ -297,6 +297,10 @@
|
|
|
297
297
|
"type": "boolean",
|
|
298
298
|
"description": "False when the reference names no bytes: no bundle key, address or inline text, or a bundle key that the layer's own bundle does not hold"
|
|
299
299
|
},
|
|
300
|
+
"reason": {
|
|
301
|
+
"description": "Why it is unavailable, when the report can tell: `not-in-bundle` is a file the layer names but does not hold (stored, then gone). An open vocabulary: treat an unknown reason as plain unavailable",
|
|
302
|
+
"type": "string"
|
|
303
|
+
},
|
|
300
304
|
"scope": {
|
|
301
305
|
"description": "What this recording shows as its producer declared it: its own `scope`, else its attempt's, check's, part's or layer's. Absent: the scope is unknown, never live, physical or audible",
|
|
302
306
|
"type": "object",
|
|
@@ -968,6 +972,10 @@
|
|
|
968
972
|
"type": "boolean",
|
|
969
973
|
"description": "False when the reference names no bytes: no bundle key, address or inline text, or a bundle key that the layer's own bundle does not hold"
|
|
970
974
|
},
|
|
975
|
+
"reason": {
|
|
976
|
+
"description": "Why it is unavailable, when the report can tell: `not-in-bundle` is a file the layer names but does not hold (stored, then gone). An open vocabulary: treat an unknown reason as plain unavailable",
|
|
977
|
+
"type": "string"
|
|
978
|
+
},
|
|
971
979
|
"scope": {
|
|
972
980
|
"description": "What this recording shows as its producer declared it: its own `scope`, else its attempt's, check's, part's or layer's. Absent: the scope is unknown, never live, physical or audible",
|
|
973
981
|
"type": "object",
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$id": "https://github.com/crvouga/atlas/schemas/verify-recordings-result.schema.json",
|
|
3
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"properties": {
|
|
6
|
+
"ok": {
|
|
7
|
+
"type": "boolean",
|
|
8
|
+
"description": "True when no layer has a gap and every layer could be read (otherwise the exit code is 1). Test results are unaffected either way"
|
|
9
|
+
},
|
|
10
|
+
"layers": {
|
|
11
|
+
"type": "integer",
|
|
12
|
+
"minimum": 0,
|
|
13
|
+
"maximum": 9007199254740991,
|
|
14
|
+
"description": "Layers examined"
|
|
15
|
+
},
|
|
16
|
+
"checked": {
|
|
17
|
+
"type": "integer",
|
|
18
|
+
"minimum": 0,
|
|
19
|
+
"maximum": 9007199254740991,
|
|
20
|
+
"description": "Videos read back from the store (those that name a bundle key)"
|
|
21
|
+
},
|
|
22
|
+
"verified": {
|
|
23
|
+
"type": "integer",
|
|
24
|
+
"minimum": 0,
|
|
25
|
+
"maximum": 9007199254740991,
|
|
26
|
+
"description": "Of those, the ones that passed every check"
|
|
27
|
+
},
|
|
28
|
+
"referenced": {
|
|
29
|
+
"type": "integer",
|
|
30
|
+
"minimum": 0,
|
|
31
|
+
"maximum": 9007199254740991,
|
|
32
|
+
"description": "Videos given only by address, which Atlas holds no bytes for and does not check"
|
|
33
|
+
},
|
|
34
|
+
"gaps": {
|
|
35
|
+
"type": "array",
|
|
36
|
+
"items": {
|
|
37
|
+
"type": "object",
|
|
38
|
+
"properties": {
|
|
39
|
+
"layer": {
|
|
40
|
+
"type": "string",
|
|
41
|
+
"description": "The layer that holds the video"
|
|
42
|
+
},
|
|
43
|
+
"check": {
|
|
44
|
+
"description": "The check that holds it, or null for a video attached to the layer or a part",
|
|
45
|
+
"type": [
|
|
46
|
+
"string",
|
|
47
|
+
"null"
|
|
48
|
+
]
|
|
49
|
+
},
|
|
50
|
+
"name": {
|
|
51
|
+
"type": "string",
|
|
52
|
+
"description": "The attachment's name"
|
|
53
|
+
},
|
|
54
|
+
"bundleKey": {
|
|
55
|
+
"type": "string",
|
|
56
|
+
"description": "Its key in the layer's bundle (`attachments/<sha256>`)"
|
|
57
|
+
},
|
|
58
|
+
"problem": {
|
|
59
|
+
"type": "string",
|
|
60
|
+
"description": "`not-in-bundle` (the layer names a file its bundle does not list), `blob-missing` (a blob holding it is gone: a 404 when played), `unreadable` (the store could not be read: not proof it is gone), `size-mismatch`, `checksum-mismatch` or `undecodable` (empty, or not the container its media type names). An open vocabulary: treat an unknown problem as a gap"
|
|
61
|
+
},
|
|
62
|
+
"reason": {
|
|
63
|
+
"type": "string",
|
|
64
|
+
"description": "A short explanation without tokens, paths or payload"
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"required": [
|
|
68
|
+
"layer",
|
|
69
|
+
"check",
|
|
70
|
+
"name",
|
|
71
|
+
"bundleKey",
|
|
72
|
+
"problem",
|
|
73
|
+
"reason"
|
|
74
|
+
],
|
|
75
|
+
"additionalProperties": {}
|
|
76
|
+
},
|
|
77
|
+
"description": "Every video that is missing, unreadable, corrupt or not the container it claims. Re-publish the layer's recordings to repair them"
|
|
78
|
+
},
|
|
79
|
+
"notices": {
|
|
80
|
+
"type": "array",
|
|
81
|
+
"items": {
|
|
82
|
+
"type": "string"
|
|
83
|
+
},
|
|
84
|
+
"description": "Layers that could not be read at all: their recordings are unknown, never verified"
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
"required": [
|
|
88
|
+
"ok",
|
|
89
|
+
"layers",
|
|
90
|
+
"checked",
|
|
91
|
+
"verified",
|
|
92
|
+
"referenced",
|
|
93
|
+
"gaps",
|
|
94
|
+
"notices"
|
|
95
|
+
],
|
|
96
|
+
"additionalProperties": {}
|
|
97
|
+
}
|