@crvouga/atlas 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/core/src/lint.js +6 -2
  3. package/dist/core/src/spec/types.d.ts +4 -1
  4. package/dist/playwright-reporter/src/evidence.d.ts +122 -0
  5. package/dist/playwright-reporter/src/evidence.js +327 -0
  6. package/dist/playwright-reporter/src/index.d.ts +114 -0
  7. package/dist/playwright-reporter/src/index.js +312 -0
  8. package/dist/reports/src/compose.js +7 -14
  9. package/dist/reports/src/recorded-specs.d.ts +33 -0
  10. package/dist/reports/src/recorded-specs.js +42 -0
  11. package/dist/reports/src/report-index.d.ts +6 -0
  12. package/dist/reports/src/report-index.js +14 -2
  13. package/dist/reports/src/test-catalog.d.ts +386 -0
  14. package/dist/reports/src/test-catalog.js +655 -0
  15. package/dist/schema/src/api.d.ts +585 -0
  16. package/dist/schema/src/api.js +105 -0
  17. package/dist/schema/src/spec.d.ts +2 -1
  18. package/dist/schema/src/spec.js +5 -2
  19. package/dist/server/src/app.js +82 -4
  20. package/dist/server/src/llms.js +14 -0
  21. package/dist/server/src/openapi.d.ts +1 -1
  22. package/dist/server/src/openapi.js +60 -3
  23. package/docs/agents.md +1 -0
  24. package/docs/integrating.md +1 -0
  25. package/docs/reference/core.md +7 -2
  26. package/docs/reference/http-api.md +134 -2
  27. package/docs/reference/playwright-reporter.md +332 -0
  28. package/docs/reference/reports.md +632 -0
  29. package/docs/reference/schema.md +58 -1
  30. package/docs/rich-evidence.md +198 -0
  31. package/docs/server.md +41 -0
  32. package/docs/statecharts.md +2 -1
  33. package/llms-full.txt +380 -3
  34. package/llms.txt +2 -0
  35. package/package.json +6 -1
  36. package/packages/schema/schemas/report-run.schema.json +251 -0
  37. package/packages/schema/schemas/report-runs.schema.json +177 -0
  38. package/packages/schema/schemas/state-node.schema.json +7 -0
  39. package/packages/schema/schemas/test-catalog.schema.json +1177 -0
  40. package/packages/schema/schemas/test-history.schema.json +696 -0
  41. package/packages/server/openapi.json +8676 -5351
  42. package/packages/visualizer/src/app/AppShell.tsx +7 -3
  43. package/packages/visualizer/src/app/router.tsx +26 -3
  44. package/packages/visualizer/src/components/Header.tsx +2 -2
  45. package/packages/visualizer/src/evidence/evidence-context.tsx +10 -1
  46. package/packages/visualizer/src/explorer/Explorer.tsx +433 -0
  47. package/packages/visualizer/src/explorer/ExplorerIndex.tsx +120 -0
  48. package/packages/visualizer/src/explorer/RunsView.tsx +164 -0
  49. package/packages/visualizer/src/explorer/TestDetail.tsx +214 -0
  50. package/packages/visualizer/src/explorer/catalog.tsx +274 -0
  51. package/packages/visualizer/src/explorer/explorer.module.css +1238 -0
  52. package/packages/visualizer/src/explorer/groups.test.ts +87 -0
  53. package/packages/visualizer/src/explorer/groups.ts +126 -0
  54. package/packages/visualizer/src/explorer/navigate.ts +36 -0
  55. package/packages/visualizer/src/explorer/parts.tsx +59 -0
  56. package/packages/visualizer/src/explorer/search.ts +97 -0
  57. package/packages/visualizer/src/find/Finder.tsx +2 -0
  58. package/packages/visualizer/src/find/corpus.ts +13 -6
  59. package/packages/visualizer/src/find/rank.ts +2 -0
  60. package/packages/visualizer/src/evidence/project-home.tsx +0 -45
package/packages/schema/schemas/test-history.schema.json ADDED
@@ -0,0 +1,696 @@
1
+ {
2
+ "$id": "https://github.com/crvouga/atlas/schemas/test-history.schema.json",
3
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
4
+ "type": "object",
5
+ "properties": {
6
+ "schemaVersion": {
7
+ "type": "number",
8
+ "const": 1
9
+ },
10
+ "version": {
11
+ "type": "string"
12
+ },
13
+ "test": {
14
+ "type": "object",
15
+ "properties": {
16
+ "key": {
17
+ "description": "The test's identity, `JSON.stringify([specPath, id])`. Pass it to `/api/reports/tests/history?key=`",
18
+ "type": "string"
19
+ },
20
+ "id": {
21
+ "type": "string"
22
+ },
23
+ "title": {
24
+ "type": "string"
25
+ },
26
+ "specPath": {
27
+ "anyOf": [
28
+ {
29
+ "type": "string"
30
+ },
31
+ {
32
+ "type": "null"
33
+ }
34
+ ]
35
+ },
36
+ "spec": {
37
+ "description": "The spec it belongs to: recorded by its layers, published in a spec set, or derived from the path",
38
+ "anyOf": [
39
+ {
40
+ "type": "object",
41
+ "properties": {
42
+ "path": {
43
+ "type": "string"
44
+ },
45
+ "title": {
46
+ "type": "string"
47
+ },
48
+ "domain": {
49
+ "type": "string"
50
+ },
51
+ "kind": {
52
+ "type": "string"
53
+ }
54
+ },
55
+ "required": [
56
+ "path",
57
+ "title",
58
+ "domain",
59
+ "kind"
60
+ ],
61
+ "additionalProperties": {}
62
+ },
63
+ {
64
+ "type": "null"
65
+ }
66
+ ]
67
+ },
68
+ "status": {
69
+ "description": "The worst current result across environments: failed, then running, then not-run, then passed",
70
+ "type": "string",
71
+ "enum": [
72
+ "passed",
73
+ "failed",
74
+ "running",
75
+ "not-run"
76
+ ]
77
+ },
78
+ "environments": {
79
+ "type": "array",
80
+ "items": {
81
+ "type": "string"
82
+ }
83
+ },
84
+ "producers": {
85
+ "type": "array",
86
+ "items": {
87
+ "type": "string"
88
+ }
89
+ },
90
+ "outcomes": {
91
+ "description": "The current result per producer, environment and parameter set, newest first",
92
+ "type": "array",
93
+ "items": {
94
+ "type": "object",
95
+ "properties": {
96
+ "at": {
97
+ "description": "When it was recorded, epoch milliseconds",
98
+ "type": "number"
99
+ },
100
+ "status": {
101
+ "type": "string",
102
+ "enum": [
103
+ "passed",
104
+ "failed",
105
+ "running",
106
+ "not-run"
107
+ ]
108
+ },
109
+ "durationMs": {
110
+ "description": "How long it took, or null when the producer did not measure it",
111
+ "anyOf": [
112
+ {
113
+ "type": "number"
114
+ },
115
+ {
116
+ "type": "null"
117
+ }
118
+ ]
119
+ },
120
+ "environment": {
121
+ "description": "The client it checked, such as `desktop-web` or `ios`",
122
+ "anyOf": [
123
+ {
124
+ "type": "string"
125
+ },
126
+ {
127
+ "type": "null"
128
+ }
129
+ ]
130
+ },
131
+ "producer": {
132
+ "type": "string"
133
+ },
134
+ "kind": {
135
+ "type": "string"
136
+ },
137
+ "parameters": {
138
+ "type": "object",
139
+ "propertyNames": {
140
+ "type": "string"
141
+ },
142
+ "additionalProperties": {
143
+ "type": "string"
144
+ }
145
+ },
146
+ "branch": {
147
+ "anyOf": [
148
+ {
149
+ "type": "string"
150
+ },
151
+ {
152
+ "type": "null"
153
+ }
154
+ ]
155
+ },
156
+ "sha": {
157
+ "anyOf": [
158
+ {
159
+ "type": "string"
160
+ },
161
+ {
162
+ "type": "null"
163
+ }
164
+ ]
165
+ },
166
+ "runUrl": {
167
+ "anyOf": [
168
+ {
169
+ "type": "string"
170
+ },
171
+ {
172
+ "type": "null"
173
+ }
174
+ ]
175
+ },
176
+ "run": {
177
+ "description": "The key of the run it belongs to, as `GET /api/reports/runs` lists it",
178
+ "type": "string"
179
+ },
180
+ "layer": {
181
+ "description": "The id of the layer that recorded it",
182
+ "type": "string"
183
+ },
184
+ "backend": {
185
+ "type": "string"
186
+ },
187
+ "detail": {
188
+ "description": "Why it has its status, at most 500 characters",
189
+ "type": "string"
190
+ },
191
+ "evidence": {
192
+ "type": "object",
193
+ "properties": {
194
+ "attachments": {
195
+ "type": "integer",
196
+ "minimum": 0,
197
+ "maximum": 9007199254740991
198
+ },
199
+ "steps": {
200
+ "type": "integer",
201
+ "minimum": 0,
202
+ "maximum": 9007199254740991
203
+ },
204
+ "annotations": {
205
+ "type": "integer",
206
+ "minimum": 0,
207
+ "maximum": 9007199254740991
208
+ },
209
+ "attempts": {
210
+ "type": "integer",
211
+ "minimum": 0,
212
+ "maximum": 9007199254740991
213
+ },
214
+ "links": {
215
+ "type": "integer",
216
+ "minimum": 0,
217
+ "maximum": 9007199254740991
218
+ },
219
+ "tags": {
220
+ "type": "array",
221
+ "items": {
222
+ "type": "string"
223
+ }
224
+ },
225
+ "labels": {
226
+ "type": "object",
227
+ "propertyNames": {
228
+ "type": "string"
229
+ },
230
+ "additionalProperties": {
231
+ "type": "string"
232
+ }
233
+ },
234
+ "roles": {
235
+ "type": "array",
236
+ "items": {
237
+ "type": "string"
238
+ }
239
+ }
240
+ },
241
+ "required": [
242
+ "attachments",
243
+ "steps",
244
+ "annotations",
245
+ "attempts",
246
+ "links",
247
+ "tags",
248
+ "labels",
249
+ "roles"
250
+ ],
251
+ "additionalProperties": {}
252
+ }
253
+ },
254
+ "required": [
255
+ "at",
256
+ "status",
257
+ "durationMs",
258
+ "environment",
259
+ "producer",
260
+ "kind",
261
+ "parameters",
262
+ "branch",
263
+ "sha",
264
+ "runUrl",
265
+ "run",
266
+ "layer",
267
+ "backend",
268
+ "detail"
269
+ ],
270
+ "additionalProperties": {}
271
+ }
272
+ },
273
+ "stats": {
274
+ "type": "object",
275
+ "properties": {
276
+ "results": {
277
+ "type": "integer",
278
+ "minimum": 0,
279
+ "maximum": 9007199254740991
280
+ },
281
+ "passed": {
282
+ "type": "integer",
283
+ "minimum": 0,
284
+ "maximum": 9007199254740991
285
+ },
286
+ "failed": {
287
+ "type": "integer",
288
+ "minimum": 0,
289
+ "maximum": 9007199254740991
290
+ },
291
+ "passRate": {
292
+ "description": "passed / (passed + failed), or null with no completed result",
293
+ "anyOf": [
294
+ {
295
+ "type": "number"
296
+ },
297
+ {
298
+ "type": "null"
299
+ }
300
+ ]
301
+ },
302
+ "flaky": {
303
+ "type": "boolean"
304
+ },
305
+ "flakyReason": {
306
+ "type": "string"
307
+ },
308
+ "flips": {
309
+ "type": "integer",
310
+ "minimum": 0,
311
+ "maximum": 9007199254740991
312
+ },
313
+ "durations": {
314
+ "anyOf": [
315
+ {
316
+ "type": "object",
317
+ "properties": {
318
+ "count": {
319
+ "type": "integer",
320
+ "minimum": 0,
321
+ "maximum": 9007199254740991
322
+ },
323
+ "totalMs": {
324
+ "type": "number"
325
+ },
326
+ "avgMs": {
327
+ "type": "number"
328
+ },
329
+ "p50Ms": {
330
+ "type": "number"
331
+ },
332
+ "p95Ms": {
333
+ "type": "number"
334
+ },
335
+ "minMs": {
336
+ "type": "number"
337
+ },
338
+ "maxMs": {
339
+ "type": "number"
340
+ },
341
+ "lastMs": {
342
+ "type": "number"
343
+ }
344
+ },
345
+ "required": [
346
+ "count",
347
+ "totalMs",
348
+ "avgMs",
349
+ "p50Ms",
350
+ "p95Ms",
351
+ "minMs",
352
+ "maxMs",
353
+ "lastMs"
354
+ ],
355
+ "additionalProperties": {}
356
+ },
357
+ {
358
+ "type": "null"
359
+ }
360
+ ]
361
+ },
362
+ "firstSeenMs": {
363
+ "anyOf": [
364
+ {
365
+ "type": "number"
366
+ },
367
+ {
368
+ "type": "null"
369
+ }
370
+ ]
371
+ },
372
+ "lastSeenMs": {
373
+ "anyOf": [
374
+ {
375
+ "type": "number"
376
+ },
377
+ {
378
+ "type": "null"
379
+ }
380
+ ]
381
+ },
382
+ "lastPassedAtMs": {
383
+ "anyOf": [
384
+ {
385
+ "type": "number"
386
+ },
387
+ {
388
+ "type": "null"
389
+ }
390
+ ]
391
+ },
392
+ "lastFailedAtMs": {
393
+ "anyOf": [
394
+ {
395
+ "type": "number"
396
+ },
397
+ {
398
+ "type": "null"
399
+ }
400
+ ]
401
+ },
402
+ "runs": {
403
+ "type": "integer",
404
+ "minimum": 0,
405
+ "maximum": 9007199254740991
406
+ },
407
+ "branches": {
408
+ "type": "array",
409
+ "items": {
410
+ "type": "string"
411
+ }
412
+ }
413
+ },
414
+ "required": [
415
+ "results",
416
+ "passed",
417
+ "failed",
418
+ "passRate",
419
+ "flaky",
420
+ "flakyReason",
421
+ "flips",
422
+ "durations",
423
+ "firstSeenMs",
424
+ "lastSeenMs",
425
+ "lastPassedAtMs",
426
+ "lastFailedAtMs",
427
+ "runs",
428
+ "branches"
429
+ ],
430
+ "additionalProperties": {}
431
+ },
432
+ "tags": {
433
+ "type": "array",
434
+ "items": {
435
+ "type": "string"
436
+ }
437
+ },
438
+ "labels": {
439
+ "type": "object",
440
+ "propertyNames": {
441
+ "type": "string"
442
+ },
443
+ "additionalProperties": {
444
+ "type": "string"
445
+ }
446
+ },
447
+ "evidence": {
448
+ "type": "object",
449
+ "properties": {
450
+ "attachments": {
451
+ "type": "integer",
452
+ "minimum": 0,
453
+ "maximum": 9007199254740991
454
+ },
455
+ "steps": {
456
+ "type": "integer",
457
+ "minimum": 0,
458
+ "maximum": 9007199254740991
459
+ },
460
+ "attempts": {
461
+ "type": "integer",
462
+ "minimum": 0,
463
+ "maximum": 9007199254740991
464
+ },
465
+ "roles": {
466
+ "type": "array",
467
+ "items": {
468
+ "type": "string"
469
+ }
470
+ }
471
+ },
472
+ "required": [
473
+ "attachments",
474
+ "steps",
475
+ "attempts",
476
+ "roles"
477
+ ],
478
+ "additionalProperties": {}
479
+ },
480
+ "lastRecordedAtMs": {
481
+ "type": "number"
482
+ },
483
+ "live": {
484
+ "type": "boolean"
485
+ }
486
+ },
487
+ "required": [
488
+ "key",
489
+ "id",
490
+ "title",
491
+ "specPath",
492
+ "spec",
493
+ "status",
494
+ "environments",
495
+ "producers",
496
+ "outcomes",
497
+ "stats",
498
+ "tags",
499
+ "labels",
500
+ "evidence",
501
+ "lastRecordedAtMs",
502
+ "live"
503
+ ],
504
+ "additionalProperties": {}
505
+ },
506
+ "history": {
507
+ "type": "array",
508
+ "items": {
509
+ "type": "object",
510
+ "properties": {
511
+ "at": {
512
+ "description": "When it was recorded, epoch milliseconds",
513
+ "type": "number"
514
+ },
515
+ "status": {
516
+ "type": "string",
517
+ "enum": [
518
+ "passed",
519
+ "failed",
520
+ "running",
521
+ "not-run"
522
+ ]
523
+ },
524
+ "durationMs": {
525
+ "description": "How long it took, or null when the producer did not measure it",
526
+ "anyOf": [
527
+ {
528
+ "type": "number"
529
+ },
530
+ {
531
+ "type": "null"
532
+ }
533
+ ]
534
+ },
535
+ "environment": {
536
+ "description": "The client it checked, such as `desktop-web` or `ios`",
537
+ "anyOf": [
538
+ {
539
+ "type": "string"
540
+ },
541
+ {
542
+ "type": "null"
543
+ }
544
+ ]
545
+ },
546
+ "producer": {
547
+ "type": "string"
548
+ },
549
+ "kind": {
550
+ "type": "string"
551
+ },
552
+ "parameters": {
553
+ "type": "object",
554
+ "propertyNames": {
555
+ "type": "string"
556
+ },
557
+ "additionalProperties": {
558
+ "type": "string"
559
+ }
560
+ },
561
+ "branch": {
562
+ "anyOf": [
563
+ {
564
+ "type": "string"
565
+ },
566
+ {
567
+ "type": "null"
568
+ }
569
+ ]
570
+ },
571
+ "sha": {
572
+ "anyOf": [
573
+ {
574
+ "type": "string"
575
+ },
576
+ {
577
+ "type": "null"
578
+ }
579
+ ]
580
+ },
581
+ "runUrl": {
582
+ "anyOf": [
583
+ {
584
+ "type": "string"
585
+ },
586
+ {
587
+ "type": "null"
588
+ }
589
+ ]
590
+ },
591
+ "run": {
592
+ "description": "The key of the run it belongs to, as `GET /api/reports/runs` lists it",
593
+ "type": "string"
594
+ },
595
+ "layer": {
596
+ "description": "The id of the layer that recorded it",
597
+ "type": "string"
598
+ },
599
+ "backend": {
600
+ "type": "string"
601
+ },
602
+ "detail": {
603
+ "description": "Why it has its status, at most 500 characters",
604
+ "type": "string"
605
+ },
606
+ "evidence": {
607
+ "type": "object",
608
+ "properties": {
609
+ "attachments": {
610
+ "type": "integer",
611
+ "minimum": 0,
612
+ "maximum": 9007199254740991
613
+ },
614
+ "steps": {
615
+ "type": "integer",
616
+ "minimum": 0,
617
+ "maximum": 9007199254740991
618
+ },
619
+ "annotations": {
620
+ "type": "integer",
621
+ "minimum": 0,
622
+ "maximum": 9007199254740991
623
+ },
624
+ "attempts": {
625
+ "type": "integer",
626
+ "minimum": 0,
627
+ "maximum": 9007199254740991
628
+ },
629
+ "links": {
630
+ "type": "integer",
631
+ "minimum": 0,
632
+ "maximum": 9007199254740991
633
+ },
634
+ "tags": {
635
+ "type": "array",
636
+ "items": {
637
+ "type": "string"
638
+ }
639
+ },
640
+ "labels": {
641
+ "type": "object",
642
+ "propertyNames": {
643
+ "type": "string"
644
+ },
645
+ "additionalProperties": {
646
+ "type": "string"
647
+ }
648
+ },
649
+ "roles": {
650
+ "type": "array",
651
+ "items": {
652
+ "type": "string"
653
+ }
654
+ }
655
+ },
656
+ "required": [
657
+ "attachments",
658
+ "steps",
659
+ "annotations",
660
+ "attempts",
661
+ "links",
662
+ "tags",
663
+ "labels",
664
+ "roles"
665
+ ],
666
+ "additionalProperties": {}
667
+ }
668
+ },
669
+ "required": [
670
+ "at",
671
+ "status",
672
+ "durationMs",
673
+ "environment",
674
+ "producer",
675
+ "kind",
676
+ "parameters",
677
+ "branch",
678
+ "sha",
679
+ "runUrl",
680
+ "run",
681
+ "layer",
682
+ "backend",
683
+ "detail"
684
+ ],
685
+ "additionalProperties": {}
686
+ }
687
+ }
688
+ },
689
+ "required": [
690
+ "schemaVersion",
691
+ "version",
692
+ "test",
693
+ "history"
694
+ ],
695
+ "additionalProperties": {}
696
+ }