npm - @crvouga/atlas - Versions diffs - 1.7.0 → 1.9.0 - Mend

@crvouga/atlas 1.7.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (46) hide show

package/CHANGELOG.md +13 -1
package/dist/reports/src/evidence-detail.js +6 -1
package/dist/reports/src/evidence-types.d.ts +2 -0
package/dist/reports/src/explorer-types.d.ts +19 -0
package/dist/reports/src/latency.d.ts +204 -0
package/dist/reports/src/latency.js +219 -0
package/dist/reports/src/reliability.d.ts +181 -0
package/dist/reports/src/reliability.js +294 -0
package/dist/reports/src/report-index.js +1 -1
package/dist/reports/src/reports-explorer-route.d.ts +19 -6
package/dist/reports/src/reports-route.d.ts +19 -6
package/dist/reports/src/test-catalog.d.ts +11 -3
package/dist/reports/src/test-catalog.js +12 -5
package/dist/schema/src/api.d.ts +377 -0
package/dist/schema/src/api.js +55 -3
package/dist/schema/src/layer.d.ts +90 -0
package/dist/schema/src/layer.js +20 -1
package/dist/server/src/app.js +15 -0
package/dist/server/src/llms.js +1 -0
package/dist/server/src/openapi.js +25 -2
package/docs/extending.md +1 -1
package/docs/reference/http-api.md +48 -0
package/docs/reference/reports.md +619 -1
package/docs/reference/schema.md +39 -0
package/docs/rich-evidence.md +43 -0
package/docs/server.md +18 -0
package/docs/ui.md +20 -1
package/llms-full.txt +130 -2
package/package.json +1 -1
package/packages/schema/schemas/check-detail.schema.json +56 -1
package/packages/schema/schemas/effective-delta.schema.json +86 -2
package/packages/schema/schemas/effective-evidence.schema.json +86 -2
package/packages/schema/schemas/evidence-page.schema.json +13 -0
package/packages/schema/schemas/layer-rows.schema.json +73 -2
package/packages/schema/schemas/layer.schema.json +112 -2
package/packages/schema/schemas/publish-checks.schema.json +56 -1
package/packages/schema/schemas/test-catalog.schema.json +102 -0
package/packages/schema/schemas/test-history.schema.json +204 -0
package/packages/schema/schemas/test-reliability.schema.json +629 -0
package/packages/server/openapi.json +2354 -661
package/packages/visualizer/src/evidence/check-detail.tsx +9 -0
package/packages/visualizer/src/explorer/ProductLatency.tsx +214 -0
package/packages/visualizer/src/explorer/Reliability.tsx +337 -0
package/packages/visualizer/src/explorer/TestDetail.tsx +12 -2
package/packages/visualizer/src/explorer/catalog.tsx +1 -1
package/packages/visualizer/src/explorer/search.ts +36 -0

package/packages/schema/schemas/test-reliability.schema.json ADDED Viewed

@@ -0,0 +1,629 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/test-reliability.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "number",
+      "const": 1
+    },
+    "key": {
+      "type": "string",
+      "description": "The test's key"
+    },
+    "scope": {
+      "type": "object",
+      "properties": {
+        "environment": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "producer": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "parameter": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "branch": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "sha": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "specRevision": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "revision": {
+          "type": "string",
+          "enum": [
+            "current",
+            "any"
+          ]
+        },
+        "since": {
+          "type": "number"
+        },
+        "until": {
+          "type": "number"
+        }
+      },
+      "additionalProperties": {},
+      "description": "What was asked: the selection the figures are about"
+    },
+    "minimumExecutions": {
+      "type": "integer",
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991,
+      "description": "Completed executions below which the sample is `insufficient`"
+    },
+    "sample": {
+      "type": "string",
+      "enum": [
+        "none",
+        "insufficient",
+        "sufficient"
+      ],
+      "description": "A count only: it says nothing about whether the executions are representative; read `notes`"
+    },
+    "eventual": {
+      "type": "object",
+      "properties": {
+        "executions": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991,
+          "description": "Independent executions that passed or failed"
+        },
+        "passed": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "failed": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "rate": {
+          "description": "passed / executions, or null with none",
+          "type": [
+            "number",
+            "null"
+          ]
+        }
+      },
+      "required": [
+        "executions",
+        "passed",
+        "failed",
+        "rate"
+      ],
+      "additionalProperties": {},
+      "description": "Executions that passed in the end, over those that passed or failed"
+    },
+    "interval": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "method": {
+              "type": "string",
+              "const": "wilson"
+            },
+            "level": {
+              "type": "number"
+            },
+            "lower": {
+              "type": "number"
+            },
+            "upper": {
+              "type": "number"
+            },
+            "assumptions": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "method",
+            "level",
+            "lower",
+            "upper",
+            "assumptions"
+          ],
+          "additionalProperties": {}
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "An interval estimate with the assumptions it needs; null without an execution"
+    },
+    "firstAttempt": {
+      "type": "object",
+      "properties": {
+        "executions": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991,
+          "description": "Independent executions that passed or failed"
+        },
+        "passed": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "failed": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "rate": {
+          "description": "passed / executions, or null with none",
+          "type": [
+            "number",
+            "null"
+          ]
+        }
+      },
+      "required": [
+        "executions",
+        "passed",
+        "failed",
+        "rate"
+      ],
+      "additionalProperties": {},
+      "description": "Over the executions that recorded their tries: those whose first try passed"
+    },
+    "unrecordedAttempts": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "Counted executions that recorded no tries: their first try is unknown, never assumed to have passed"
+    },
+    "attempts": {
+      "type": "object",
+      "properties": {
+        "total": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "retried": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "recovered": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        }
+      },
+      "required": [
+        "total",
+        "retried",
+        "recovered"
+      ],
+      "additionalProperties": {}
+    },
+    "currentRevision": {
+      "description": "The spec revision `revision=current` counted, or null",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "historical": {
+      "type": "object",
+      "properties": {
+        "executions": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "passed": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "failed": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "revisions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "specRevision": {
+                "type": "string"
+              },
+              "executions": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              }
+            },
+            "required": [
+              "specRevision",
+              "executions"
+            ],
+            "additionalProperties": {}
+          }
+        }
+      },
+      "required": [
+        "executions",
+        "passed",
+        "failed",
+        "revisions"
+      ],
+      "additionalProperties": {},
+      "description": "Earlier spec revisions: context only, counted nowhere above"
+    },
+    "excluded": {
+      "type": "object",
+      "propertyNames": {
+        "type": "string"
+      },
+      "additionalProperties": {
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 9007199254740991
+      },
+      "description": "Executions in the scope that are not trials, by reason: `not-run`, `unfinished`, `unknown-status`, `historical-revision`, `revision-not-recorded`"
+    },
+    "outsideScope": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "Executions the scope did not select"
+    },
+    "snapshots": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "Raw recorded snapshots behind the counted executions"
+    },
+    "cohorts": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "executions": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 9007199254740991,
+            "description": "Independent executions that passed or failed"
+          },
+          "passed": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 9007199254740991
+          },
+          "failed": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 9007199254740991
+          },
+          "rate": {
+            "description": "passed / executions, or null with none",
+            "type": [
+              "number",
+              "null"
+            ]
+          },
+          "environment": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "producer": {
+            "type": "string"
+          },
+          "parameters": {
+            "type": "object",
+            "propertyNames": {
+              "type": "string"
+            },
+            "additionalProperties": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "executions",
+          "passed",
+          "failed",
+          "rate",
+          "environment",
+          "producer",
+          "parameters"
+        ],
+        "additionalProperties": {}
+      }
+    },
+    "available": {
+      "type": "object",
+      "properties": {
+        "environments": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "value": {
+                "type": "string"
+              },
+              "count": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              }
+            },
+            "required": [
+              "value",
+              "count"
+            ],
+            "additionalProperties": {}
+          }
+        },
+        "producers": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "value": {
+                "type": "string"
+              },
+              "count": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              }
+            },
+            "required": [
+              "value",
+              "count"
+            ],
+            "additionalProperties": {}
+          }
+        },
+        "branches": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "value": {
+                "type": "string"
+              },
+              "count": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              }
+            },
+            "required": [
+              "value",
+              "count"
+            ],
+            "additionalProperties": {}
+          }
+        },
+        "specRevisions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "value": {
+                "type": "string"
+              },
+              "count": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              }
+            },
+            "required": [
+              "value",
+              "count"
+            ],
+            "additionalProperties": {}
+          }
+        }
+      },
+      "required": [
+        "environments",
+        "producers",
+        "branches",
+        "specRevisions"
+      ],
+      "additionalProperties": {},
+      "description": "What a scope could choose among, with independent executions"
+    },
+    "notes": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "code": {
+            "type": "string",
+            "description": "An open set: treat a code you do not know as informational"
+          },
+          "severity": {
+            "type": "string",
+            "enum": [
+              "warning",
+              "info"
+            ]
+          },
+          "message": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "code",
+          "severity",
+          "message"
+        ],
+        "additionalProperties": {}
+      }
+    },
+    "executions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "run": {
+            "type": "string"
+          },
+          "at": {
+            "type": "number"
+          },
+          "status": {
+            "type": "string"
+          },
+          "counted": {
+            "type": "boolean"
+          },
+          "excluded": {
+            "type": "string"
+          },
+          "environment": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "producer": {
+            "type": "string"
+          },
+          "kind": {
+            "type": "string"
+          },
+          "parameters": {
+            "type": "object",
+            "propertyNames": {
+              "type": "string"
+            },
+            "additionalProperties": {
+              "type": "string"
+            }
+          },
+          "branch": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "sha": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "runUrl": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "layer": {
+            "type": "string"
+          },
+          "backend": {
+            "type": "string"
+          },
+          "durationMs": {
+            "type": [
+              "number",
+              "null"
+            ]
+          },
+          "specRevision": {
+            "type": "string"
+          },
+          "snapshots": {
+            "type": "integer",
+            "exclusiveMinimum": 0,
+            "maximum": 9007199254740991
+          },
+          "attempts": {
+            "description": "The status of each recorded try, in order",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "firstAttempt": {
+            "type": "string"
+          },
+          "republished": {
+            "type": "boolean",
+            "description": "A failed snapshot of this execution was followed by a later one"
+          }
+        },
+        "required": [
+          "run",
+          "at",
+          "status",
+          "counted",
+          "environment",
+          "producer",
+          "kind",
+          "parameters",
+          "branch",
+          "sha",
+          "runUrl",
+          "layer",
+          "backend",
+          "durationMs",
+          "snapshots",
+          "republished"
+        ],
+        "additionalProperties": {}
+      },
+      "description": "Every independent execution of the selection, newest first: the denominator, open to inspection"
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "key",
+    "scope",
+    "minimumExecutions",
+    "sample",
+    "eventual",
+    "interval",
+    "firstAttempt",
+    "unrecordedAttempts",
+    "attempts",
+    "currentRevision",
+    "historical",
+    "excluded",
+    "outsideScope",
+    "snapshots",
+    "cohorts",
+    "available",
+    "notes",
+    "executions"
+  ],
+  "additionalProperties": {}
+}