@crvouga/atlas 0.4.0 → 0.6.0

package/packages/schema/schemas/server-settings-snapshot.schema.json

@@ -0,0 +1,285 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/server-settings-snapshot.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "revision": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    },
+    "settings": {
+      "type": "object",
+      "properties": {
+        "data": {
+          "description": "The folder that is the default backend and the default write target: layers, specs and runs live here",
+          "default": "./atlas-data",
+          "type": "string"
+        },
+        "backends": {
+          "description": "Further backends, read together with the data folder: S3, another server, a device lab",
+          "default": [],
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "id": {
+                "description": "Lowercase letters, digits and hyphens. Names where a result came from",
+                "type": "string",
+                "pattern": "^[a-z0-9-]+$"
+              },
+              "label": {
+                "type": "string"
+              },
+              "url": {
+                "description": "A folder, s3://bucket/prefix, r2://, gs://, or another Atlas server (https://…)",
+                "type": "string",
+                "minLength": 1
+              },
+              "write": {
+                "description": "Uploads go here instead of the data folder. At most one backend is the write target",
+                "type": "boolean"
+              }
+            },
+            "required": [
+              "id",
+              "url"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "write": {
+          "description": "The id of the backend uploads go to. Default: the data folder",
+          "type": "string"
+        },
+        "outlineBranch": {
+          "description": "Which branch's specs the report is laid over. Default main, then master, then the newest",
+          "type": "string"
+        },
+        "include": {
+          "default": {},
+          "type": "object",
+          "properties": {
+            "branches": {
+              "description": "Only layers recorded on these branches take part",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "ignoreModified": {
+              "description": "Leave out layers from modified checkouts and unknown origins",
+              "type": "boolean"
+            }
+          },
+          "additionalProperties": false
+        },
+        "refreshSeconds": {
+          "description": "How often the backends are looked at for layers written by something other than this server",
+          "default": 10,
+          "type": "number",
+          "minimum": 1
+        },
+        "limits": {
+          "default": {
+            "maxObjectMiB": 256,
+            "maxJsonMiB": 32,
+            "maxStreams": 500
+          },
+          "type": "object",
+          "properties": {
+            "maxObjectMiB": {
+              "description": "The largest object an upload may be",
+              "default": 256,
+              "type": "number",
+              "exclusiveMinimum": 0
+            },
+            "maxJsonMiB": {
+              "description": "The largest JSON body (a manifest, a checks request)",
+              "default": 32,
+              "type": "number",
+              "exclusiveMinimum": 0
+            },
+            "maxStreams": {
+              "description": "Viewers that may be connected for live updates at once",
+              "default": 500,
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "maximum": 9007199254740991
+            }
+          },
+          "additionalProperties": false
+        },
+        "heartbeatSeconds": {
+          "description": "How often a live connection is touched, so proxies keep it open",
+          "default": 15,
+          "type": "number",
+          "minimum": 1
+        },
+        "runTimeoutSeconds": {
+          "description": "Interrupt a live run after this many seconds without a producer heartbeat",
+          "default": 30,
+          "type": "number",
+          "minimum": 10,
+          "maximum": 3600
+        },
+        "runStallSeconds": {
+          "description": "Interrupt after this many seconds without test progress; 0 disables progress expiry",
+          "default": 900,
+          "type": "number",
+          "minimum": 0,
+          "maximum": 86400
+        },
+        "log": {
+          "description": "One JSON line per request",
+          "default": true,
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
+    "active": {
+      "type": "object",
+      "properties": {
+        "data": {
+          "description": "The folder that is the default backend and the default write target: layers, specs and runs live here",
+          "default": "./atlas-data",
+          "type": "string"
+        },
+        "backends": {
+          "description": "Further backends, read together with the data folder: S3, another server, a device lab",
+          "default": [],
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "id": {
+                "description": "Lowercase letters, digits and hyphens. Names where a result came from",
+                "type": "string",
+                "pattern": "^[a-z0-9-]+$"
+              },
+              "label": {
+                "type": "string"
+              },
+              "url": {
+                "description": "A folder, s3://bucket/prefix, r2://, gs://, or another Atlas server (https://…)",
+                "type": "string",
+                "minLength": 1
+              },
+              "write": {
+                "description": "Uploads go here instead of the data folder. At most one backend is the write target",
+                "type": "boolean"
+              }
+            },
+            "required": [
+              "id",
+              "url"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "write": {
+          "description": "The id of the backend uploads go to. Default: the data folder",
+          "type": "string"
+        },
+        "outlineBranch": {
+          "description": "Which branch's specs the report is laid over. Default main, then master, then the newest",
+          "type": "string"
+        },
+        "include": {
+          "default": {},
+          "type": "object",
+          "properties": {
+            "branches": {
+              "description": "Only layers recorded on these branches take part",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "ignoreModified": {
+              "description": "Leave out layers from modified checkouts and unknown origins",
+              "type": "boolean"
+            }
+          },
+          "additionalProperties": false
+        },
+        "refreshSeconds": {
+          "description": "How often the backends are looked at for layers written by something other than this server",
+          "default": 10,
+          "type": "number",
+          "minimum": 1
+        },
+        "limits": {
+          "default": {
+            "maxObjectMiB": 256,
+            "maxJsonMiB": 32,
+            "maxStreams": 500
+          },
+          "type": "object",
+          "properties": {
+            "maxObjectMiB": {
+              "description": "The largest object an upload may be",
+              "default": 256,
+              "type": "number",
+              "exclusiveMinimum": 0
+            },
+            "maxJsonMiB": {
+              "description": "The largest JSON body (a manifest, a checks request)",
+              "default": 32,
+              "type": "number",
+              "exclusiveMinimum": 0
+            },
+            "maxStreams": {
+              "description": "Viewers that may be connected for live updates at once",
+              "default": 500,
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "maximum": 9007199254740991
+            }
+          },
+          "additionalProperties": false
+        },
+        "heartbeatSeconds": {
+          "description": "How often a live connection is touched, so proxies keep it open",
+          "default": 15,
+          "type": "number",
+          "minimum": 1
+        },
+        "runTimeoutSeconds": {
+          "description": "Interrupt a live run after this many seconds without a producer heartbeat",
+          "default": 30,
+          "type": "number",
+          "minimum": 10,
+          "maximum": 3600
+        },
+        "runStallSeconds": {
+          "description": "Interrupt after this many seconds without test progress; 0 disables progress expiry",
+          "default": 900,
+          "type": "number",
+          "minimum": 0,
+          "maximum": 86400
+        },
+        "log": {
+          "description": "One JSON line per request",
+          "default": true,
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
+    "restartRequired": {
+      "type": "boolean"
+    },
+    "precedence": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "revision",
+    "settings",
+    "active",
+    "restartRequired",
+    "precedence"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/spec-set.schema.json

@@ -0,0 +1,103 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/spec-set.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "number",
+      "const": 1
+    },
+    "id": {
+      "description": "SHA-256 of the branch, commit, freshness and the digest of every file: the same specs from the same commit are the same set",
+      "type": "string",
+      "pattern": "^[0-9a-f]{64}$"
+    },
+    "branch": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "sha": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "recordedAtMs": {
+      "description": "For a clean checkout, the commit time; for a modified one, when it was published",
+      "type": "number",
+      "minimum": 0
+    },
+    "freshness": {
+      "type": "string",
+      "enum": [
+        "unknown",
+        "modified"
+      ]
+    },
+    "files": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "path": {
+            "description": "`specs/<path inside the specs directory>`, the name a check uses in `specPath`",
+            "type": "string",
+            "minLength": 1
+          },
+          "digest": {
+            "description": "SHA-256 of the file. Its text is at layers/specs/blobs/<digest>.txt",
+            "type": "string",
+            "pattern": "^[0-9a-f]{64}$"
+          },
+          "fingerprint": {
+            "description": "The advisory fingerprint layers record, to warn when a spec changed since they ran",
+            "type": "string"
+          },
+          "kind": {
+            "type": "string"
+          },
+          "title": {
+            "type": "string"
+          },
+          "domain": {
+            "type": "string"
+          },
+          "bytes": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 9007199254740991
+          }
+        },
+        "required": [
+          "path",
+          "digest",
+          "fingerprint",
+          "kind",
+          "title",
+          "domain",
+          "bytes"
+        ],
+        "additionalProperties": {}
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "id",
+    "branch",
+    "sha",
+    "recordedAtMs",
+    "files"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/store-listing.schema.json

@@ -0,0 +1,29 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/store-listing.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "keys": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "next": {
+      "description": "Pass as `?after=` for the next page; null when there are no more",
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    }
+  },
+  "required": [
+    "keys",
+    "next"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/store-write-result.schema.json

@@ -0,0 +1,28 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/store-write-result.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "ok": {
+      "type": "boolean",
+      "const": true
+    },
+    "key": {
+      "type": "string"
+    },
+    "kind": {
+      "description": "What the server recognised the key as: `layer`, `layer-catalog`, `blob`, `spec-body`, `spec-set`, `spec-catalog`, `run-manifest`, `run-catalog`, `run-file` or `scratch`",
+      "type": "string"
+    },
+    "created": {
+      "type": "boolean"
+    }
+  },
+  "required": [
+    "ok",
+    "key",
+    "kind",
+    "created"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/stream-result.schema.json

@@ -0,0 +1,59 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/stream-result.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "ok": {
+      "description": "False when any check could not be sent (the exit code is then 1)",
+      "type": "boolean"
+    },
+    "producer": {
+      "type": "string"
+    },
+    "environment": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "layers": {
+      "description": "Layers that arrived",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "checks": {
+      "description": "Checks that arrived",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "lost": {
+      "description": "Checks that never arrived",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "warnings": {
+      "description": "Input lines that were skipped, and sends that failed",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": [
+    "ok",
+    "producer",
+    "environment",
+    "layers",
+    "checks",
+    "lost",
+    "warnings"
+  ],
+  "additionalProperties": {}
+}

package/packages/server/README.md

@@ -0,0 +1,43 @@
+# `@crvouga/atlas/server`
+
+The Atlas server: one process per project that takes in results from every environment, keeps them, and shows
+everyone the combined report, live. A Bun API that serves the Vite-built viewer from the same host (and, under
+`atlas dev`, the viewer's dev server proxies `/api` to it, so development is one host too).
+
+Use it from the command line: `atlas serve`, `atlas dev` ([the guide](../../docs/server.md)). Use it from code when
+you want the server inside something else:
+
+```ts
+import { createAtlasServer } from '@crvouga/atlas/server';
+import { listen } from '@crvouga/atlas/server';
+import { directoryStore } from '@crvouga/atlas/reports/directory-store';
+
+const server = createAtlasServer({
+  backends: [{ id: 'data', store: directoryStore('./atlas-data') }],
+  write: 'data',                                  // where uploads go
+  tokens: [{ name: 'ci', role: 'publish', token: process.env.ATLAS_PUBLISH_TOKEN! }],
+  site: './atlas-site'                            // the built viewer; without it only the API is served
+});
+const running = await listen(server.fetch, { port: 8080, host: '127.0.0.1', maxBodyBytes: 256 * 1024 * 1024 });
+await server.ready;                               // the first pass over the backends is done
+```
+
+`server.fetch` is a plain `(Request) => Promise<Response>`, so it also mounts in Hono, Express (through an adapter),
+Cloudflare-style runtimes and tests. `server.index` is the live report (`ReportIndex`), `server.events` the stream
+to viewers, `server.close()` stops both.
+
+| Module | What |
+| --- | --- |
+| `app` | `createAtlasServer(options)`: the routes, the live report, compression and the page. |
+| `host` | `listen(handler, { port, host, maxBodyBytes })`: `Bun.serve`, or `node:http` where there is no Bun. |
+| `store-api`, `validate` | The upload protocol and what it refuses (content-addressed layers, dependency order, immutability). |
+| `events` | `EventHub`: server-sent events to every viewer, with a heartbeat and a backlog limit. |
+| `auth` | Roles (`read`, `publish`, `admin`), Bearer tokens, constant-time comparison. |
+| `config` | `loadServerConfig`: the file, then the environment, then the flags; every mistake said in words. |
+| `main`, `dev` | `atlas serve` and `atlas dev`. |
+
+The server describes itself: `GET /api/openapi.json` is an OpenAPI 3.1 document of every route (also `@crvouga/atlas/openapi.json`),
+`GET /llms.txt` a guide for an agent given only the address, and every error is an RFC 9457 problem document. See
+[Atlas for AI agents](../../docs/agents.md) and the [generated reference](../../docs/reference/server.md).
+
+`bun packages/server/bench/bench.ts [layers] [checks]` measures the server on a long history.