@crvouga/atlas 0.4.0 → 0.5.0

package/packages/schema/schemas/server-config.schema.json

@@ -0,0 +1,194 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/server-config.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "host": {
+      "description": "Where to listen. 127.0.0.1 keeps it on this machine; 0.0.0.0 is every interface and needs tokens (or `auth.insecure`)",
+      "default": "127.0.0.1",
+      "type": "string"
+    },
+    "port": {
+      "default": 8080,
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 65535
+    },
+    "data": {
+      "description": "The folder that is the default backend and the default write target: layers, specs and runs live here",
+      "default": "./atlas-data",
+      "type": "string"
+    },
+    "backends": {
+      "description": "Further backends, read together with the data folder: S3, another server, a device lab",
+      "default": [],
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "description": "Lowercase letters, digits and hyphens. Names where a result came from",
+            "type": "string",
+            "pattern": "^[a-z0-9-]+$"
+          },
+          "label": {
+            "type": "string"
+          },
+          "url": {
+            "description": "A folder, s3://bucket/prefix, r2://, gs://, or another Atlas server (https://…)",
+            "type": "string",
+            "minLength": 1
+          },
+          "write": {
+            "description": "Uploads go here instead of the data folder. At most one backend is the write target",
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "id",
+          "url"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "write": {
+      "description": "The id of the backend uploads go to. Default: the data folder",
+      "type": "string"
+    },
+    "auth": {
+      "default": {
+        "tokens": [],
+        "anonymousRead": false,
+        "insecure": false
+      },
+      "type": "object",
+      "properties": {
+        "tokens": {
+          "default": [],
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "description": "Shown in logs, never the token itself",
+                "type": "string",
+                "minLength": 1
+              },
+              "role": {
+                "description": "`read` may look. `publish` may also upload. `admin` may also remove scratch objects",
+                "type": "string",
+                "enum": [
+                  "read",
+                  "publish",
+                  "admin"
+                ]
+              },
+              "token": {
+                "description": "The token. Prefer `tokenEnv`, so the file holds no secret",
+                "type": "string",
+                "minLength": 12
+              },
+              "tokenEnv": {
+                "description": "The name of the environment variable that holds the token",
+                "type": "string"
+              }
+            },
+            "required": [
+              "name",
+              "role"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "anonymousRead": {
+          "description": "Let anyone look without a token. Uploading still needs one",
+          "default": false,
+          "type": "boolean"
+        },
+        "insecure": {
+          "description": "Allow listening on every interface with no tokens at all",
+          "default": false,
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
+    "site": {
+      "description": "The built viewer. Default: the one shipped with Atlas, built on first start",
+      "type": "string"
+    },
+    "outlineBranch": {
+      "description": "Which branch's specs the report is laid over. Default main, then master, then the newest",
+      "type": "string"
+    },
+    "include": {
+      "default": {},
+      "type": "object",
+      "properties": {
+        "branches": {
+          "description": "Only layers recorded on these branches take part",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "ignoreModified": {
+          "description": "Leave out layers from modified checkouts and unknown origins",
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
+    "refreshSeconds": {
+      "description": "How often the backends are looked at for layers written by something other than this server",
+      "default": 10,
+      "type": "number",
+      "minimum": 1
+    },
+    "limits": {
+      "default": {
+        "maxObjectMiB": 256,
+        "maxJsonMiB": 32,
+        "maxStreams": 500
+      },
+      "type": "object",
+      "properties": {
+        "maxObjectMiB": {
+          "description": "The largest object an upload may be",
+          "default": 256,
+          "type": "number",
+          "exclusiveMinimum": 0
+        },
+        "maxJsonMiB": {
+          "description": "The largest JSON body (a manifest, a checks request)",
+          "default": 32,
+          "type": "number",
+          "exclusiveMinimum": 0
+        },
+        "maxStreams": {
+          "description": "Viewers that may be connected for live updates at once",
+          "default": 500,
+          "type": "integer",
+          "exclusiveMinimum": 0,
+          "maximum": 9007199254740991
+        }
+      },
+      "additionalProperties": false
+    },
+    "heartbeatSeconds": {
+      "description": "How often a live connection is touched, so proxies keep it open",
+      "default": 15,
+      "type": "number",
+      "minimum": 1
+    },
+    "log": {
+      "description": "One JSON line per request",
+      "default": true,
+      "type": "boolean"
+    }
+  },
+  "additionalProperties": false
+}

package/packages/schema/schemas/server-info.schema.json

@@ -0,0 +1,114 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/server-info.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string",
+      "const": "atlas"
+    },
+    "version": {
+      "description": "The @crvouga/atlas version the server runs, or null when it does not know",
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "role": {
+      "description": "What the token of this request may do. A server with no tokens treats everyone as admin",
+      "type": "string",
+      "enum": [
+        "read",
+        "publish",
+        "admin"
+      ]
+    },
+    "open": {
+      "description": "True when the server has no tokens at all",
+      "type": "boolean"
+    },
+    "write": {
+      "description": "The id of the backend uploads go to",
+      "type": "string"
+    },
+    "ready": {
+      "type": "boolean"
+    },
+    "viewers": {
+      "description": "Viewers connected to /api/events right now",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "time": {
+      "description": "The server clock, epoch milliseconds",
+      "type": "number"
+    },
+    "backends": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string"
+          },
+          "label": {
+            "type": "string"
+          },
+          "available": {
+            "type": "boolean"
+          },
+          "write": {
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "id",
+          "label",
+          "available",
+          "write"
+        ],
+        "additionalProperties": {}
+      }
+    },
+    "layers": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "slots": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "capabilities": {
+      "description": "Features this server has, by name: `events`, `ingest`, `store`, `checks`, `status`, `effective`. A client checks a name before using what it names",
+      "type": "object",
+      "propertyNames": {
+        "type": "string"
+      },
+      "additionalProperties": {
+        "type": "boolean"
+      }
+    }
+  },
+  "required": [
+    "name",
+    "version",
+    "role",
+    "open",
+    "write",
+    "ready",
+    "viewers",
+    "time",
+    "backends",
+    "layers",
+    "slots",
+    "capabilities"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/spec-set.schema.json

@@ -0,0 +1,103 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/spec-set.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "number",
+      "const": 1
+    },
+    "id": {
+      "description": "SHA-256 of the branch, commit, freshness and the digest of every file: the same specs from the same commit are the same set",
+      "type": "string",
+      "pattern": "^[0-9a-f]{64}$"
+    },
+    "branch": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "sha": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "recordedAtMs": {
+      "description": "For a clean checkout, the commit time; for a modified one, when it was published",
+      "type": "number",
+      "minimum": 0
+    },
+    "freshness": {
+      "type": "string",
+      "enum": [
+        "unknown",
+        "modified"
+      ]
+    },
+    "files": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "path": {
+            "description": "`specs/<path inside the specs directory>`, the name a check uses in `specPath`",
+            "type": "string",
+            "minLength": 1
+          },
+          "digest": {
+            "description": "SHA-256 of the file. Its text is at layers/specs/blobs/<digest>.txt",
+            "type": "string",
+            "pattern": "^[0-9a-f]{64}$"
+          },
+          "fingerprint": {
+            "description": "The advisory fingerprint layers record, to warn when a spec changed since they ran",
+            "type": "string"
+          },
+          "kind": {
+            "type": "string"
+          },
+          "title": {
+            "type": "string"
+          },
+          "domain": {
+            "type": "string"
+          },
+          "bytes": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 9007199254740991
+          }
+        },
+        "required": [
+          "path",
+          "digest",
+          "fingerprint",
+          "kind",
+          "title",
+          "domain",
+          "bytes"
+        ],
+        "additionalProperties": {}
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "id",
+    "branch",
+    "sha",
+    "recordedAtMs",
+    "files"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/store-listing.schema.json

@@ -0,0 +1,29 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/store-listing.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "keys": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "next": {
+      "description": "Pass as `?after=` for the next page; null when there are no more",
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    }
+  },
+  "required": [
+    "keys",
+    "next"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/store-write-result.schema.json

@@ -0,0 +1,28 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/store-write-result.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "ok": {
+      "type": "boolean",
+      "const": true
+    },
+    "key": {
+      "type": "string"
+    },
+    "kind": {
+      "description": "What the server recognised the key as: `layer`, `layer-catalog`, `blob`, `spec-body`, `spec-set`, `spec-catalog`, `run-manifest`, `run-catalog`, `run-file` or `scratch`",
+      "type": "string"
+    },
+    "created": {
+      "type": "boolean"
+    }
+  },
+  "required": [
+    "ok",
+    "key",
+    "kind",
+    "created"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/stream-result.schema.json

@@ -0,0 +1,59 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/stream-result.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "ok": {
+      "description": "False when any check could not be sent (the exit code is then 1)",
+      "type": "boolean"
+    },
+    "producer": {
+      "type": "string"
+    },
+    "environment": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "layers": {
+      "description": "Layers that arrived",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "checks": {
+      "description": "Checks that arrived",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "lost": {
+      "description": "Checks that never arrived",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "warnings": {
+      "description": "Input lines that were skipped, and sends that failed",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": [
+    "ok",
+    "producer",
+    "environment",
+    "layers",
+    "checks",
+    "lost",
+    "warnings"
+  ],
+  "additionalProperties": {}
+}

package/packages/server/README.md

@@ -0,0 +1,43 @@
+# `@crvouga/atlas/server`
+
+The Atlas server: one process per project that takes in results from every environment, keeps them, and shows
+everyone the combined report, live. A Bun API that serves the Vite-built viewer from the same host (and, under
+`atlas dev`, the viewer's dev server proxies `/api` to it, so development is one host too).
+
+Use it from the command line: `atlas serve`, `atlas dev` ([the guide](../../docs/server.md)). Use it from code when
+you want the server inside something else:
+
+```ts
+import { createAtlasServer } from '@crvouga/atlas/server';
+import { listen } from '@crvouga/atlas/server';
+import { directoryStore } from '@crvouga/atlas/reports/directory-store';
+
+const server = createAtlasServer({
+  backends: [{ id: 'data', store: directoryStore('./atlas-data') }],
+  write: 'data',                                  // where uploads go
+  tokens: [{ name: 'ci', role: 'publish', token: process.env.ATLAS_PUBLISH_TOKEN! }],
+  site: './atlas-site'                            // the built viewer; without it only the API is served
+});
+const running = await listen(server.fetch, { port: 8080, host: '127.0.0.1', maxBodyBytes: 256 * 1024 * 1024 });
+await server.ready;                               // the first pass over the backends is done
+```
+
+`server.fetch` is a plain `(Request) => Promise<Response>`, so it also mounts in Hono, Express (through an adapter),
+Cloudflare-style runtimes and tests. `server.index` is the live report (`ReportIndex`), `server.events` the stream
+to viewers, `server.close()` stops both.
+
+| Module | What |
+| --- | --- |
+| `app` | `createAtlasServer(options)`: the routes, the live report, compression and the page. |
+| `host` | `listen(handler, { port, host, maxBodyBytes })`: `Bun.serve`, or `node:http` where there is no Bun. |
+| `store-api`, `validate` | The upload protocol and what it refuses (content-addressed layers, dependency order, immutability). |
+| `events` | `EventHub`: server-sent events to every viewer, with a heartbeat and a backlog limit. |
+| `auth` | Roles (`read`, `publish`, `admin`), Bearer tokens, constant-time comparison. |
+| `config` | `loadServerConfig`: the file, then the environment, then the flags; every mistake said in words. |
+| `main`, `dev` | `atlas serve` and `atlas dev`. |
+
+The server describes itself: `GET /api/openapi.json` is an OpenAPI 3.1 document of every route (also `@crvouga/atlas/openapi.json`),
+`GET /llms.txt` a guide for an agent given only the address, and every error is an RFC 9457 problem document. See
+[Atlas for AI agents](../../docs/agents.md) and the [generated reference](../../docs/reference/server.md).
+
+`bun packages/server/bench/bench.ts [layers] [checks]` measures the server on a long history.