@crvouga/atlas 0.17.0 → 1.1.0

package/dist/schema/src/user-state.js

@@ -0,0 +1,54 @@
+/**
+ * @module User state schema
+ *
+ * The one JSON document that holds everything a person using Atlas has chosen, as opposed to what an operator deployed:
+ * the garbage collection policy and the saved server settings. The server keeps it in S3-compatible object storage
+ * (`server/state.json` under the configured prefix) and nowhere else: not in a file, a database or the browser. A
+ * document that does not exist yet means nothing has been chosen; a document that cannot be read is an error, never
+ * an empty state. Unknown keys are kept, so an older server does not drop what a newer one wrote.
+ *
+ * @see docs/server.md#user-state-and-garbage-collection
+ */
+import { z } from 'zod/v4';
+import { ServerEditableSettingsSchema } from './server-config.js';
+/**
+ * The garbage collection policy: which report layers the server may delete from S3, and when. Only layers that the
+ * report has loaded and that no longer decide any result (every check they carry has since been superseded by a newer
+ * layer) are ever candidates, and the newest `keepLatest` layers are always kept. With `enabled: false` nothing is deleted.
+ */
+export const GcPolicySchema = z.strictObject({
+    enabled: z.boolean().describe('Delete eligible layers on a schedule. `false` keeps the policy but deletes nothing'),
+    maxAgeDays: z.number().int().min(1).max(3650).describe('A superseded layer is eligible once it was recorded more than this many days ago'),
+    keepLatest: z.number().int().min(0).max(1_000_000).default(20).describe('Always keep this many of the newest layers, however old, so recent history stays'),
+    intervalHours: z.number().min(1).max(720).default(24).describe('How often the server applies the policy')
+});
+/** What one pass of garbage collection found and did, for one project. */
+export const GcRunSchema = z.looseObject({
+    project: z.string(),
+    dryRun: z.boolean(),
+    at: z.string().describe('When the pass finished, ISO 8601'),
+    candidates: z.number().int().min(0).describe('Layers the policy would delete'),
+    deletedLayers: z.number().int().min(0),
+    deletedBlobs: z.number().int().min(0),
+    keptLayers: z.number().int().min(0),
+    error: z.string().optional().describe('Why the pass stopped early, when it did')
+});
+/** The user state document stored in S3. Both parts are optional: absent means "never chosen". */
+export const UserStateSchema = z.looseObject({
+    schemaVersion: z.literal(1),
+    gc: GcPolicySchema.optional().describe('The garbage collection policy. Absent until someone saves one: then nothing is deleted'),
+    settings: ServerEditableSettingsSchema.optional().describe('Report and live-run settings saved from the settings page, applied at the next start')
+});
+/**
+ * `GET /api/settings/gc` and `PUT /api/settings/gc`: the policy as saved in S3 (`null` when none has been saved), the
+ * revision to send back in `If-Match`, and the result of the last pass this server ran.
+ */
+export const GcSnapshotSchema = z.looseObject({
+    revision: z.string().regex(/^[a-f0-9]{64}$/),
+    policy: GcPolicySchema.nullable(),
+    saved: z.boolean().describe('Whether a policy exists in S3. `false` means garbage collection is off: nothing is deleted'),
+    lastRuns: z.array(GcRunSchema).describe('The most recent pass over each project, newest first; empty before the first pass'),
+    stateKey: z.string().describe('Where the user state lives in the bucket, relative to the configured prefix')
+});
+/** `POST /api/settings/gc/run`: the passes that ran, one per project. */
+export const GcRunResultSchema = z.looseObject({ runs: z.array(GcRunSchema), policy: GcPolicySchema.nullable() });

package/dist/server/src/app.d.ts

@@ -13,7 +13,8 @@ import { type BackendInput } from '../../reports/src/compose.js';
 import type { LayerHeader } from '../../reports/src/evidence-types.js';
 import type { ObjectStore } from '../../reports/src/object-store.js';
 import { ReportIndex } from '../../reports/src/report-index.js';
-import { type ServerSettings } from './settings.js';
+import { type GcSettings, type ServerSettings } from './settings.js';
+import type { GcPolicy, GcRun } from '../../schema/src/user-state.js';
 import type { ResolvedToken } from './config.js';
 import { EventHub } from './events.js';
 /** What the server is built from: its `backends` (read together) and which one takes uploads (`write`), `tokens`, the built `site`, the report's filters (`include`, `outlineBranch`), `limits`, timings, `log`, and `now` for tests. */
@@ -44,8 +45,12 @@ export type AtlasServerOptions = {
     readonly log?: (line: Record<string, unknown>) => void;
     readonly now?: () => number;
     readonly version?: string;
-    /** Admin configuration persistence, supplied by atlas serve. Changes apply on restart. */
+    /** Admin configuration persistence, supplied by atlas serve; kept in S3. Changes apply on restart. */
     readonly settings?: ServerSettings;
+    /** The garbage collection policy, kept in S3 and read from there each time the server collects. Supplied by atlas serve. */
+    readonly gc?: GcSettings;
+    /** How often (ms) the server checks whether a garbage collection pass is due. Default 60 000; for tests. */
+    readonly gcCheckMs?: number;
     /** Extra runtime configuration written into the page. */
     readonly page?: Record<string, unknown>;
 };
@@ -61,6 +66,15 @@ export type AtlasServer = {
     }[];
     /** Resolves when the first pass over the backends is done. */
     readonly ready: Promise<void>;
+    /**
+     * Apply a garbage collection policy to this project's upload backend. Deletes only layers the report has read and
+     * that no longer decide any result, older than the policy allows and beyond its `keepLatest`. With `dryRun` it only
+     * counts. Never throws: a failure is the `error` of the returned run.
+     */
+    collectGarbage(policy: GcPolicy, options: {
+        readonly dryRun: boolean;
+        readonly project: string;
+    }): Promise<GcRun>;
     /** Stop refreshing and end every live connection. */
     close(): void;
 };

package/dist/server/src/app.js

@@ -35,6 +35,7 @@ import { LiveRunStartSchema, LiveRunUpdateSchema } from '../../schema/src/live-r
 import { LiveRuns, LiveRunError } from './live-runs.js';
 import { ConfigError } from './config.js';
 import { renderSettingsPage } from './settings.js';
+import { collectGarbage, planGarbage } from '../../reports/src/garbage.js';
 import { Refusal } from './validate.js';
 import { createAuth } from './auth.js';
 import { EventHub } from './events.js';
@@ -340,6 +341,50 @@ function createProjectApplication(options) {
             throw error;
         }
     });
+    // The garbage collection policy: its own document in the user state, saved in S3 with the same revision rules.
+    app.get('/api/settings/gc', async (c) => {
+        const verdict = auth.check(c.req.raw, 'admin');
+        if (!verdict.ok)
+            return unauthorized(verdict);
+        if (!options.gc)
+            return problemResponse(503, 'Garbage collection settings are managed by the embedding application.');
+        try {
+            return c.json(await options.gc.read(), 200, { 'cache-control': 'no-store' });
+        }
+        catch (error) {
+            if (error instanceof ConfigError)
+                return problemResponse(503, error.message);
+            throw error;
+        }
+    });
+    app.put('/api/settings/gc', async (c) => {
+        const verdict = auth.check(c.req.raw, 'admin');
+        if (!verdict.ok)
+            return unauthorized(verdict);
+        if (!options.gc)
+            return problemResponse(503, 'Garbage collection settings are managed by the embedding application.');
+        const revision = /^"([a-f0-9]{64})"$/.exec(c.req.header('if-match') ?? '')?.[1];
+        if (!revision)
+            return problemResponse(428, 'Load the policy first and send its revision in If-Match.');
+        try {
+            const bytes = await readLimited(c.req.raw, Math.min(limits.json, 16 * 1024));
+            let input;
+            try {
+                input = JSON.parse(new TextDecoder().decode(bytes));
+            }
+            catch {
+                return problemResponse(422, 'The body is not JSON.');
+            }
+            return c.json(await options.gc.save(input, revision), 200, { 'cache-control': 'no-store' });
+        }
+        catch (error) {
+            if (error instanceof ConfigError)
+                return problemResponse(error.message.startsWith('Settings changed') ? 409 : 422, error.message);
+            if (error instanceof Refusal)
+                return problemResponse(error.status, error.message);
+            throw error;
+        }
+    });
     // Live updates: told when the report changes, and when specs or runs do.
     app.get('/api/events', c => events.stream(c.req.raw, () => ({ version: index.snapshot().version, ready: index.ready, time: now(), heartbeatMs: options.heartbeatMs ?? 15_000 })));
     // The report as it stands: one request, only what wins. Everything else about a layer is history.
@@ -665,11 +710,28 @@ function createProjectApplication(options) {
         options.log?.({ at: new Date(now()).toISOString(), level: 'error', path: new URL(c.req.url).pathname, error: error instanceof Error ? error.message : String(error) });
         return c.json({ error: 'Something went wrong on the server. It has been logged.' }, 500);
     });
+    async function collect(policy, run) {
+        const base = { project: run.project, dryRun: run.dryRun };
+        try {
+            // Look at the store once more first, so "superseded" is judged on everything uploaded so far.
+            await index.refreshNow();
+            const plan = await planGarbage(writeBackend.store, { policy, nowMs: now(), eligible: index.superseded() });
+            const outcome = run.dryRun ? { deletedLayers: 0, deletedBlobs: 0 } : await collectGarbage(writeBackend.store, plan);
+            if (outcome.deletedLayers)
+                options.log?.({ at: new Date(now()).toISOString(), level: 'info', gc: run.project, deletedLayers: outcome.deletedLayers, deletedBlobs: outcome.deletedBlobs });
+            return { ...base, at: new Date(now()).toISOString(), candidates: plan.candidates.length, ...outcome, keptLayers: plan.kept };
+        }
+        catch (error) {
+            options.log?.({ at: new Date(now()).toISOString(), level: 'error', gc: run.project, error: String(error) });
+            return { ...base, at: new Date(now()).toISOString(), candidates: 0, deletedLayers: 0, deletedBlobs: 0, keptLayers: 0, error: error instanceof Error ? error.message : String(error) };
+        }
+    }
     return {
         fetch: async (request) => app.fetch(request),
         routes: app.routes.map(({ method, path }) => ({ method, path })),
         index,
         events,
+        collectGarbage: collect,
         ready: index.start(options.refreshMs ?? 10_000),
         close: () => { clearInterval(leaseTimer); stopListening(); index.stop(); events.close(); }
     };

package/dist/server/src/config.d.ts

@@ -1,15 +1,18 @@
 /**
  * @module Server configuration
  *
- * Loads how the server runs: `atlas.server.json`, then environment variables, then flags, each overriding the last, checked
- * against `ServerConfigSchema` and resolved (paths made absolute, backends named, tokens read). Every mistake is a
- * `ConfigError` that says what to change, and a server that would listen beyond this machine with no tokens refuses to
- * start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is generated from it and a test proves
- * each row.
+ * Loads how the server runs: the settings saved in S3 (the user state), then `atlas.server.json`, then environment
+ * variables, then flags, each overriding the last, checked against `ServerConfigSchema` and resolved (storage read from
+ * the required `ATLAS_S3_*` variables, tokens read). Storage is never optional: all reports and user state live in
+ * S3-compatible object storage, so missing or invalid storage variables are an `S3ConfigError` naming every variable.
+ * Every other mistake is a `ConfigError` that says what to change, and a server that would listen beyond this machine
+ * with no tokens refuses to start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is
+ * generated from it and a test proves each row.
  *
  * @see docs/server.md#configuration
  */
 import { type ServerConfig, type ServerRole } from '../../schema/src/server-config.js';
+import { type S3Config } from './storage.js';
 type Env = Readonly<Record<string, string | undefined>>;
 /** A token as the server holds it: the `name` it is shown under in logs, its `role`, and the secret `token`. */
 export type ResolvedToken = {
@@ -17,19 +20,10 @@ export type ResolvedToken = {
     readonly role: ServerRole;
     readonly token: string;
 };
-/** A backend as configured: its `id`, the `label` people see, and the `url` (a folder, `s3://…`, or another server) to open. */
-export type BackendSpec = {
-    readonly id: string;
-    readonly label: string;
-    readonly url: string;
-};
 /** The configuration the server runs with: the file, then the environment, then the flags, checked and resolved. */
-export type ResolvedServerConfig = Omit<ServerConfig, 'backends' | 'auth' | 'write'> & {
-    /** The folder that is the default backend, absolute. */
-    readonly data: string;
-    /** Every backend, the data folder first. All are read; uploads go to `writeId`. */
-    readonly backends: readonly BackendSpec[];
-    readonly writeId: string;
+export type ResolvedServerConfig = Omit<ServerConfig, 'auth'> & {
+    /** The S3-compatible storage that holds every report and all user state. Required; there is no other. */
+    readonly s3: S3Config;
     readonly tokens: readonly ResolvedToken[];
     readonly anonymousRead: boolean;
     readonly insecure: boolean;
@@ -71,13 +65,14 @@ export type ServerSetting = {
  */
 export declare const SERVER_SETTINGS: readonly ServerSetting[];
 /** The text `atlas serve --help` prints. Also the source of the `atlas serve` section of `docs/reference/cli.md`. */
-export declare const SERVER_USAGE = "atlas serve: the Atlas server\n\n  atlas serve [--config atlas.server.json] [--data <dir>] [--port 8080] [--host 127.0.0.1]\n              [--store <url|id=url>]... [--write <id>] [--site <dir>]\n              [--outline-branch <b>] [--branch <name>]... [--ignore-modified] [--refresh <seconds>]\n              [--anonymous-read] [--insecure] [--no-log]\n\n  One process serves the client and the API. Test results are uploaded to it from anywhere (laptops, CI) and it\n  shows everyone the combined report, live. It keeps its data in --data (default ./atlas-data) and can read more\n  backends (--store): S3, other servers, folders. Uploads go to the data folder unless --write names another.\n\n  Tokens (ATLAS_TOKEN admin, ATLAS_PUBLISH_TOKEN, ATLAS_READ_TOKEN, or \"auth.tokens\" in the config) are required\n  to listen beyond this machine unless --insecure. Every option has an environment variable (docs/deploying.md).\n";
+export declare const SERVER_USAGE = "atlas serve: the Atlas server\n\n  atlas serve [--config atlas.server.json] [--port 8080] [--host 127.0.0.1] [--site <dir>]\n              [--outline-branch <b>] [--branch <name>]... [--ignore-modified] [--refresh <seconds>]\n              [--anonymous-read] [--insecure] [--no-log]\n\n  One process serves the client and the API. Test results are uploaded to it from anywhere (laptops, CI) and it\n  shows everyone the combined report, live.\n\n  Storage is required. ALL reports and ALL user state (the garbage collection policy, saved settings) are kept in\n  S3-compatible object storage and nowhere else; there is no local-disk, in-memory or database fallback. Without\n  these environment variables the server prints every one of them with what is wrong, and exits with status 2:\n    ATLAS_S3_ENDPOINT  ATLAS_S3_BUCKET  ATLAS_S3_REGION  ATLAS_S3_ACCESS_KEY_ID  ATLAS_S3_SECRET_ACCESS_KEY\n  Optional: ATLAS_S3_PREFIX, ATLAS_S3_PATH_STYLE, ATLAS_S3_SESSION_TOKEN. See .env.example.\n\n  Tokens (ATLAS_TOKEN admin, ATLAS_PUBLISH_TOKEN, ATLAS_READ_TOKEN, or \"auth.tokens\" in the config) are required\n  to listen beyond this machine unless --insecure. Every option has an environment variable (docs/deploying.md).\n";
 /**
  * Read the file named by `--config`/`ATLAS_CONFIG`, or `atlas.server.json` in the folder if there is one, lay the
- * environment and then the flags over it, and check the result. Every problem is a {@link ConfigError} that says
- * what to change.
+ * environment and then the flags over it, and check the result. `saved` is the settings kept in S3 (the user state),
+ * the lowest layer. Every problem is a {@link ConfigError} that says what to change; missing or invalid storage
+ * variables are an `S3ConfigError` that names all of them, and are checked first.
  */
-export declare function loadServerConfig(argv: readonly string[], env: Env, cwd: string, read?: (file: string) => string): ResolvedServerConfig;
+export declare function loadServerConfig(argv: readonly string[], env: Env, cwd: string, read?: (file: string) => string, saved?: Record<string, unknown>): ResolvedServerConfig;
 /** A server that anyone on the network can upload to must say who may. */
 export declare function assertSafeToListen(config: Pick<ResolvedServerConfig, 'host' | 'tokens' | 'insecure'>): void;
 export {};

package/dist/server/src/config.js

@@ -1,11 +1,13 @@
 /**
  * @module Server configuration
  *
- * Loads how the server runs: `atlas.server.json`, then environment variables, then flags, each overriding the last, checked
- * against `ServerConfigSchema` and resolved (paths made absolute, backends named, tokens read). Every mistake is a
- * `ConfigError` that says what to change, and a server that would listen beyond this machine with no tokens refuses to
- * start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is generated from it and a test proves
- * each row.
+ * Loads how the server runs: the settings saved in S3 (the user state), then `atlas.server.json`, then environment
+ * variables, then flags, each overriding the last, checked against `ServerConfigSchema` and resolved (storage read from
+ * the required `ATLAS_S3_*` variables, tokens read). Storage is never optional: all reports and user state live in
+ * S3-compatible object storage, so missing or invalid storage variables are an `S3ConfigError` naming every variable.
+ * Every other mistake is a `ConfigError` that says what to change, and a server that would listen beyond this machine
+ * with no tokens refuses to start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is
+ * generated from it and a test proves each row.
  *
  * @see docs/server.md#configuration
  */
@@ -13,13 +15,14 @@ import { existsSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import { parseArgs } from 'node:util';
 import { ServerConfigSchema } from '../../schema/src/server-config.js';
+import { resolveS3Config, S3_VARIABLES } from './storage.js';
 /** A configuration that cannot be used. The message says what to change; `atlas serve` prints it and exits 2. */
 export class ConfigError extends Error {
 }
 const DEFAULT_FILE = 'atlas.server.json';
 const MIN_TOKEN = 12;
 const OPTIONS = {
-    config: { type: 'string' }, data: { type: 'string' }, store: { type: 'string', multiple: true }, write: { type: 'string' },
+    config: { type: 'string' },
     host: { type: 'string' }, port: { type: 'string' }, site: { type: 'string' }, 'outline-branch': { type: 'string' },
     branch: { type: 'string', multiple: true }, 'ignore-modified': { type: 'boolean' }, refresh: { type: 'string' },
     insecure: { type: 'boolean' }, 'no-log': { type: 'boolean' }, 'anonymous-read': { type: 'boolean' }
@@ -33,11 +36,23 @@ export const SERVER_FLAGS = Object.keys(OPTIONS).map(name => `--${name}`);
  * table; one documented here that the loader ignores fails the test.
  */
 export const SERVER_SETTINGS = [
+    // Storage is required and comes only from the environment: reports and user state live in S3 and nowhere else.
+    ...(Object.entries({
+        ATLAS_S3_ENDPOINT: ['endpoint', 'S3 endpoint URL', c => c.s3.endpoint, 'https://s3.example.com'],
+        ATLAS_S3_BUCKET: ['bucket', 'Bucket for all reports and user state', c => c.s3.bucket, 'atlas-reports'],
+        ATLAS_S3_REGION: ['region', 'Bucket region', c => c.s3.region, 'eu-west-1'],
+        ATLAS_S3_ACCESS_KEY_ID: ['accessKeyId', 'Access key id', c => c.s3.accessKeyId, 'AKIAEXAMPLE'],
+        ATLAS_S3_SECRET_ACCESS_KEY: ['secretAccessKey', 'Secret access key', c => c.s3.secretAccessKey, 'secret-example-key'],
+        ATLAS_S3_PREFIX: ['prefix', 'Folder in the bucket for everything Atlas writes', c => c.s3.prefix, 'atlas'],
+        ATLAS_S3_PATH_STYLE: ['pathStyle', 'Path-style bucket addressing (`0` for virtual-hosted)', c => c.s3.pathStyle, '0'],
+        ATLAS_S3_SESSION_TOKEN: ['sessionToken', 'Session token for temporary credentials', c => c.s3.sessionToken, 'session-example-token']
+    }).map(([variable, [field, summary, read, value]]) => {
+        const required = S3_VARIABLES.find(item => item.name === variable).required;
+        const expected = field === 'pathStyle' ? false : value;
+        return { summary: `${summary} (${required ? '**required**' : 'optional'})`, key: `storage.s3.${field}`, flag: null, env: [variable], default: required ? '**none: the server refuses to start**' : field === 'pathStyle' ? 'yes' : field === 'prefix' ? 'the bucket root' : 'none', example: { value, read, expected } };
+    })),
     { summary: 'Address to listen on', key: 'host', flag: '--host <address>', env: ['ATLAS_HOST', 'HOST'], default: '`127.0.0.1`', example: { value: '0.0.0.0', read: c => c.host, expected: '0.0.0.0' } },
     { summary: 'Port (`0` picks a free one)', key: 'port', flag: '--port <n>', env: ['ATLAS_PORT', 'PORT'], default: '`8080`', example: { value: '9191', read: c => c.port, expected: 9191 } },
-    { summary: 'The server\'s own folder: the default backend and write target', key: 'data', flag: '--data <dir>', env: ['ATLAS_DATA'], default: '`./atlas-data`', example: { value: 'results', read: c => path.basename(c.data), expected: 'results' } },
-    { summary: 'More backends, read together with the data folder', key: 'backends', flag: '--store <[id=]url>` (repeatable)', env: ['ATLAS_STORE'], default: 'none', example: { value: 'lab=https://lab.example.com', read: c => c.backends.map(backend => backend.id), expected: ['data', 'lab'] } },
-    { summary: 'The backend uploads go to', key: 'write', flag: '--write <id>', env: ['ATLAS_WRITE'], default: '`data`', example: { value: 'data', read: c => c.writeId, expected: 'data' } },
     { summary: 'The built client', key: 'site', flag: '--site <dir>', env: ['ATLAS_SITE'], default: '`./atlas-site`, else built once', example: { value: 'client', read: c => path.basename(c.site ?? ''), expected: 'client' } },
     { summary: 'Whose specs the report is laid over', key: 'outlineBranch', flag: '--outline-branch <branch>', env: ['ATLAS_OUTLINE_BRANCH'], default: '`main`, then `master`, then the newest', example: { value: 'trunk', read: c => c.outlineBranch, expected: 'trunk' } },
     { summary: 'Only layers recorded on these branches take part', key: 'include.branches', flag: '--branch <name>` (repeatable)', env: ['ATLAS_ONLY_BRANCH'], default: 'all', example: { value: 'main', read: c => c.include.branches, expected: ['main'] } },
@@ -58,14 +73,18 @@ export const SERVER_SETTINGS = [
 /** The text `atlas serve --help` prints. Also the source of the `atlas serve` section of `docs/reference/cli.md`. */
 export const SERVER_USAGE = `atlas serve: the Atlas server
 
-  atlas serve [--config atlas.server.json] [--data <dir>] [--port 8080] [--host 127.0.0.1]
-              [--store <url|id=url>]... [--write <id>] [--site <dir>]
+  atlas serve [--config atlas.server.json] [--port 8080] [--host 127.0.0.1] [--site <dir>]
               [--outline-branch <b>] [--branch <name>]... [--ignore-modified] [--refresh <seconds>]
               [--anonymous-read] [--insecure] [--no-log]
 
   One process serves the client and the API. Test results are uploaded to it from anywhere (laptops, CI) and it
-  shows everyone the combined report, live. It keeps its data in --data (default ./atlas-data) and can read more
-  backends (--store): S3, other servers, folders. Uploads go to the data folder unless --write names another.
+  shows everyone the combined report, live.
+
+  Storage is required. ALL reports and ALL user state (the garbage collection policy, saved settings) are kept in
+  S3-compatible object storage and nowhere else; there is no local-disk, in-memory or database fallback. Without
+  these environment variables the server prints every one of them with what is wrong, and exits with status 2:
+    ATLAS_S3_ENDPOINT  ATLAS_S3_BUCKET  ATLAS_S3_REGION  ATLAS_S3_ACCESS_KEY_ID  ATLAS_S3_SECRET_ACCESS_KEY
+  Optional: ATLAS_S3_PREFIX, ATLAS_S3_PATH_STYLE, ATLAS_S3_SESSION_TOKEN. See .env.example.
 
   Tokens (ATLAS_TOKEN admin, ATLAS_PUBLISH_TOKEN, ATLAS_READ_TOKEN, or "auth.tokens" in the config) are required
   to listen beyond this machine unless --insecure. Every option has an environment variable (docs/deploying.md).
@@ -80,24 +99,16 @@ function parseFlags(argv) {
 }
 const list = (text) => (text ?? '').split(/[\n,]+/).map(item => item.trim()).filter(Boolean);
 const truthy = (value) => value === '1' || value === 'true';
-/** `id=url`, or a bare url with an id made from it. */
-function backendOf(spec, taken) {
-    const named = /^([a-z0-9-]+)=(.+)$/.exec(spec);
-    if (named)
-        return { id: named[1], url: named[2] };
-    const base = spec.replace(/^[a-z0-9+.-]+:\/\//i, '').replace(/[?#].*$/, '').replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase().slice(0, 40) || 'store';
-    let id = base;
-    for (let n = 2; taken.has(id); n++)
-        id = `${base}-${n}`;
-    return { id, url: spec };
-}
 /**
  * Read the file named by `--config`/`ATLAS_CONFIG`, or `atlas.server.json` in the folder if there is one, lay the
- * environment and then the flags over it, and check the result. Every problem is a {@link ConfigError} that says
- * what to change.
+ * environment and then the flags over it, and check the result. `saved` is the settings kept in S3 (the user state),
+ * the lowest layer. Every problem is a {@link ConfigError} that says what to change; missing or invalid storage
+ * variables are an `S3ConfigError` that names all of them, and are checked first.
  */
-export function loadServerConfig(argv, env, cwd, read = file => readFileSync(file, 'utf8')) {
+export function loadServerConfig(argv, env, cwd, read = file => readFileSync(file, 'utf8'), saved = {}) {
     const flags = parseFlags(argv);
+    // Storage first: nothing else matters while there is nowhere to keep the reports. Throws S3ConfigError naming every variable.
+    const s3 = resolveS3Config(env);
     const named = flags.config ?? env.ATLAS_CONFIG;
     const file = named ? path.resolve(cwd, named) : existsSync(path.join(cwd, DEFAULT_FILE)) ? path.join(cwd, DEFAULT_FILE) : null;
     let raw = {};
@@ -109,18 +120,21 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
             throw new ConfigError(`Cannot read ${file}: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
+    for (const removed of ['data', 'backends', 'write']) {
+        if (removed in raw)
+            throw new ConfigError(`${file}: "${removed}" is no longer supported. Reports and user state are kept only in S3-compatible storage, set with the required ATLAS_S3_* environment variables (see .env.example). Remove "${removed}" from the file.`);
+    }
     const base = file ? path.dirname(file) : cwd;
+    // The settings saved in S3 are the lowest layer: the file, the environment and the flags all override them.
     const auth = { ...raw.auth };
-    const include = { ...raw.include };
-    const merged = { ...raw, auth, include };
+    const include = { ...saved.include, ...raw.include };
+    const merged = { ...saved, ...raw, auth, include };
     const set = (key, value) => { if (value !== undefined)
         merged[key] = value; };
     // ATLAS_HOST and ATLAS_PORT first: HOST is often the machine's own name, and PORT belongs to whatever else is running here.
     set('host', flags.host ?? env.ATLAS_HOST ?? env.HOST);
     const port = env.ATLAS_PORT ?? env.PORT;
     set('port', flags.port !== undefined ? Number(flags.port) : port !== undefined ? Number(port) : undefined);
-    set('data', flags.data ?? env.ATLAS_DATA);
-    set('write', flags.write ?? env.ATLAS_WRITE);
     set('site', flags.site ?? env.ATLAS_SITE);
     set('outlineBranch', flags['outline-branch'] ?? env.ATLAS_OUTLINE_BRANCH);
     set('refreshSeconds', flags.refresh !== undefined ? Number(flags.refresh) : env.ATLAS_REFRESH_SECONDS !== undefined ? Number(env.ATLAS_REFRESH_SECONDS) : undefined);
@@ -131,7 +145,7 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
     set('runTimeoutSeconds', env.ATLAS_RUN_TIMEOUT_SECONDS === undefined ? undefined : Number(env.ATLAS_RUN_TIMEOUT_SECONDS));
     set('runStallSeconds', env.ATLAS_RUN_STALL_SECONDS === undefined ? undefined : Number(env.ATLAS_RUN_STALL_SECONDS));
     set('heartbeatSeconds', env.ATLAS_HEARTBEAT_SECONDS === undefined ? undefined : Number(env.ATLAS_HEARTBEAT_SECONDS));
-    const limits = { ...raw.limits };
+    const limits = { ...saved.limits, ...raw.limits };
     for (const [key, variable] of [['maxObjectMiB', 'ATLAS_MAX_OBJECT_MIB'], ['maxJsonMiB', 'ATLAS_MAX_JSON_MIB'], ['maxStreams', 'ATLAS_MAX_STREAMS']]) {
         if (env[variable] !== undefined)
             limits[key] = Number(env[variable]);
@@ -150,29 +164,6 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
     if (!parsed.success)
         throw new ConfigError(`${file ? `${file}: ` : 'Configuration: '}${parsed.error.issues.map(issue => `${issue.path.join('.') || '(config)'}: ${issue.message}`).join('; ')}`);
     const config = parsed.data;
-    // Backends: the data folder, then those in the file, then --store and ATLAS_STORE.
-    const taken = new Set(['data']);
-    const backends = [{ id: 'data', label: 'Server data', url: path.resolve(base, config.data) }];
-    const extra = config.backends.map(item => ({ id: item.id, label: item.label ?? item.id, url: item.url, write: item.write === true }));
-    for (const item of extra) {
-        if (taken.has(item.id))
-            throw new ConfigError(`Two backends are called "${item.id}" ("data" is the data folder)`);
-        taken.add(item.id);
-        backends.push({ id: item.id, label: item.label, url: item.url });
-    }
-    for (const spec of [...(flags.store ?? []), ...(flags.store ? [] : list(env.ATLAS_STORE))]) {
-        const item = backendOf(spec, taken);
-        if (taken.has(item.id))
-            throw new ConfigError(`Two backends are called "${item.id}"`);
-        taken.add(item.id);
-        backends.push({ id: item.id, label: item.id, url: item.url });
-    }
-    const marked = extra.filter(item => item.write).map(item => item.id);
-    if (marked.length > 1)
-        throw new ConfigError(`Only one backend can take uploads; ${marked.join(', ')} are all marked write`);
-    const writeId = config.write ?? marked[0] ?? 'data';
-    if (!backends.some(item => item.id === writeId))
-        throw new ConfigError(`write names "${writeId}", which is not a backend (${backends.map(item => item.id).join(', ')})`);
     // Tokens: the file's, then the environment's.
     const tokens = [];
     for (const item of config.auth.tokens) {
@@ -190,8 +181,8 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
             throw new ConfigError(`Token "${item.name}" is shorter than ${MIN_TOKEN} characters: use a long random one (openssl rand -hex 24)`);
     if (new Set(tokens.map(item => item.token)).size !== tokens.length)
         throw new ConfigError('Two tokens are the same: each token must say who it is');
-    const { backends: _backends, auth: _auth, write: _write, ...rest } = config;
-    const resolved = { ...rest, data: backends[0].url, backends, writeId, tokens, anonymousRead: config.auth.anonymousRead, insecure: config.auth.insecure, configFile: file, ...(config.site ? { site: path.resolve(base, config.site) } : {}) };
+    const { auth: _auth, ...rest } = config;
+    const resolved = { ...rest, s3, tokens, anonymousRead: config.auth.anonymousRead, insecure: config.auth.insecure, configFile: file, ...(config.site ? { site: path.resolve(base, config.site) } : {}) };
     assertSafeToListen(resolved);
     return resolved;
 }

package/dist/server/src/dev.d.ts

@@ -7,6 +7,6 @@
  */
 import { type ServeIo } from './main.js';
 /** The text `atlas dev --help` prints. Also the source of the `atlas dev` section of `docs/reference/cli.md`. */
-export declare const DEV_USAGE = "atlas dev: the Atlas server and the client, for working on either\n\n  atlas dev [--port 5180] [--host 127.0.0.1] [--data <dir>] [--store <url|id=url>]... [--config atlas.server.json]\n\n  Starts the server (API only, on a port of its own) and the client's dev server, which proxies /api to it, so the\n  browser talks to one host and a change to the client shows at once. In production `atlas serve` does both from\n  one process.\n";
+export declare const DEV_USAGE = "atlas dev: the Atlas server and the client, for working on either\n\n  atlas dev [--port 5180] [--host 127.0.0.1] [--config atlas.server.json]\n\n  Starts the server (API only, on a port of its own) and the client's dev server, which proxies /api to it, so the\n  browser talks to one host and a change to the client shows at once. In production `atlas serve` does both from\n  one process. Storage is the same as for `atlas serve`: the required ATLAS_S3_* environment variables name the S3\n  bucket that holds all reports and user state; without them it exits with status 2 and lists every variable.\n";
 /** `atlas dev`. */
 export declare function devMain(argv: readonly string[], io: ServeIo): Promise<number>;

package/dist/server/src/dev.js

@@ -6,20 +6,23 @@
  * uploaders and the client all use one host, exactly as in production, and a change to the client shows at once.
  */
 import path from 'node:path';
-import { openConfiguredBackends, serverOptionsOf } from './main.js';
+import { connectStorage, serverOptionsOf } from './main.js';
 import { createAtlasServer } from './app.js';
 import { ConfigError, loadServerConfig, SERVER_USAGE } from './config.js';
+import { describeS3, S3ConfigError } from './storage.js';
+import { userStateStore } from './user-state.js';
 import { listen } from './host.js';
-import { fileServerSettings } from './settings.js';
+import { storedSettings } from './settings.js';
 import { visualizerRoot } from './site.js';
 /** The text `atlas dev --help` prints. Also the source of the `atlas dev` section of `docs/reference/cli.md`. */
 export const DEV_USAGE = `atlas dev: the Atlas server and the client, for working on either
 
-  atlas dev [--port 5180] [--host 127.0.0.1] [--data <dir>] [--store <url|id=url>]... [--config atlas.server.json]
+  atlas dev [--port 5180] [--host 127.0.0.1] [--config atlas.server.json]
 
   Starts the server (API only, on a port of its own) and the client's dev server, which proxies /api to it, so the
   browser talks to one host and a change to the client shows at once. In production \`atlas serve\` does both from
-  one process.
+  one process. Storage is the same as for \`atlas serve\`: the required ATLAS_S3_* environment variables name the S3
+  bucket that holds all reports and user state; without them it exits with status 2 and lists every variable.
 `;
 /** `atlas dev`. */
 export async function devMain(argv, io) {
@@ -44,14 +47,24 @@ export async function devMain(argv, io) {
         config = loadServerConfig([...forServer, '--port', '0'], { ...env, PORT: undefined, ATLAS_PORT: undefined }, cwd);
     }
     catch (error) {
+        if (error instanceof S3ConfigError) {
+            io.out(`${error.message}\n`);
+            return 2;
+        }
         if (error instanceof ConfigError) {
             io.out(`atlas dev: ${error.message}\n${SERVER_USAGE.split('\n')[0]}\n`);
             return 2;
         }
         throw error;
     }
-    const backends = openConfiguredBackends(config, env, cwd);
-    const api = createAtlasServer({ ...serverOptionsOf(config, backends, line => io.out(`${JSON.stringify(line)}\n`)), settings: fileServerSettings(config, forServer, env, cwd) });
+    const connected = await connectStorage(config, env, io);
+    if ('failure' in connected) {
+        io.out(`${connected.failure}\n`);
+        return 2;
+    }
+    const { backends } = connected;
+    const { settings, gc } = storedSettings(config, forServer, env, cwd, userStateStore(backends[0].store));
+    const api = createAtlasServer({ ...serverOptionsOf(config, backends, line => io.out(`${JSON.stringify(line)}\n`)), settings, gc });
     const running = await listen(api.fetch, { port: 0, host: '127.0.0.1', maxBodyBytes: Math.max(config.limits.maxObjectMiB, config.limits.maxJsonMiB) * 1024 * 1024 });
     await api.ready;
     // The client's config reads this to proxy /api, and to write the same page config a server would.
@@ -74,7 +87,7 @@ export async function devMain(argv, io) {
         throw error;
     }
     vite.printUrls();
-    io.out(`Data in ${path.relative(cwd, config.data) || config.data}. Publish with: ATLAS_STORE=${vite.resolvedUrls?.local[0] ?? `http://localhost:${publicPort}`} atlas report publish …\n`);
+    io.out(`Reports are stored in ${describeS3(config.s3)}. Publish with: ATLAS_STORE=${vite.resolvedUrls?.local[0] ?? `http://localhost:${publicPort}`} atlas report publish …\n`);
     await new Promise(resolve => {
         io.signal?.addEventListener('abort', () => resolve(), { once: true });
         if (!io.signal) {

package/dist/server/src/index.d.ts

@@ -4,15 +4,19 @@
  * Everything needed to run the Atlas server from code: `createAtlasServer` (a `fetch` handler with the live report, uploads
  * and the viewer), `listen` (serve it over HTTP on Bun or Node), `loadServerConfig` (the file, the environment and the flags
  * as one checked configuration), `serveMain` (what `atlas serve` runs), `EventHub`, `createAuth`, and the upload checks.
+ * `atlas serve` keeps everything in S3: `resolveS3Config` reads the required `ATLAS_S3_*` variables, `userStateStore` the user
+ * state in the bucket, and `storedSettings` serves the settings page from it.
  *
  * @see docs/server.md
  */
 export { createAtlasServer, type AtlasServer, type AtlasServerOptions } from './app.js';
 export { createAuth, type Auth, type Identity } from './auth.js';
 export { assertSafeToListen, ConfigError, loadServerConfig, SERVER_USAGE, type ResolvedServerConfig, type ResolvedToken } from './config.js';
+export { resolveS3Config, S3_VARIABLES, S3ConfigError, startupFailure, type S3Config } from './storage.js';
+export { userStateStore, USER_STATE_KEY, UserStateError, type UserStateStore } from './user-state.js';
 export { EventHub, type EventName } from './events.js';
 export { listen, type RunningServer } from './host.js';
 export { serveMain } from './main.js';
 export { classify, validateUpload, validKey, Refusal } from './validate.js';
-export { fileServerSettings, type ServerSettings, type SettingsSnapshot } from './settings.js';
+export { storedSettings, type GcSettings, type ServerSettings, type SettingsSnapshot } from './settings.js';
 export { LiveRuns, LiveRunError } from './live-runs.js';

package/dist/server/src/index.js

@@ -4,15 +4,19 @@
  * Everything needed to run the Atlas server from code: `createAtlasServer` (a `fetch` handler with the live report, uploads
  * and the viewer), `listen` (serve it over HTTP on Bun or Node), `loadServerConfig` (the file, the environment and the flags
  * as one checked configuration), `serveMain` (what `atlas serve` runs), `EventHub`, `createAuth`, and the upload checks.
+ * `atlas serve` keeps everything in S3: `resolveS3Config` reads the required `ATLAS_S3_*` variables, `userStateStore` the user
+ * state in the bucket, and `storedSettings` serves the settings page from it.
  *
  * @see docs/server.md
  */
 export { createAtlasServer } from './app.js';
 export { createAuth } from './auth.js';
 export { assertSafeToListen, ConfigError, loadServerConfig, SERVER_USAGE } from './config.js';
+export { resolveS3Config, S3_VARIABLES, S3ConfigError, startupFailure } from './storage.js';
+export { userStateStore, USER_STATE_KEY, UserStateError } from './user-state.js';
 export { EventHub } from './events.js';
 export { listen } from './host.js';
 export { serveMain } from './main.js';
 export { classify, validateUpload, validKey, Refusal } from './validate.js';
-export { fileServerSettings } from './settings.js';
+export { storedSettings } from './settings.js';
 export { LiveRuns, LiveRunError } from './live-runs.js';