@crvouga/atlas 0.16.0 → 1.0.0

package/dist/server/src/config.d.ts

@@ -1,15 +1,18 @@
 /**
  * @module Server configuration
  *
- * Loads how the server runs: `atlas.server.json`, then environment variables, then flags, each overriding the last, checked
- * against `ServerConfigSchema` and resolved (paths made absolute, backends named, tokens read). Every mistake is a
- * `ConfigError` that says what to change, and a server that would listen beyond this machine with no tokens refuses to
- * start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is generated from it and a test proves
- * each row.
+ * Loads how the server runs: the settings saved in S3 (the user state), then `atlas.server.json`, then environment
+ * variables, then flags, each overriding the last, checked against `ServerConfigSchema` and resolved (storage read from
+ * the required `ATLAS_S3_*` variables, tokens read). Storage is never optional: all reports and user state live in
+ * S3-compatible object storage, so missing or invalid storage variables are an `S3ConfigError` naming every variable.
+ * Every other mistake is a `ConfigError` that says what to change, and a server that would listen beyond this machine
+ * with no tokens refuses to start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is
+ * generated from it and a test proves each row.
  *
  * @see docs/server.md#configuration
  */
 import { type ServerConfig, type ServerRole } from '../../schema/src/server-config.js';
+import { type S3Config } from './storage.js';
 type Env = Readonly<Record<string, string | undefined>>;
 /** A token as the server holds it: the `name` it is shown under in logs, its `role`, and the secret `token`. */
 export type ResolvedToken = {
@@ -17,19 +20,10 @@ export type ResolvedToken = {
     readonly role: ServerRole;
     readonly token: string;
 };
-/** A backend as configured: its `id`, the `label` people see, and the `url` (a folder, `s3://…`, or another server) to open. */
-export type BackendSpec = {
-    readonly id: string;
-    readonly label: string;
-    readonly url: string;
-};
 /** The configuration the server runs with: the file, then the environment, then the flags, checked and resolved. */
-export type ResolvedServerConfig = Omit<ServerConfig, 'backends' | 'auth' | 'write'> & {
-    /** The folder that is the default backend, absolute. */
-    readonly data: string;
-    /** Every backend, the data folder first. All are read; uploads go to `writeId`. */
-    readonly backends: readonly BackendSpec[];
-    readonly writeId: string;
+export type ResolvedServerConfig = Omit<ServerConfig, 'auth'> & {
+    /** The S3-compatible storage that holds every report and all user state. Required; there is no other. */
+    readonly s3: S3Config;
     readonly tokens: readonly ResolvedToken[];
     readonly anonymousRead: boolean;
     readonly insecure: boolean;
@@ -71,13 +65,14 @@ export type ServerSetting = {
  */
 export declare const SERVER_SETTINGS: readonly ServerSetting[];
 /** The text `atlas serve --help` prints. Also the source of the `atlas serve` section of `docs/reference/cli.md`. */
-export declare const SERVER_USAGE = "atlas serve: the Atlas server\n\n  atlas serve [--config atlas.server.json] [--data <dir>] [--port 8080] [--host 127.0.0.1]\n              [--store <url|id=url>]... [--write <id>] [--site <dir>]\n              [--outline-branch <b>] [--branch <name>]... [--ignore-modified] [--refresh <seconds>]\n              [--anonymous-read] [--insecure] [--no-log]\n\n  One process serves the client and the API. Test results are uploaded to it from anywhere (laptops, CI) and it\n  shows everyone the combined report, live. It keeps its data in --data (default ./atlas-data) and can read more\n  backends (--store): S3, other servers, folders. Uploads go to the data folder unless --write names another.\n\n  Tokens (ATLAS_TOKEN admin, ATLAS_PUBLISH_TOKEN, ATLAS_READ_TOKEN, or \"auth.tokens\" in the config) are required\n  to listen beyond this machine unless --insecure. Every option has an environment variable (docs/deploying.md).\n";
+export declare const SERVER_USAGE = "atlas serve: the Atlas server\n\n  atlas serve [--config atlas.server.json] [--port 8080] [--host 127.0.0.1] [--site <dir>]\n              [--outline-branch <b>] [--branch <name>]... [--ignore-modified] [--refresh <seconds>]\n              [--anonymous-read] [--insecure] [--no-log]\n\n  One process serves the client and the API. Test results are uploaded to it from anywhere (laptops, CI) and it\n  shows everyone the combined report, live.\n\n  Storage is required. ALL reports and ALL user state (the garbage collection policy, saved settings) are kept in\n  S3-compatible object storage and nowhere else; there is no local-disk, in-memory or database fallback. Without\n  these environment variables the server prints every one of them with what is wrong, and exits with status 2:\n    ATLAS_S3_ENDPOINT  ATLAS_S3_BUCKET  ATLAS_S3_REGION  ATLAS_S3_ACCESS_KEY_ID  ATLAS_S3_SECRET_ACCESS_KEY\n  Optional: ATLAS_S3_PREFIX, ATLAS_S3_PATH_STYLE, ATLAS_S3_SESSION_TOKEN. See .env.example.\n\n  Tokens (ATLAS_TOKEN admin, ATLAS_PUBLISH_TOKEN, ATLAS_READ_TOKEN, or \"auth.tokens\" in the config) are required\n  to listen beyond this machine unless --insecure. Every option has an environment variable (docs/deploying.md).\n";
 /**
  * Read the file named by `--config`/`ATLAS_CONFIG`, or `atlas.server.json` in the folder if there is one, lay the
- * environment and then the flags over it, and check the result. Every problem is a {@link ConfigError} that says
- * what to change.
+ * environment and then the flags over it, and check the result. `saved` is the settings kept in S3 (the user state),
+ * the lowest layer. Every problem is a {@link ConfigError} that says what to change; missing or invalid storage
+ * variables are an `S3ConfigError` that names all of them, and are checked first.
  */
-export declare function loadServerConfig(argv: readonly string[], env: Env, cwd: string, read?: (file: string) => string): ResolvedServerConfig;
+export declare function loadServerConfig(argv: readonly string[], env: Env, cwd: string, read?: (file: string) => string, saved?: Record<string, unknown>): ResolvedServerConfig;
 /** A server that anyone on the network can upload to must say who may. */
 export declare function assertSafeToListen(config: Pick<ResolvedServerConfig, 'host' | 'tokens' | 'insecure'>): void;
 export {};

package/dist/server/src/config.js

@@ -1,11 +1,13 @@
 /**
  * @module Server configuration
  *
- * Loads how the server runs: `atlas.server.json`, then environment variables, then flags, each overriding the last, checked
- * against `ServerConfigSchema` and resolved (paths made absolute, backends named, tokens read). Every mistake is a
- * `ConfigError` that says what to change, and a server that would listen beyond this machine with no tokens refuses to
- * start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is generated from it and a test proves
- * each row.
+ * Loads how the server runs: the settings saved in S3 (the user state), then `atlas.server.json`, then environment
+ * variables, then flags, each overriding the last, checked against `ServerConfigSchema` and resolved (storage read from
+ * the required `ATLAS_S3_*` variables, tokens read). Storage is never optional: all reports and user state live in
+ * S3-compatible object storage, so missing or invalid storage variables are an `S3ConfigError` naming every variable.
+ * Every other mistake is a `ConfigError` that says what to change, and a server that would listen beyond this machine
+ * with no tokens refuses to start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is
+ * generated from it and a test proves each row.
  *
  * @see docs/server.md#configuration
  */
@@ -13,13 +15,14 @@ import { existsSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import { parseArgs } from 'node:util';
 import { ServerConfigSchema } from '../../schema/src/server-config.js';
+import { resolveS3Config, S3_VARIABLES } from './storage.js';
 /** A configuration that cannot be used. The message says what to change; `atlas serve` prints it and exits 2. */
 export class ConfigError extends Error {
 }
 const DEFAULT_FILE = 'atlas.server.json';
 const MIN_TOKEN = 12;
 const OPTIONS = {
-    config: { type: 'string' }, data: { type: 'string' }, store: { type: 'string', multiple: true }, write: { type: 'string' },
+    config: { type: 'string' },
     host: { type: 'string' }, port: { type: 'string' }, site: { type: 'string' }, 'outline-branch': { type: 'string' },
     branch: { type: 'string', multiple: true }, 'ignore-modified': { type: 'boolean' }, refresh: { type: 'string' },
     insecure: { type: 'boolean' }, 'no-log': { type: 'boolean' }, 'anonymous-read': { type: 'boolean' }
@@ -33,11 +36,23 @@ export const SERVER_FLAGS = Object.keys(OPTIONS).map(name => `--${name}`);
  * table; one documented here that the loader ignores fails the test.
  */
 export const SERVER_SETTINGS = [
+    // Storage is required and comes only from the environment: reports and user state live in S3 and nowhere else.
+    ...(Object.entries({
+        ATLAS_S3_ENDPOINT: ['endpoint', 'S3 endpoint URL', c => c.s3.endpoint, 'https://s3.example.com'],
+        ATLAS_S3_BUCKET: ['bucket', 'Bucket for all reports and user state', c => c.s3.bucket, 'atlas-reports'],
+        ATLAS_S3_REGION: ['region', 'Bucket region', c => c.s3.region, 'eu-west-1'],
+        ATLAS_S3_ACCESS_KEY_ID: ['accessKeyId', 'Access key id', c => c.s3.accessKeyId, 'AKIAEXAMPLE'],
+        ATLAS_S3_SECRET_ACCESS_KEY: ['secretAccessKey', 'Secret access key', c => c.s3.secretAccessKey, 'secret-example-key'],
+        ATLAS_S3_PREFIX: ['prefix', 'Folder in the bucket for everything Atlas writes', c => c.s3.prefix, 'atlas'],
+        ATLAS_S3_PATH_STYLE: ['pathStyle', 'Path-style bucket addressing (`0` for virtual-hosted)', c => c.s3.pathStyle, '0'],
+        ATLAS_S3_SESSION_TOKEN: ['sessionToken', 'Session token for temporary credentials', c => c.s3.sessionToken, 'session-example-token']
+    }).map(([variable, [field, summary, read, value]]) => {
+        const required = S3_VARIABLES.find(item => item.name === variable).required;
+        const expected = field === 'pathStyle' ? false : value;
+        return { summary: `${summary} (${required ? '**required**' : 'optional'})`, key: `storage.s3.${field}`, flag: null, env: [variable], default: required ? '**none: the server refuses to start**' : field === 'pathStyle' ? 'yes' : field === 'prefix' ? 'the bucket root' : 'none', example: { value, read, expected } };
+    })),
     { summary: 'Address to listen on', key: 'host', flag: '--host <address>', env: ['ATLAS_HOST', 'HOST'], default: '`127.0.0.1`', example: { value: '0.0.0.0', read: c => c.host, expected: '0.0.0.0' } },
     { summary: 'Port (`0` picks a free one)', key: 'port', flag: '--port <n>', env: ['ATLAS_PORT', 'PORT'], default: '`8080`', example: { value: '9191', read: c => c.port, expected: 9191 } },
-    { summary: 'The server\'s own folder: the default backend and write target', key: 'data', flag: '--data <dir>', env: ['ATLAS_DATA'], default: '`./atlas-data`', example: { value: 'results', read: c => path.basename(c.data), expected: 'results' } },
-    { summary: 'More backends, read together with the data folder', key: 'backends', flag: '--store <[id=]url>` (repeatable)', env: ['ATLAS_STORE'], default: 'none', example: { value: 'lab=https://lab.example.com', read: c => c.backends.map(backend => backend.id), expected: ['data', 'lab'] } },
-    { summary: 'The backend uploads go to', key: 'write', flag: '--write <id>', env: ['ATLAS_WRITE'], default: '`data`', example: { value: 'data', read: c => c.writeId, expected: 'data' } },
     { summary: 'The built client', key: 'site', flag: '--site <dir>', env: ['ATLAS_SITE'], default: '`./atlas-site`, else built once', example: { value: 'client', read: c => path.basename(c.site ?? ''), expected: 'client' } },
     { summary: 'Whose specs the report is laid over', key: 'outlineBranch', flag: '--outline-branch <branch>', env: ['ATLAS_OUTLINE_BRANCH'], default: '`main`, then `master`, then the newest', example: { value: 'trunk', read: c => c.outlineBranch, expected: 'trunk' } },
     { summary: 'Only layers recorded on these branches take part', key: 'include.branches', flag: '--branch <name>` (repeatable)', env: ['ATLAS_ONLY_BRANCH'], default: 'all', example: { value: 'main', read: c => c.include.branches, expected: ['main'] } },
@@ -58,14 +73,18 @@ export const SERVER_SETTINGS = [
 /** The text `atlas serve --help` prints. Also the source of the `atlas serve` section of `docs/reference/cli.md`. */
 export const SERVER_USAGE = `atlas serve: the Atlas server
 
-  atlas serve [--config atlas.server.json] [--data <dir>] [--port 8080] [--host 127.0.0.1]
-              [--store <url|id=url>]... [--write <id>] [--site <dir>]
+  atlas serve [--config atlas.server.json] [--port 8080] [--host 127.0.0.1] [--site <dir>]
               [--outline-branch <b>] [--branch <name>]... [--ignore-modified] [--refresh <seconds>]
               [--anonymous-read] [--insecure] [--no-log]
 
   One process serves the client and the API. Test results are uploaded to it from anywhere (laptops, CI) and it
-  shows everyone the combined report, live. It keeps its data in --data (default ./atlas-data) and can read more
-  backends (--store): S3, other servers, folders. Uploads go to the data folder unless --write names another.
+  shows everyone the combined report, live.
+
+  Storage is required. ALL reports and ALL user state (the garbage collection policy, saved settings) are kept in
+  S3-compatible object storage and nowhere else; there is no local-disk, in-memory or database fallback. Without
+  these environment variables the server prints every one of them with what is wrong, and exits with status 2:
+    ATLAS_S3_ENDPOINT  ATLAS_S3_BUCKET  ATLAS_S3_REGION  ATLAS_S3_ACCESS_KEY_ID  ATLAS_S3_SECRET_ACCESS_KEY
+  Optional: ATLAS_S3_PREFIX, ATLAS_S3_PATH_STYLE, ATLAS_S3_SESSION_TOKEN. See .env.example.
 
   Tokens (ATLAS_TOKEN admin, ATLAS_PUBLISH_TOKEN, ATLAS_READ_TOKEN, or "auth.tokens" in the config) are required
   to listen beyond this machine unless --insecure. Every option has an environment variable (docs/deploying.md).
@@ -80,24 +99,16 @@ function parseFlags(argv) {
 }
 const list = (text) => (text ?? '').split(/[\n,]+/).map(item => item.trim()).filter(Boolean);
 const truthy = (value) => value === '1' || value === 'true';
-/** `id=url`, or a bare url with an id made from it. */
-function backendOf(spec, taken) {
-    const named = /^([a-z0-9-]+)=(.+)$/.exec(spec);
-    if (named)
-        return { id: named[1], url: named[2] };
-    const base = spec.replace(/^[a-z0-9+.-]+:\/\//i, '').replace(/[?#].*$/, '').replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase().slice(0, 40) || 'store';
-    let id = base;
-    for (let n = 2; taken.has(id); n++)
-        id = `${base}-${n}`;
-    return { id, url: spec };
-}
 /**
  * Read the file named by `--config`/`ATLAS_CONFIG`, or `atlas.server.json` in the folder if there is one, lay the
- * environment and then the flags over it, and check the result. Every problem is a {@link ConfigError} that says
- * what to change.
+ * environment and then the flags over it, and check the result. `saved` is the settings kept in S3 (the user state),
+ * the lowest layer. Every problem is a {@link ConfigError} that says what to change; missing or invalid storage
+ * variables are an `S3ConfigError` that names all of them, and are checked first.
  */
-export function loadServerConfig(argv, env, cwd, read = file => readFileSync(file, 'utf8')) {
+export function loadServerConfig(argv, env, cwd, read = file => readFileSync(file, 'utf8'), saved = {}) {
     const flags = parseFlags(argv);
+    // Storage first: nothing else matters while there is nowhere to keep the reports. Throws S3ConfigError naming every variable.
+    const s3 = resolveS3Config(env);
     const named = flags.config ?? env.ATLAS_CONFIG;
     const file = named ? path.resolve(cwd, named) : existsSync(path.join(cwd, DEFAULT_FILE)) ? path.join(cwd, DEFAULT_FILE) : null;
     let raw = {};
@@ -109,18 +120,21 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
             throw new ConfigError(`Cannot read ${file}: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
+    for (const removed of ['data', 'backends', 'write']) {
+        if (removed in raw)
+            throw new ConfigError(`${file}: "${removed}" is no longer supported. Reports and user state are kept only in S3-compatible storage, set with the required ATLAS_S3_* environment variables (see .env.example). Remove "${removed}" from the file.`);
+    }
     const base = file ? path.dirname(file) : cwd;
+    // The settings saved in S3 are the lowest layer: the file, the environment and the flags all override them.
     const auth = { ...raw.auth };
-    const include = { ...raw.include };
-    const merged = { ...raw, auth, include };
+    const include = { ...saved.include, ...raw.include };
+    const merged = { ...saved, ...raw, auth, include };
     const set = (key, value) => { if (value !== undefined)
         merged[key] = value; };
     // ATLAS_HOST and ATLAS_PORT first: HOST is often the machine's own name, and PORT belongs to whatever else is running here.
     set('host', flags.host ?? env.ATLAS_HOST ?? env.HOST);
     const port = env.ATLAS_PORT ?? env.PORT;
     set('port', flags.port !== undefined ? Number(flags.port) : port !== undefined ? Number(port) : undefined);
-    set('data', flags.data ?? env.ATLAS_DATA);
-    set('write', flags.write ?? env.ATLAS_WRITE);
     set('site', flags.site ?? env.ATLAS_SITE);
     set('outlineBranch', flags['outline-branch'] ?? env.ATLAS_OUTLINE_BRANCH);
     set('refreshSeconds', flags.refresh !== undefined ? Number(flags.refresh) : env.ATLAS_REFRESH_SECONDS !== undefined ? Number(env.ATLAS_REFRESH_SECONDS) : undefined);
@@ -131,7 +145,7 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
     set('runTimeoutSeconds', env.ATLAS_RUN_TIMEOUT_SECONDS === undefined ? undefined : Number(env.ATLAS_RUN_TIMEOUT_SECONDS));
     set('runStallSeconds', env.ATLAS_RUN_STALL_SECONDS === undefined ? undefined : Number(env.ATLAS_RUN_STALL_SECONDS));
     set('heartbeatSeconds', env.ATLAS_HEARTBEAT_SECONDS === undefined ? undefined : Number(env.ATLAS_HEARTBEAT_SECONDS));
-    const limits = { ...raw.limits };
+    const limits = { ...saved.limits, ...raw.limits };
     for (const [key, variable] of [['maxObjectMiB', 'ATLAS_MAX_OBJECT_MIB'], ['maxJsonMiB', 'ATLAS_MAX_JSON_MIB'], ['maxStreams', 'ATLAS_MAX_STREAMS']]) {
         if (env[variable] !== undefined)
             limits[key] = Number(env[variable]);
@@ -150,29 +164,6 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
     if (!parsed.success)
         throw new ConfigError(`${file ? `${file}: ` : 'Configuration: '}${parsed.error.issues.map(issue => `${issue.path.join('.') || '(config)'}: ${issue.message}`).join('; ')}`);
     const config = parsed.data;
-    // Backends: the data folder, then those in the file, then --store and ATLAS_STORE.
-    const taken = new Set(['data']);
-    const backends = [{ id: 'data', label: 'Server data', url: path.resolve(base, config.data) }];
-    const extra = config.backends.map(item => ({ id: item.id, label: item.label ?? item.id, url: item.url, write: item.write === true }));
-    for (const item of extra) {
-        if (taken.has(item.id))
-            throw new ConfigError(`Two backends are called "${item.id}" ("data" is the data folder)`);
-        taken.add(item.id);
-        backends.push({ id: item.id, label: item.label, url: item.url });
-    }
-    for (const spec of [...(flags.store ?? []), ...(flags.store ? [] : list(env.ATLAS_STORE))]) {
-        const item = backendOf(spec, taken);
-        if (taken.has(item.id))
-            throw new ConfigError(`Two backends are called "${item.id}"`);
-        taken.add(item.id);
-        backends.push({ id: item.id, label: item.id, url: item.url });
-    }
-    const marked = extra.filter(item => item.write).map(item => item.id);
-    if (marked.length > 1)
-        throw new ConfigError(`Only one backend can take uploads; ${marked.join(', ')} are all marked write`);
-    const writeId = config.write ?? marked[0] ?? 'data';
-    if (!backends.some(item => item.id === writeId))
-        throw new ConfigError(`write names "${writeId}", which is not a backend (${backends.map(item => item.id).join(', ')})`);
     // Tokens: the file's, then the environment's.
     const tokens = [];
     for (const item of config.auth.tokens) {
@@ -190,8 +181,8 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
             throw new ConfigError(`Token "${item.name}" is shorter than ${MIN_TOKEN} characters: use a long random one (openssl rand -hex 24)`);
     if (new Set(tokens.map(item => item.token)).size !== tokens.length)
         throw new ConfigError('Two tokens are the same: each token must say who it is');
-    const { backends: _backends, auth: _auth, write: _write, ...rest } = config;
-    const resolved = { ...rest, data: backends[0].url, backends, writeId, tokens, anonymousRead: config.auth.anonymousRead, insecure: config.auth.insecure, configFile: file, ...(config.site ? { site: path.resolve(base, config.site) } : {}) };
+    const { auth: _auth, ...rest } = config;
+    const resolved = { ...rest, s3, tokens, anonymousRead: config.auth.anonymousRead, insecure: config.auth.insecure, configFile: file, ...(config.site ? { site: path.resolve(base, config.site) } : {}) };
     assertSafeToListen(resolved);
     return resolved;
 }

package/dist/server/src/dev.d.ts

@@ -7,6 +7,6 @@
  */
 import { type ServeIo } from './main.js';
 /** The text `atlas dev --help` prints. Also the source of the `atlas dev` section of `docs/reference/cli.md`. */
-export declare const DEV_USAGE = "atlas dev: the Atlas server and the client, for working on either\n\n  atlas dev [--port 5180] [--host 127.0.0.1] [--data <dir>] [--store <url|id=url>]... [--config atlas.server.json]\n\n  Starts the server (API only, on a port of its own) and the client's dev server, which proxies /api to it, so the\n  browser talks to one host and a change to the client shows at once. In production `atlas serve` does both from\n  one process.\n";
+export declare const DEV_USAGE = "atlas dev: the Atlas server and the client, for working on either\n\n  atlas dev [--port 5180] [--host 127.0.0.1] [--config atlas.server.json]\n\n  Starts the server (API only, on a port of its own) and the client's dev server, which proxies /api to it, so the\n  browser talks to one host and a change to the client shows at once. In production `atlas serve` does both from\n  one process. Storage is the same as for `atlas serve`: the required ATLAS_S3_* environment variables name the S3\n  bucket that holds all reports and user state; without them it exits with status 2 and lists every variable.\n";
 /** `atlas dev`. */
 export declare function devMain(argv: readonly string[], io: ServeIo): Promise<number>;

package/dist/server/src/dev.js

@@ -6,20 +6,23 @@
  * uploaders and the client all use one host, exactly as in production, and a change to the client shows at once.
  */
 import path from 'node:path';
-import { openConfiguredBackends, serverOptionsOf } from './main.js';
+import { connectStorage, serverOptionsOf } from './main.js';
 import { createAtlasServer } from './app.js';
 import { ConfigError, loadServerConfig, SERVER_USAGE } from './config.js';
+import { describeS3, S3ConfigError } from './storage.js';
+import { userStateStore } from './user-state.js';
 import { listen } from './host.js';
-import { fileServerSettings } from './settings.js';
+import { storedSettings } from './settings.js';
 import { visualizerRoot } from './site.js';
 /** The text `atlas dev --help` prints. Also the source of the `atlas dev` section of `docs/reference/cli.md`. */
 export const DEV_USAGE = `atlas dev: the Atlas server and the client, for working on either
 
-  atlas dev [--port 5180] [--host 127.0.0.1] [--data <dir>] [--store <url|id=url>]... [--config atlas.server.json]
+  atlas dev [--port 5180] [--host 127.0.0.1] [--config atlas.server.json]
 
   Starts the server (API only, on a port of its own) and the client's dev server, which proxies /api to it, so the
   browser talks to one host and a change to the client shows at once. In production \`atlas serve\` does both from
-  one process.
+  one process. Storage is the same as for \`atlas serve\`: the required ATLAS_S3_* environment variables name the S3
+  bucket that holds all reports and user state; without them it exits with status 2 and lists every variable.
 `;
 /** `atlas dev`. */
 export async function devMain(argv, io) {
@@ -44,14 +47,24 @@ export async function devMain(argv, io) {
         config = loadServerConfig([...forServer, '--port', '0'], { ...env, PORT: undefined, ATLAS_PORT: undefined }, cwd);
     }
     catch (error) {
+        if (error instanceof S3ConfigError) {
+            io.out(`${error.message}\n`);
+            return 2;
+        }
         if (error instanceof ConfigError) {
             io.out(`atlas dev: ${error.message}\n${SERVER_USAGE.split('\n')[0]}\n`);
             return 2;
         }
         throw error;
     }
-    const backends = openConfiguredBackends(config, env, cwd);
-    const api = createAtlasServer({ ...serverOptionsOf(config, backends, line => io.out(`${JSON.stringify(line)}\n`)), settings: fileServerSettings(config, forServer, env, cwd) });
+    const connected = await connectStorage(config, env, io);
+    if ('failure' in connected) {
+        io.out(`${connected.failure}\n`);
+        return 2;
+    }
+    const { backends } = connected;
+    const { settings, gc } = storedSettings(config, forServer, env, cwd, userStateStore(backends[0].store));
+    const api = createAtlasServer({ ...serverOptionsOf(config, backends, line => io.out(`${JSON.stringify(line)}\n`)), settings, gc });
     const running = await listen(api.fetch, { port: 0, host: '127.0.0.1', maxBodyBytes: Math.max(config.limits.maxObjectMiB, config.limits.maxJsonMiB) * 1024 * 1024 });
     await api.ready;
     // The client's config reads this to proxy /api, and to write the same page config a server would.
@@ -74,7 +87,7 @@ export async function devMain(argv, io) {
         throw error;
     }
     vite.printUrls();
-    io.out(`Data in ${path.relative(cwd, config.data) || config.data}. Publish with: ATLAS_STORE=${vite.resolvedUrls?.local[0] ?? `http://localhost:${publicPort}`} atlas report publish …\n`);
+    io.out(`Reports are stored in ${describeS3(config.s3)}. Publish with: ATLAS_STORE=${vite.resolvedUrls?.local[0] ?? `http://localhost:${publicPort}`} atlas report publish …\n`);
     await new Promise(resolve => {
         io.signal?.addEventListener('abort', () => resolve(), { once: true });
         if (!io.signal) {

package/dist/server/src/index.d.ts

@@ -4,15 +4,19 @@
  * Everything needed to run the Atlas server from code: `createAtlasServer` (a `fetch` handler with the live report, uploads
  * and the viewer), `listen` (serve it over HTTP on Bun or Node), `loadServerConfig` (the file, the environment and the flags
  * as one checked configuration), `serveMain` (what `atlas serve` runs), `EventHub`, `createAuth`, and the upload checks.
+ * `atlas serve` keeps everything in S3: `resolveS3Config` reads the required `ATLAS_S3_*` variables, `userStateStore` the user
+ * state in the bucket, and `storedSettings` serves the settings page from it.
  *
  * @see docs/server.md
  */
 export { createAtlasServer, type AtlasServer, type AtlasServerOptions } from './app.js';
 export { createAuth, type Auth, type Identity } from './auth.js';
 export { assertSafeToListen, ConfigError, loadServerConfig, SERVER_USAGE, type ResolvedServerConfig, type ResolvedToken } from './config.js';
+export { resolveS3Config, S3_VARIABLES, S3ConfigError, startupFailure, type S3Config } from './storage.js';
+export { userStateStore, USER_STATE_KEY, UserStateError, type UserStateStore } from './user-state.js';
 export { EventHub, type EventName } from './events.js';
 export { listen, type RunningServer } from './host.js';
 export { serveMain } from './main.js';
 export { classify, validateUpload, validKey, Refusal } from './validate.js';
-export { fileServerSettings, type ServerSettings, type SettingsSnapshot } from './settings.js';
+export { storedSettings, type GcSettings, type ServerSettings, type SettingsSnapshot } from './settings.js';
 export { LiveRuns, LiveRunError } from './live-runs.js';

package/dist/server/src/index.js

@@ -4,15 +4,19 @@
  * Everything needed to run the Atlas server from code: `createAtlasServer` (a `fetch` handler with the live report, uploads
  * and the viewer), `listen` (serve it over HTTP on Bun or Node), `loadServerConfig` (the file, the environment and the flags
  * as one checked configuration), `serveMain` (what `atlas serve` runs), `EventHub`, `createAuth`, and the upload checks.
+ * `atlas serve` keeps everything in S3: `resolveS3Config` reads the required `ATLAS_S3_*` variables, `userStateStore` the user
+ * state in the bucket, and `storedSettings` serves the settings page from it.
  *
  * @see docs/server.md
  */
 export { createAtlasServer } from './app.js';
 export { createAuth } from './auth.js';
 export { assertSafeToListen, ConfigError, loadServerConfig, SERVER_USAGE } from './config.js';
+export { resolveS3Config, S3_VARIABLES, S3ConfigError, startupFailure } from './storage.js';
+export { userStateStore, USER_STATE_KEY, UserStateError } from './user-state.js';
 export { EventHub } from './events.js';
 export { listen } from './host.js';
 export { serveMain } from './main.js';
 export { classify, validateUpload, validKey, Refusal } from './validate.js';
-export { fileServerSettings } from './settings.js';
+export { storedSettings } from './settings.js';
 export { LiveRuns, LiveRunError } from './live-runs.js';

package/dist/server/src/main.d.ts

@@ -1,30 +1,46 @@
 /**
  * @module `atlas serve`
  *
- * Starts the Atlas server from its configuration: opens every backend (creating the data folder), proves the upload
- * backend can be written to before listening, finds or builds the client, listens at once (so a liveness probe finds the
- * process while a large backend is still being read), and stops cleanly on SIGTERM or SIGINT. `serverOptionsOf` is shared
- * with `atlas dev`, so the two run the same server.
+ * Starts the Atlas server from its configuration. Storage comes first and is not optional: the required `ATLAS_S3_*`
+ * variables are checked (every problem listed together), the bucket is proven reachable and writable, and the user
+ * state (garbage collection policy and saved settings) is read from it, all before anything listens. Any failure prints
+ * a banner naming every variable and exits 2; there is no fallback to a local folder, memory or a database. Then it
+ * finds or builds the client, listens at once (so a liveness probe finds the process while a large bucket is still being
+ * read), and stops cleanly on SIGTERM or SIGINT. `serverOptionsOf` is shared with `atlas dev`, so the two run the same server.
  *
  * @see docs/server.md#run-it
  */
 import type { BackendInput } from '../../reports/src/compose.js';
 import { type AtlasServerOptions } from './app.js';
 import { type ResolvedServerConfig } from './config.js';
+import { type StoredUserState } from './user-state.js';
 type Env = Readonly<Record<string, string | undefined>>;
-/** What `serveMain` and `devMain` are given besides arguments: where output goes, and, for tests and embedding, the environment, working directory and an `AbortSignal` that stops the server (otherwise SIGTERM and SIGINT do). */
+/** What `serveMain` and `devMain` are given besides arguments: where output goes, and, for tests and embedding, the environment, working directory, the `fetch` S3 requests are sent with, and an `AbortSignal` that stops the server (otherwise SIGTERM and SIGINT do). */
 export type ServeIo = {
     readonly out: (text: string) => void;
     readonly env?: Env;
     readonly cwd?: string;
     readonly signal?: AbortSignal;
+    readonly fetch?: typeof fetch;
 };
-/** Open every backend of a configuration. The data folder is made if it is not there. */
-export declare function openConfiguredBackends(config: ResolvedServerConfig, env: Env, cwd: string): (BackendInput & {
+/** The id of the one backend: the S3 bucket. */
+export declare const S3_BACKEND_ID = "s3";
+/** Open the S3 bucket named by a configuration, with exactly the credentials it gives (no ambient AWS lookup). Nothing is contacted yet. */
+export declare function openConfiguredBackends(config: ResolvedServerConfig, fetchImpl?: typeof fetch): (BackendInput & {
     readonly writable: boolean;
 })[];
 /** What `createAtlasServer` is given for a configuration: `atlas serve` and `atlas dev` run the same server. */
 export declare function serverOptionsOf(config: ResolvedServerConfig, backends: ReturnType<typeof openConfiguredBackends>, log?: (line: Record<string, unknown>) => void): AtlasServerOptions;
+/**
+ * Check the storage and read the user state, in that order, with the failure banner for each way it can go wrong.
+ * Returns the opened backends and the state, or the text to print before exiting 2.
+ */
+export declare function connectStorage(config: ResolvedServerConfig, env: Env, io: Pick<ServeIo, 'fetch'>): Promise<{
+    readonly backends: ReturnType<typeof openConfiguredBackends>;
+    readonly stored: StoredUserState;
+} | {
+    readonly failure: string;
+}>;
 /** `atlas serve`. Runs until it is told to stop; returns the exit code. */
 export declare function serveMain(argv: readonly string[], io: ServeIo): Promise<number>;
 export {};

package/dist/server/src/main.js

@@ -1,38 +1,45 @@
 /**
  * @module `atlas serve`
  *
- * Starts the Atlas server from its configuration: opens every backend (creating the data folder), proves the upload
- * backend can be written to before listening, finds or builds the client, listens at once (so a liveness probe finds the
- * process while a large backend is still being read), and stops cleanly on SIGTERM or SIGINT. `serverOptionsOf` is shared
- * with `atlas dev`, so the two run the same server.
+ * Starts the Atlas server from its configuration. Storage comes first and is not optional: the required `ATLAS_S3_*`
+ * variables are checked (every problem listed together), the bucket is proven reachable and writable, and the user
+ * state (garbage collection policy and saved settings) is read from it, all before anything listens. Any failure prints
+ * a banner naming every variable and exits 2; there is no fallback to a local folder, memory or a database. Then it
+ * finds or builds the client, listens at once (so a liveness probe finds the process while a large bucket is still being
+ * read), and stops cleanly on SIGTERM or SIGINT. `serverOptionsOf` is shared with `atlas dev`, so the two run the same server.
  *
  * @see docs/server.md#run-it
  */
-import { mkdirSync } from 'node:fs';
 import path from 'node:path';
-import { openStore } from '../../reports/src/store-url.js';
+import { s3Store } from '../../reports/src/s3-store.js';
 import { createAtlasServer } from './app.js';
 import { ConfigError, loadServerConfig, SERVER_USAGE } from './config.js';
 import { listen } from './host.js';
-import { fileServerSettings } from './settings.js';
+import { storedSettings } from './settings.js';
 import { findSite } from './site.js';
-/** Open every backend of a configuration. The data folder is made if it is not there. */
-export function openConfiguredBackends(config, env, cwd) {
-    return config.backends.map((spec, i) => {
-        if (i === 0)
-            mkdirSync(spec.url, { recursive: true });
-        const opened = openStore(spec.url, { base: cwd, env });
-        return { id: spec.id, label: spec.label, store: opened.store, writable: spec.id === config.writeId };
+import { describeS3, inspectS3Env, S3ConfigError, startupFailure } from './storage.js';
+import { USER_STATE_KEY, UserStateError, userStateStore } from './user-state.js';
+/** The id of the one backend: the S3 bucket. */
+export const S3_BACKEND_ID = 's3';
+/** Open the S3 bucket named by a configuration, with exactly the credentials it gives (no ambient AWS lookup). Nothing is contacted yet. */
+export function openConfiguredBackends(config, fetchImpl) {
+    const { s3 } = config;
+    const store = s3Store({
+        bucket: s3.bucket, prefix: s3.prefix, region: s3.region, endpoint: s3.endpoint, forcePathStyle: s3.pathStyle,
+        credentials: { accessKeyId: s3.accessKeyId, secretAccessKey: s3.secretAccessKey, ...(s3.sessionToken ? { sessionToken: s3.sessionToken } : {}) },
+        env: {}, ...(fetchImpl ? { fetch: fetchImpl } : {})
     });
+    return [{ id: S3_BACKEND_ID, label: describeS3(s3), store, writable: true }];
 }
-/** Can the server put things in the place uploads go? Said at the start, not at the first upload. */
-async function probeWritable(backend) {
+/** Can the server use the bucket? Said at the start, not at the first upload: list, write, read back and delete. */
+async function probeStorage(backend) {
     const key = `layers/doctor-${Math.random().toString(36).slice(2, 10)}/probe`;
     try {
+        await backend.store.listKeys('server/');
         await backend.store.put(key, new TextEncoder().encode('atlas'), 'text/plain');
         if (!(await backend.store.head(key)))
             return 'an object was written but could not be found again';
-        await backend.store.delete(key).catch(() => { });
+        await backend.store.delete(key);
         return null;
     }
     catch (error) {
@@ -46,13 +53,35 @@ export function serverOptionsOf(config, backends, log) {
         ? (layer) => (!branches?.length || (layer.branch !== null && branches.includes(layer.branch))) && (!config.include.ignoreModified || layer.freshness === undefined)
         : undefined;
     return {
-        backends, write: config.writeId,
-        ...(/^https?:\/\//.test(config.backends.find(backend => backend.id === config.writeId).url) ? { liveStore: backends[0].store } : {}), tokens: config.tokens, anonymousRead: config.anonymousRead,
+        backends, write: S3_BACKEND_ID,
+        tokens: config.tokens, anonymousRead: config.anonymousRead,
         ...(config.outlineBranch ? { outlineBranch: config.outlineBranch } : {}), ...(include ? { include } : {}),
         runTimeoutMs: config.runTimeoutSeconds * 1000, runStallMs: config.runStallSeconds * 1000, refreshMs: config.refreshSeconds * 1000, limits: config.limits, heartbeatMs: config.heartbeatSeconds * 1000,
         ...(config.log && log ? { log } : {})
     };
 }
+/** What the garbage collection policy in S3 means, in one line for the log. */
+const gcLine = (stored) => stored.state.gc
+    ? `Garbage collection policy loaded from S3 (${USER_STATE_KEY}): ${stored.state.gc.enabled ? `ON, delete superseded layers older than ${stored.state.gc.maxAgeDays} days, keep the newest ${stored.state.gc.keepLatest}, every ${stored.state.gc.intervalHours} h` : 'OFF (saved but disabled)'}.\n`
+    : `No garbage collection policy has been saved in S3 (${USER_STATE_KEY}) yet: garbage collection is OFF and nothing is deleted. Define a policy in the Atlas UI at /admin.\n`;
+/**
+ * Check the storage and read the user state, in that order, with the failure banner for each way it can go wrong.
+ * Returns the opened backends and the state, or the text to print before exiting 2.
+ */
+export async function connectStorage(config, env, io) {
+    const backends = openConfiguredBackends(config, io.fetch);
+    const problem = await probeStorage(backends[0]);
+    if (problem)
+        return { failure: startupFailure(inspectS3Env(env), `the S3 storage ${describeS3(config.s3)} cannot be used with these settings`, problem) };
+    try {
+        return { backends, stored: await userStateStore(backends[0].store).read() };
+    }
+    catch (error) {
+        if (error instanceof UserStateError)
+            return { failure: startupFailure(inspectS3Env(env), 'the user state stored in S3 cannot be read', error.message) };
+        throw error;
+    }
+}
 /** `atlas serve`. Runs until it is told to stop; returns the exit code. */
 export async function serveMain(argv, io) {
     const cwd = io.cwd ?? process.cwd();
@@ -66,24 +95,41 @@ export async function serveMain(argv, io) {
         config = loadServerConfig(argv, env, cwd);
     }
     catch (error) {
+        if (error instanceof S3ConfigError) {
+            io.out(`${error.message}\n`);
+            return 2;
+        }
         if (error instanceof ConfigError) {
             io.out(`atlas serve: ${error.message}\n`);
             return 2;
         }
         throw error;
     }
-    const backends = openConfiguredBackends(config, env, cwd);
-    const writeBackend = backends.find(backend => backend.writable);
-    io.out(`${config.configFile ? `Configuration ${path.relative(cwd, config.configFile) || config.configFile}. ` : ''}Backends: ${backends.map(backend => `${backend.id}${backend.writable ? ' (uploads)' : ''}`).join(', ')}\n`);
-    const problem = await probeWritable(writeBackend);
-    if (problem) {
-        io.out(`atlas serve: uploads would go to "${writeBackend.id}" and it cannot be written to: ${problem}\n`);
+    const connected = await connectStorage(config, env, io);
+    if ('failure' in connected) {
+        io.out(`${connected.failure}\n`);
         return 2;
     }
+    const { backends, stored } = connected;
+    if (stored.state.settings) {
+        try {
+            config = loadServerConfig(argv, env, cwd, undefined, stored.state.settings);
+        }
+        catch (error) {
+            if (error instanceof ConfigError) {
+                io.out(`atlas serve: the settings saved in S3 (${USER_STATE_KEY}) do not work with this deployment: ${error.message}\n`);
+                return 2;
+            }
+            throw error;
+        }
+    }
+    io.out(`${config.configFile ? `Configuration ${path.relative(cwd, config.configFile) || config.configFile}. ` : ''}Storage: ${describeS3(config.s3)}. All reports and user state live there.\n`);
+    io.out(gcLine(stored));
     const site = await findSite(config.site, cwd, io.out);
     if (!site)
         io.out('No Atlas client to serve: build one with `atlas client build --no-data --out atlas-site`. The API is served meanwhile.\n');
-    const server = createAtlasServer({ settings: fileServerSettings(config, argv, env, cwd), ...serverOptionsOf(config, backends, line => io.out(`${JSON.stringify(line)}\n`)), ...(site ? { site } : {}) });
+    const { settings, gc } = storedSettings(config, argv, env, cwd, userStateStore(backends[0].store));
+    const server = createAtlasServer({ settings, gc, ...serverOptionsOf(config, backends, line => io.out(`${JSON.stringify(line)}\n`)), ...(site ? { site } : {}) });
     // Listen at once: the first pass over a large backend can take a while, and a liveness probe must find the
     // process meanwhile. /readyz is 503 until there is a report to show.
     const running = await listen(server.fetch, { port: config.port, host: config.host, maxBodyBytes: Math.max(config.limits.maxObjectMiB, config.limits.maxJsonMiB) * 1024 * 1024 });