@crvouga/atlas 0.17.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +52 -8
- package/dist/core/src/cli.d.ts +1 -1
- package/dist/core/src/cli.js +1 -1
- package/dist/reports/src/garbage.d.ts +39 -0
- package/dist/reports/src/garbage.js +82 -0
- package/dist/reports/src/report-index.d.ts +6 -0
- package/dist/reports/src/report-index.js +14 -0
- package/dist/schema/src/index.d.ts +1 -0
- package/dist/schema/src/index.js +1 -0
- package/dist/schema/src/server-config.d.ts +7 -49
- package/dist/schema/src/server-config.js +8 -21
- package/dist/schema/src/user-state.d.ts +115 -0
- package/dist/schema/src/user-state.js +54 -0
- package/dist/server/src/app.d.ts +16 -2
- package/dist/server/src/app.js +62 -0
- package/dist/server/src/config.d.ts +16 -21
- package/dist/server/src/config.js +48 -57
- package/dist/server/src/dev.d.ts +1 -1
- package/dist/server/src/dev.js +20 -7
- package/dist/server/src/index.d.ts +5 -1
- package/dist/server/src/index.js +5 -1
- package/dist/server/src/main.d.ts +23 -7
- package/dist/server/src/main.js +72 -26
- package/dist/server/src/openapi.js +24 -5
- package/dist/server/src/projects.js +89 -14
- package/dist/server/src/settings.d.ts +23 -7
- package/dist/server/src/settings.js +50 -60
- package/dist/server/src/storage.d.ts +68 -0
- package/dist/server/src/storage.js +125 -0
- package/dist/server/src/user-state.d.ts +31 -0
- package/dist/server/src/user-state.js +74 -0
- package/docs/agents.md +1 -1
- package/docs/deploying.md +38 -39
- package/docs/reference/cli.md +12 -7
- package/docs/reference/http-api.md +30 -2
- package/docs/reference/reports.md +71 -0
- package/docs/reference/schema.md +102 -16
- package/docs/reference/server.md +263 -41
- package/docs/report-backends.md +0 -144
- package/docs/report-contract.md +1 -1
- package/docs/runners.md +1 -1
- package/docs/server.md +141 -36
- package/docs/statecharts.md +1 -1
- package/docs/ui.md +1 -1
- package/docs/workspaces.md +5 -2
- package/llms-full.txt +231 -234
- package/llms.txt +1 -1
- package/package.json +2 -2
- package/packages/reports/STORAGE.md +1 -1
- package/packages/schema/schemas/gc-policy.schema.json +36 -0
- package/packages/schema/schemas/gc-run-result.schema.json +106 -0
- package/packages/schema/schemas/gc-snapshot.schema.json +122 -0
- package/packages/schema/schemas/server-config.schema.json +0 -41
- package/packages/schema/schemas/server-editable-settings.schema.json +0 -41
- package/packages/schema/schemas/server-settings-snapshot.schema.json +0 -82
- package/packages/schema/schemas/user-state.schema.json +140 -0
- package/packages/server/README.md +11 -6
- package/packages/server/openapi.json +510 -126
- package/packages/visualizer/src/admin/SettingsPage.tsx +183 -88
|
@@ -2802,6 +2802,76 @@ export function neighborStepFrame(
|
|
|
2802
2802
|
|
|
2803
2803
|
First frame of the next / previous step (skipping same-step frames).
|
|
2804
2804
|
|
|
2805
|
+
### Layer garbage collection
|
|
2806
|
+
|
|
2807
|
+
Source: `packages/reports/src/garbage.ts`. Import: `@crvouga/atlas/reports/garbage`
|
|
2808
|
+
|
|
2809
|
+
Applies a garbage collection policy (`GcPolicy`, chosen by a person in the Atlas UI and kept in the user state) to the
|
|
2810
|
+
layers in a store. Atlas is otherwise deletion-free, so this is deliberately conservative: a layer is deleted only when
|
|
2811
|
+
(1) the caller says the report has read it and every result it carries has been replaced by a newer layer (`eligible`),
|
|
2812
|
+
(2) it was recorded longer ago than the policy allows, and (3) it is not among the newest `keepLatest` layers. Deleting
|
|
2813
|
+
it therefore cannot change the combined report. Its catalog entry goes first, so nothing visible ever points at a
|
|
2814
|
+
missing manifest; its manifest next; then the captured-file blobs that no remaining layer refers to. If any remaining
|
|
2815
|
+
layer cannot be read, no blob is deleted: what cannot be read is unknown, never unused. Specs, runs and commit
|
|
2816
|
+
inventories are not touched.
|
|
2817
|
+
|
|
2818
|
+
See docs/server.md#user-state-and-garbage-collection
|
|
2819
|
+
|
|
2820
|
+
#### `GcCandidate`
|
|
2821
|
+
|
|
2822
|
+
```ts
|
|
2823
|
+
export type GcCandidate = { readonly id: string; readonly catalogKey: string; readonly recordedAtMs: number };
|
|
2824
|
+
```
|
|
2825
|
+
|
|
2826
|
+
One layer the policy would delete.
|
|
2827
|
+
|
|
2828
|
+
#### `GcPlan`
|
|
2829
|
+
|
|
2830
|
+
```ts
|
|
2831
|
+
export type GcPlan = {
|
|
2832
|
+
/** Layers to delete, oldest first. */
|
|
2833
|
+
readonly candidates: readonly GcCandidate[];
|
|
2834
|
+
/** Layers in the catalog that stay. */
|
|
2835
|
+
readonly kept: number;
|
|
2836
|
+
};
|
|
2837
|
+
```
|
|
2838
|
+
|
|
2839
|
+
What a policy would delete from a store right now.
|
|
2840
|
+
|
|
2841
|
+
#### `GcOutcome`
|
|
2842
|
+
|
|
2843
|
+
```ts
|
|
2844
|
+
export type GcOutcome = { readonly deletedLayers: number; readonly deletedBlobs: number };
|
|
2845
|
+
```
|
|
2846
|
+
|
|
2847
|
+
The outcome of applying a plan.
|
|
2848
|
+
|
|
2849
|
+
#### `planGarbage`
|
|
2850
|
+
|
|
2851
|
+
```ts
|
|
2852
|
+
export async function planGarbage(store: ObjectStore, input: { readonly policy: GcPolicy; readonly nowMs: number; readonly eligible: ReadonlySet<string> }): Promise<GcPlan>
|
|
2853
|
+
```
|
|
2854
|
+
|
|
2855
|
+
Decides which layers a policy deletes, without deleting anything. `eligible` is the set of layer ids the caller has
|
|
2856
|
+
read and knows no longer decide any result (`ReportIndex.superseded()`); a layer not in it is never a candidate.
|
|
2857
|
+
Lists the catalog only. With `policy.enabled === false` the plan is empty.
|
|
2858
|
+
|
|
2859
|
+
**Example**
|
|
2860
|
+
|
|
2861
|
+
```ts
|
|
2862
|
+
const plan = await planGarbage(store, { policy, nowMs: Date.now(), eligible: index.superseded() });
|
|
2863
|
+
```
|
|
2864
|
+
|
|
2865
|
+
#### `collectGarbage`
|
|
2866
|
+
|
|
2867
|
+
```ts
|
|
2868
|
+
export async function collectGarbage(store: ObjectStore, plan: GcPlan): Promise<GcOutcome>
|
|
2869
|
+
```
|
|
2870
|
+
|
|
2871
|
+
Deletes the layers of a plan, then the blobs only they used. Store errors throw after whatever was already deleted;
|
|
2872
|
+
running the same plan again finishes the job, because every delete is idempotent and the catalog entry (what
|
|
2873
|
+
makes a layer visible) always goes before the content.
|
|
2874
|
+
|
|
2805
2875
|
### URL hash routes of the reports viewer
|
|
2806
2876
|
|
|
2807
2877
|
Source: `packages/reports/src/hash.ts`. Import: `@crvouga/atlas/reports/hash`
|
|
@@ -5693,6 +5763,7 @@ It reads only immutable layers (`layers/index/…`), which is what makes "read o
|
|
|
5693
5763
|
- `json(): { readonly version: string; readonly text: string }` — The report as JSON text: exactly what `JSON.stringify({ ...snapshot(), active: [] })` writes, put together from the text of each layer, which is written once. `active` is what a store without live runs always sends: none.
|
|
5694
5764
|
- `delta(since: string): { readonly text: string; readonly version: string } | null` — What differs between the report a viewer holds (named by its version) and the current one: the layers that changed in full, and the ids of those that no longer hold a winner. Null when that version is not remembered, and the viewer should ask for the whole report. See `applyEffectiveDelta`.
|
|
5695
5765
|
- `get history(): TestHistory { return this.#history; }` — Every recorded result of every test and every run, as layers were folded in: what the test catalog, a test's history and the run list are built from. Read-only for callers; `history.rev` changes whenever a layer is taken in.
|
|
5766
|
+
- `superseded(): ReadonlySet<string>` — The ids of layers this index has read and that no longer decide any result. Removing one from its store cannot change the report (every slot it filled is held by a newer layer), which is what makes it safe for garbage collection to delete. A layer that failed to load, is hidden by `include`, or still holds a result is not here.
|
|
5696
5767
|
- `stats()` — For `/readyz` and logs.
|
|
5697
5768
|
|
|
5698
5769
|
### Test loading state
|
package/docs/reference/schema.md
CHANGED
|
@@ -1630,9 +1630,10 @@ export const RunsIndexSchema = …
|
|
|
1630
1630
|
Source: `packages/schema/src/server-config.ts`
|
|
1631
1631
|
|
|
1632
1632
|
The schema of `atlas.server.json`, the file that configures `atlas serve` and `atlas dev`: where to listen, which
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
a
|
|
1633
|
+
bearer tokens may read, publish or administer, and limits. Where reports are kept is not here: the server stores
|
|
1634
|
+
everything in S3-compatible storage named by required `ATLAS_S3_*` environment variables, never in this file.
|
|
1635
|
+
Every field has a default. Unlike the report format, unknown keys are an error, so a typo (or a removed `data`,
|
|
1636
|
+
`backends` or `write` key) is reported instead of ignored. Flags and environment variables override the file.
|
|
1636
1637
|
|
|
1637
1638
|
See docs/deploying.md
|
|
1638
1639
|
See docs/server.md
|
|
@@ -1653,15 +1654,6 @@ export type ServerRole = z.infer<typeof ServerRoleSchema>;
|
|
|
1653
1654
|
|
|
1654
1655
|
One of the roles of `ServerRoleSchema`.
|
|
1655
1656
|
|
|
1656
|
-
#### `ServerBackendSchema`
|
|
1657
|
-
|
|
1658
|
-
```ts
|
|
1659
|
-
export const ServerBackendSchema = …
|
|
1660
|
-
```
|
|
1661
|
-
|
|
1662
|
-
A further store the server reads together with its data folder, and optionally writes to: an id, a label and a
|
|
1663
|
-
location URL.
|
|
1664
|
-
|
|
1665
1657
|
#### `ServerTokenSchema`
|
|
1666
1658
|
|
|
1667
1659
|
```ts
|
|
@@ -1677,8 +1669,7 @@ A bearer token: a name for logs, a role, and the token itself or the name of the
|
|
|
1677
1669
|
export const ServerConfigSchema = …
|
|
1678
1670
|
```
|
|
1679
1671
|
|
|
1680
|
-
The whole `atlas.server.json`: listening address,
|
|
1681
|
-
viewer to serve, which layers take part in the report, refresh and heartbeat intervals, request limits and logging.
|
|
1672
|
+
The whole `atlas.server.json`: listening address, authentication, the viewer to serve, which layers take part in the report, refresh and heartbeat intervals, request limits and logging.
|
|
1682
1673
|
|
|
1683
1674
|
#### `ServerConfig`
|
|
1684
1675
|
|
|
@@ -1702,7 +1693,7 @@ A server configuration as written, before defaults are applied.
|
|
|
1702
1693
|
export const ServerEditableSettingsSchema = …
|
|
1703
1694
|
```
|
|
1704
1695
|
|
|
1705
|
-
|
|
1696
|
+
Report and live-run settings editable by an admin. Storage, authentication, listeners and the served site stay deployment-owned. Replaces the saved portion of the user state; omitted fields use defaults after restart.
|
|
1706
1697
|
|
|
1707
1698
|
#### `ServerSettingsSnapshotSchema`
|
|
1708
1699
|
|
|
@@ -1710,7 +1701,7 @@ Storage and report settings editable by an admin. Authentication, listeners and
|
|
|
1710
1701
|
export const ServerSettingsSnapshotSchema = …
|
|
1711
1702
|
```
|
|
1712
1703
|
|
|
1713
|
-
The settings API response:
|
|
1704
|
+
The settings API response: editable configuration saved in the user state in S3, active values, concurrency revision and restart/precedence guidance. Authentication token settings are excluded.
|
|
1714
1705
|
|
|
1715
1706
|
### Statechart spec schemas
|
|
1716
1707
|
|
|
@@ -1962,3 +1953,98 @@ export function isStatechartFile(path: string)
|
|
|
1962
1953
|
```
|
|
1963
1954
|
|
|
1964
1955
|
Whether a spec-relative path (using `/`) matches `STATECHART_FILE_PATTERNS`.
|
|
1956
|
+
|
|
1957
|
+
### User state schema
|
|
1958
|
+
|
|
1959
|
+
Source: `packages/schema/src/user-state.ts`
|
|
1960
|
+
|
|
1961
|
+
The one JSON document that holds everything a person using Atlas has chosen, as opposed to what an operator deployed:
|
|
1962
|
+
the garbage collection policy and the saved server settings. The server keeps it in S3-compatible object storage
|
|
1963
|
+
(`server/state.json` under the configured prefix) and nowhere else: not in a file, a database or the browser. A
|
|
1964
|
+
document that does not exist yet means nothing has been chosen; a document that cannot be read is an error, never
|
|
1965
|
+
an empty state. Unknown keys are kept, so an older server does not drop what a newer one wrote.
|
|
1966
|
+
|
|
1967
|
+
See docs/server.md#user-state-and-garbage-collection
|
|
1968
|
+
|
|
1969
|
+
#### `GcPolicySchema`
|
|
1970
|
+
|
|
1971
|
+
```ts
|
|
1972
|
+
export const GcPolicySchema = …
|
|
1973
|
+
```
|
|
1974
|
+
|
|
1975
|
+
The garbage collection policy: which report layers the server may delete from S3, and when. Only layers that the
|
|
1976
|
+
report has loaded and that no longer decide any result (every check they carry has since been superseded by a newer
|
|
1977
|
+
layer) are ever candidates, and the newest `keepLatest` layers are always kept. With `enabled: false` nothing is deleted.
|
|
1978
|
+
|
|
1979
|
+
#### `GcPolicy`
|
|
1980
|
+
|
|
1981
|
+
```ts
|
|
1982
|
+
export type GcPolicy = z.output<typeof GcPolicySchema>;
|
|
1983
|
+
```
|
|
1984
|
+
|
|
1985
|
+
A garbage collection policy as stored, defaults applied.
|
|
1986
|
+
|
|
1987
|
+
#### `GcPolicyInput`
|
|
1988
|
+
|
|
1989
|
+
```ts
|
|
1990
|
+
export type GcPolicyInput = z.input<typeof GcPolicySchema>;
|
|
1991
|
+
```
|
|
1992
|
+
|
|
1993
|
+
A garbage collection policy as written, before defaults are applied.
|
|
1994
|
+
|
|
1995
|
+
#### `GcRunSchema`
|
|
1996
|
+
|
|
1997
|
+
```ts
|
|
1998
|
+
export const GcRunSchema = …
|
|
1999
|
+
```
|
|
2000
|
+
|
|
2001
|
+
What one pass of garbage collection found and did, for one project.
|
|
2002
|
+
|
|
2003
|
+
#### `GcRun`
|
|
2004
|
+
|
|
2005
|
+
```ts
|
|
2006
|
+
export type GcRun = z.output<typeof GcRunSchema>;
|
|
2007
|
+
```
|
|
2008
|
+
|
|
2009
|
+
One pass of garbage collection, see {@link GcRunSchema}.
|
|
2010
|
+
|
|
2011
|
+
#### `UserStateSchema`
|
|
2012
|
+
|
|
2013
|
+
```ts
|
|
2014
|
+
export const UserStateSchema = …
|
|
2015
|
+
```
|
|
2016
|
+
|
|
2017
|
+
The user state document stored in S3. Both parts are optional: absent means "never chosen".
|
|
2018
|
+
|
|
2019
|
+
#### `UserState`
|
|
2020
|
+
|
|
2021
|
+
```ts
|
|
2022
|
+
export type UserState = z.output<typeof UserStateSchema>;
|
|
2023
|
+
```
|
|
2024
|
+
|
|
2025
|
+
The user state document, see {@link UserStateSchema}.
|
|
2026
|
+
|
|
2027
|
+
#### `GcSnapshotSchema`
|
|
2028
|
+
|
|
2029
|
+
```ts
|
|
2030
|
+
export const GcSnapshotSchema = …
|
|
2031
|
+
```
|
|
2032
|
+
|
|
2033
|
+
`GET /api/settings/gc` and `PUT /api/settings/gc`: the policy as saved in S3 (`null` when none has been saved), the
|
|
2034
|
+
revision to send back in `If-Match`, and the result of the last pass this server ran.
|
|
2035
|
+
|
|
2036
|
+
#### `GcSnapshot`
|
|
2037
|
+
|
|
2038
|
+
```ts
|
|
2039
|
+
export type GcSnapshot = z.output<typeof GcSnapshotSchema>;
|
|
2040
|
+
```
|
|
2041
|
+
|
|
2042
|
+
The garbage collection settings response, see {@link GcSnapshotSchema}.
|
|
2043
|
+
|
|
2044
|
+
#### `GcRunResultSchema`
|
|
2045
|
+
|
|
2046
|
+
```ts
|
|
2047
|
+
export const GcRunResultSchema = …
|
|
2048
|
+
```
|
|
2049
|
+
|
|
2050
|
+
`POST /api/settings/gc/run`: the passes that ran, one per project.
|
package/docs/reference/server.md
CHANGED
|
@@ -42,8 +42,12 @@ export type AtlasServerOptions = {
|
|
|
42
42
|
readonly log?: (line: Record<string, unknown>) => void;
|
|
43
43
|
readonly now?: () => number;
|
|
44
44
|
readonly version?: string;
|
|
45
|
-
/** Admin configuration persistence, supplied by atlas serve. Changes apply on restart. */
|
|
45
|
+
/** Admin configuration persistence, supplied by atlas serve; kept in S3. Changes apply on restart. */
|
|
46
46
|
readonly settings?: ServerSettings;
|
|
47
|
+
/** The garbage collection policy, kept in S3 and read from there each time the server collects. Supplied by atlas serve. */
|
|
48
|
+
readonly gc?: GcSettings;
|
|
49
|
+
/** How often (ms) the server checks whether a garbage collection pass is due. Default 60 000; for tests. */
|
|
50
|
+
readonly gcCheckMs?: number;
|
|
47
51
|
/** Extra runtime configuration written into the page. */
|
|
48
52
|
readonly page?: Record<string, unknown>;
|
|
49
53
|
};
|
|
@@ -62,6 +66,12 @@ export type AtlasServer = {
|
|
|
62
66
|
readonly routes: readonly { readonly method: string; readonly path: string }[];
|
|
63
67
|
/** Resolves when the first pass over the backends is done. */
|
|
64
68
|
readonly ready: Promise<void>;
|
|
69
|
+
/**
|
|
70
|
+
* Apply a garbage collection policy to this project's upload backend. Deletes only layers the report has read and
|
|
71
|
+
* that no longer decide any result, older than the policy allows and beyond its `keepLatest`. With `dryRun` it only
|
|
72
|
+
* counts. Never throws: a failure is the `error` of the returned run.
|
|
73
|
+
*/
|
|
74
|
+
collectGarbage(policy: GcPolicy, options: { readonly dryRun: boolean; readonly project: string }): Promise<GcRun>;
|
|
65
75
|
/** Stop refreshing and end every live connection. */
|
|
66
76
|
close(): void;
|
|
67
77
|
};
|
|
@@ -136,11 +146,13 @@ The authentication of one server: `identify` (who is this), `check` (may they, f
|
|
|
136
146
|
|
|
137
147
|
Source: `packages/server/src/config.ts`
|
|
138
148
|
|
|
139
|
-
Loads how the server runs: `atlas.server.json`, then environment
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
149
|
+
Loads how the server runs: the settings saved in S3 (the user state), then `atlas.server.json`, then environment
|
|
150
|
+
variables, then flags, each overriding the last, checked against `ServerConfigSchema` and resolved (storage read from
|
|
151
|
+
the required `ATLAS_S3_*` variables, tokens read). Storage is never optional: all reports and user state live in
|
|
152
|
+
S3-compatible object storage, so missing or invalid storage variables are an `S3ConfigError` naming every variable.
|
|
153
|
+
Every other mistake is a `ConfigError` that says what to change, and a server that would listen beyond this machine
|
|
154
|
+
with no tokens refuses to start. `SERVER_SETTINGS` documents every setting once; the table in docs/server.md is
|
|
155
|
+
generated from it and a test proves each row.
|
|
144
156
|
|
|
145
157
|
See docs/server.md#configuration
|
|
146
158
|
|
|
@@ -152,23 +164,12 @@ export type ResolvedToken = { readonly name: string; readonly role: ServerRole;
|
|
|
152
164
|
|
|
153
165
|
A token as the server holds it: the `name` it is shown under in logs, its `role`, and the secret `token`.
|
|
154
166
|
|
|
155
|
-
#### `BackendSpec`
|
|
156
|
-
|
|
157
|
-
```ts
|
|
158
|
-
export type BackendSpec = { readonly id: string; readonly label: string; readonly url: string };
|
|
159
|
-
```
|
|
160
|
-
|
|
161
|
-
A backend as configured: its `id`, the `label` people see, and the `url` (a folder, `s3://…`, or another server) to open.
|
|
162
|
-
|
|
163
167
|
#### `ResolvedServerConfig`
|
|
164
168
|
|
|
165
169
|
```ts
|
|
166
|
-
export type ResolvedServerConfig = Omit<ServerConfig, '
|
|
167
|
-
/** The
|
|
168
|
-
readonly
|
|
169
|
-
/** Every backend, the data folder first. All are read; uploads go to `writeId`. */
|
|
170
|
-
readonly backends: readonly BackendSpec[];
|
|
171
|
-
readonly writeId: string;
|
|
170
|
+
export type ResolvedServerConfig = Omit<ServerConfig, 'auth'> & {
|
|
171
|
+
/** The S3-compatible storage that holds every report and all user state. Required; there is no other. */
|
|
172
|
+
readonly s3: S3Config;
|
|
172
173
|
readonly tokens: readonly ResolvedToken[];
|
|
173
174
|
readonly anonymousRead: boolean;
|
|
174
175
|
readonly insecure: boolean;
|
|
@@ -241,12 +242,13 @@ The text `atlas serve --help` prints. Also the source of the `atlas serve` secti
|
|
|
241
242
|
#### `loadServerConfig`
|
|
242
243
|
|
|
243
244
|
```ts
|
|
244
|
-
export function loadServerConfig(argv: readonly string[], env: Env, cwd: string, read: (file: string) => string = file => readFileSync(file, 'utf8')): ResolvedServerConfig
|
|
245
|
+
export function loadServerConfig(argv: readonly string[], env: Env, cwd: string, read: (file: string) => string = file => readFileSync(file, 'utf8'), saved: Record<string, unknown> = {}): ResolvedServerConfig
|
|
245
246
|
```
|
|
246
247
|
|
|
247
248
|
Read the file named by `--config`/`ATLAS_CONFIG`, or `atlas.server.json` in the folder if there is one, lay the
|
|
248
|
-
environment and then the flags over it, and check the result.
|
|
249
|
-
what to change
|
|
249
|
+
environment and then the flags over it, and check the result. `saved` is the settings kept in S3 (the user state),
|
|
250
|
+
the lowest layer. Every problem is a {@link ConfigError} that says what to change; missing or invalid storage
|
|
251
|
+
variables are an `S3ConfigError` that names all of them, and are checked first.
|
|
250
252
|
|
|
251
253
|
#### `assertSafeToListen`
|
|
252
254
|
|
|
@@ -365,6 +367,8 @@ Source: `packages/server/src/index.ts`. Import: `@crvouga/atlas/server`
|
|
|
365
367
|
Everything needed to run the Atlas server from code: `createAtlasServer` (a `fetch` handler with the live report, uploads
|
|
366
368
|
and the viewer), `listen` (serve it over HTTP on Bun or Node), `loadServerConfig` (the file, the environment and the flags
|
|
367
369
|
as one checked configuration), `serveMain` (what `atlas serve` runs), `EventHub`, `createAuth`, and the upload checks.
|
|
370
|
+
`atlas serve` keeps everything in S3: `resolveS3Config` reads the required `ATLAS_S3_*` variables, `userStateStore` the user
|
|
371
|
+
state in the bucket, and `storedSettings` serves the settings page from it.
|
|
368
372
|
|
|
369
373
|
See docs/server.md
|
|
370
374
|
|
|
@@ -442,28 +446,38 @@ The guide for one server, Markdown. Stable in structure: an agent may read it by
|
|
|
442
446
|
|
|
443
447
|
Source: `packages/server/src/main.ts`
|
|
444
448
|
|
|
445
|
-
Starts the Atlas server from its configuration
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
+
Starts the Atlas server from its configuration. Storage comes first and is not optional: the required `ATLAS_S3_*`
|
|
450
|
+
variables are checked (every problem listed together), the bucket is proven reachable and writable, and the user
|
|
451
|
+
state (garbage collection policy and saved settings) is read from it, all before anything listens. Any failure prints
|
|
452
|
+
a banner naming every variable and exits 2; there is no fallback to a local folder, memory or a database. Then it
|
|
453
|
+
finds or builds the client, listens at once (so a liveness probe finds the process while a large bucket is still being
|
|
454
|
+
read), and stops cleanly on SIGTERM or SIGINT. `serverOptionsOf` is shared with `atlas dev`, so the two run the same server.
|
|
449
455
|
|
|
450
456
|
See docs/server.md#run-it
|
|
451
457
|
|
|
452
458
|
#### `ServeIo`
|
|
453
459
|
|
|
454
460
|
```ts
|
|
455
|
-
export type ServeIo = { readonly out: (text: string) => void; readonly env?: Env; readonly cwd?: string; readonly signal?: AbortSignal };
|
|
461
|
+
export type ServeIo = { readonly out: (text: string) => void; readonly env?: Env; readonly cwd?: string; readonly signal?: AbortSignal; readonly fetch?: typeof fetch };
|
|
456
462
|
```
|
|
457
463
|
|
|
458
|
-
What `serveMain` and `devMain` are given besides arguments: where output goes, and, for tests and embedding, the environment, working directory and an `AbortSignal` that stops the server (otherwise SIGTERM and SIGINT do).
|
|
464
|
+
What `serveMain` and `devMain` are given besides arguments: where output goes, and, for tests and embedding, the environment, working directory, the `fetch` S3 requests are sent with, and an `AbortSignal` that stops the server (otherwise SIGTERM and SIGINT do).
|
|
465
|
+
|
|
466
|
+
#### `S3_BACKEND_ID`
|
|
467
|
+
|
|
468
|
+
```ts
|
|
469
|
+
export const S3_BACKEND_ID = …
|
|
470
|
+
```
|
|
471
|
+
|
|
472
|
+
The id of the one backend: the S3 bucket.
|
|
459
473
|
|
|
460
474
|
#### `openConfiguredBackends`
|
|
461
475
|
|
|
462
476
|
```ts
|
|
463
|
-
export function openConfiguredBackends(config: ResolvedServerConfig,
|
|
477
|
+
export function openConfiguredBackends(config: ResolvedServerConfig, fetchImpl?: typeof fetch): (BackendInput & { readonly writable: boolean })[]
|
|
464
478
|
```
|
|
465
479
|
|
|
466
|
-
Open
|
|
480
|
+
Open the S3 bucket named by a configuration, with exactly the credentials it gives (no ambient AWS lookup). Nothing is contacted yet.
|
|
467
481
|
|
|
468
482
|
#### `serverOptionsOf`
|
|
469
483
|
|
|
@@ -473,6 +487,15 @@ export function serverOptionsOf(config: ResolvedServerConfig, backends: ReturnTy
|
|
|
473
487
|
|
|
474
488
|
What `createAtlasServer` is given for a configuration: `atlas serve` and `atlas dev` run the same server.
|
|
475
489
|
|
|
490
|
+
#### `connectStorage`
|
|
491
|
+
|
|
492
|
+
```ts
|
|
493
|
+
export async function connectStorage(config: ResolvedServerConfig, env: Env, io: Pick<ServeIo, 'fetch'>): Promise<{ readonly backends: ReturnType<typeof openConfiguredBackends>; readonly stored: StoredUserState } | { readonly failure: string }>
|
|
494
|
+
```
|
|
495
|
+
|
|
496
|
+
Check the storage and read the user state, in that order, with the failure banner for each way it can go wrong.
|
|
497
|
+
Returns the opened backends and the state, or the text to print before exiting 2.
|
|
498
|
+
|
|
476
499
|
#### `serveMain`
|
|
477
500
|
|
|
478
501
|
```ts
|
|
@@ -638,8 +661,10 @@ const html = await renderSettingsPage();
|
|
|
638
661
|
Source: `packages/server/src/settings.ts`
|
|
639
662
|
|
|
640
663
|
@module Persisted server settings
|
|
641
|
-
The
|
|
642
|
-
|
|
664
|
+
The settings page edits two things, and both are kept in the user state document in S3 (`server/state.json`), never on
|
|
665
|
+
the server's disk: the report and live-run settings (applied at the next start, below the configuration file,
|
|
666
|
+
the environment and the flags in precedence) and the garbage collection policy (read by the server each time it
|
|
667
|
+
collects). Storage and authentication are deployment-owned and are not editable here.
|
|
643
668
|
|
|
644
669
|
#### `SettingsSnapshot`
|
|
645
670
|
|
|
@@ -647,7 +672,7 @@ environment variables and flags keep their documented precedence. Authentication
|
|
|
647
672
|
export type SettingsSnapshot = { readonly revision: string; readonly settings: Record<string, unknown>; readonly active: Record<string, unknown>; readonly restartRequired: boolean; readonly precedence: string };
|
|
648
673
|
```
|
|
649
674
|
|
|
650
|
-
The
|
|
675
|
+
The saved editable settings, the active settings and a revision for optimistic concurrency. Authentication token settings are excluded.
|
|
651
676
|
|
|
652
677
|
#### `ServerSettings`
|
|
653
678
|
|
|
@@ -655,22 +680,38 @@ The persisted editable settings, active settings and a revision for optimistic c
|
|
|
655
680
|
export type ServerSettings = { read(): Promise<SettingsSnapshot>; save(settings: unknown, revision: string): Promise<SettingsSnapshot> };
|
|
656
681
|
```
|
|
657
682
|
|
|
658
|
-
Storage for the admin API; saving is
|
|
683
|
+
Storage for the admin API; saving is serialised and refuses a stale revision.
|
|
684
|
+
|
|
685
|
+
#### `GcSettings`
|
|
686
|
+
|
|
687
|
+
```ts
|
|
688
|
+
export type GcSettings = {
|
|
689
|
+
/** The policy in S3 with its revision and the last passes. */
|
|
690
|
+
read(): Promise<GcSnapshot>;
|
|
691
|
+
/** Validates and saves a policy; a stale `revision` is refused. */
|
|
692
|
+
save(policy: unknown, revision: string): Promise<GcSnapshot>;
|
|
693
|
+
/** The policy in S3 right now, or `null` when none has been saved. Read from S3 every time. */
|
|
694
|
+
policy(): Promise<GcPolicy | null>;
|
|
695
|
+
/** Remembers the result of a pass, for `read`. */
|
|
696
|
+
record(runs: readonly GcRun[]): void;
|
|
697
|
+
};
|
|
698
|
+
```
|
|
699
|
+
|
|
700
|
+
The garbage collection policy as the admin API and the collector see it.
|
|
659
701
|
|
|
660
|
-
#### `
|
|
702
|
+
#### `storedSettings`
|
|
661
703
|
|
|
662
704
|
```ts
|
|
663
|
-
export function
|
|
705
|
+
export function storedSettings(config: ResolvedServerConfig, argv: readonly string[], env: Readonly<Record<string, string | undefined>>, cwd: string, userState: UserStateStore): { settings: ServerSettings; gc: GcSettings }
|
|
664
706
|
```
|
|
665
707
|
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
Changes need a server restart; deployment environment variables and flags override the file.
|
|
708
|
+
The settings and garbage collection policy kept in `userState` (S3). Validates a settings change against the
|
|
709
|
+
whole deployment (file, environment and flags) before saving, so what is saved can start. Never moves report data.
|
|
669
710
|
|
|
670
711
|
**Example**
|
|
671
712
|
|
|
672
713
|
```ts
|
|
673
|
-
const settings =
|
|
714
|
+
const { settings, gc } = storedSettings(config, argv, process.env, process.cwd(), userStateStore(store));
|
|
674
715
|
```
|
|
675
716
|
|
|
676
717
|
### Finding the client
|
|
@@ -713,6 +754,118 @@ export async function findSite(named: string | undefined, cwd: string, log: (tex
|
|
|
713
754
|
|
|
714
755
|
The client to serve: the one named, a build beside the project, or one built into the user's cache.
|
|
715
756
|
|
|
757
|
+
### Required S3 storage configuration
|
|
758
|
+
|
|
759
|
+
Source: `packages/server/src/storage.ts`
|
|
760
|
+
|
|
761
|
+
The Atlas server keeps ALL reports and ALL user state (the garbage collection policy and saved settings) in
|
|
762
|
+
S3-compatible object storage, and nowhere else: no local folder, no memory, no database. So the storage settings
|
|
763
|
+
are required, come only from environment variables (they hold secrets), and have no defaults for where or as whom
|
|
764
|
+
to connect. `resolveS3Config` checks every variable in one pass and reports all the problems together;
|
|
765
|
+
`startupFailure` renders them as a banner that cannot be missed. `S3_VARIABLES` is the single description of the
|
|
766
|
+
variables: the banner, `.env.example` and the documentation table are all made from it.
|
|
767
|
+
|
|
768
|
+
See docs/server.md#required-s3-storage
|
|
769
|
+
|
|
770
|
+
#### `S3Variable`
|
|
771
|
+
|
|
772
|
+
```ts
|
|
773
|
+
export type S3Variable = {
|
|
774
|
+
readonly name: string;
|
|
775
|
+
readonly required: boolean;
|
|
776
|
+
/** Shown in the banner and the documentation. Never a real secret. */
|
|
777
|
+
readonly meaning: string;
|
|
778
|
+
readonly example: string;
|
|
779
|
+
/** Secrets are never printed, not even when invalid. */
|
|
780
|
+
readonly secret?: true;
|
|
781
|
+
/** Returns what is wrong with a value that is set, or null when it is fine. */
|
|
782
|
+
readonly check: (value: string) => string | null;
|
|
783
|
+
};
|
|
784
|
+
```
|
|
785
|
+
|
|
786
|
+
One environment variable of the storage configuration.
|
|
787
|
+
|
|
788
|
+
#### `S3_VARIABLES`
|
|
789
|
+
|
|
790
|
+
```ts
|
|
791
|
+
export const S3_VARIABLES: readonly S3Variable[]
|
|
792
|
+
```
|
|
793
|
+
|
|
794
|
+
Every variable that configures storage, required ones first.
|
|
795
|
+
|
|
796
|
+
#### `S3Config`
|
|
797
|
+
|
|
798
|
+
```ts
|
|
799
|
+
export type S3Config = {
|
|
800
|
+
readonly endpoint: string;
|
|
801
|
+
readonly bucket: string;
|
|
802
|
+
readonly region: string;
|
|
803
|
+
readonly prefix: string;
|
|
804
|
+
readonly pathStyle: boolean;
|
|
805
|
+
readonly accessKeyId: string;
|
|
806
|
+
readonly secretAccessKey: string;
|
|
807
|
+
readonly sessionToken?: string;
|
|
808
|
+
};
|
|
809
|
+
```
|
|
810
|
+
|
|
811
|
+
The storage settings once checked: everything needed to open the bucket, with credentials spelled out.
|
|
812
|
+
|
|
813
|
+
#### `S3Finding`
|
|
814
|
+
|
|
815
|
+
```ts
|
|
816
|
+
export type S3Finding = { readonly variable: S3Variable; readonly status: 'ok' | 'missing' | 'invalid' | 'unset'; readonly problem?: string };
|
|
817
|
+
```
|
|
818
|
+
|
|
819
|
+
The state of one variable after checking, for the banner.
|
|
820
|
+
|
|
821
|
+
#### `S3ConfigError`
|
|
822
|
+
|
|
823
|
+
```ts
|
|
824
|
+
export class S3ConfigError extends Error
|
|
825
|
+
```
|
|
826
|
+
|
|
827
|
+
The storage configuration could not be used. `findings` has every variable, not only the first problem.
|
|
828
|
+
|
|
829
|
+
#### `inspectS3Env`
|
|
830
|
+
|
|
831
|
+
```ts
|
|
832
|
+
export function inspectS3Env(env: Env): readonly S3Finding[]
|
|
833
|
+
```
|
|
834
|
+
|
|
835
|
+
Looks at every storage variable and returns what each is. Nothing here throws.
|
|
836
|
+
|
|
837
|
+
#### `resolveS3Config`
|
|
838
|
+
|
|
839
|
+
```ts
|
|
840
|
+
export function resolveS3Config(env: Env): S3Config
|
|
841
|
+
```
|
|
842
|
+
|
|
843
|
+
The storage settings from the environment. Throws {@link S3ConfigError}, whose message lists every variable and
|
|
844
|
+
marks each missing or invalid one, when any required variable is absent or any set variable is malformed.
|
|
845
|
+
|
|
846
|
+
**Example**
|
|
847
|
+
|
|
848
|
+
```ts
|
|
849
|
+
const s3 = resolveS3Config(process.env);
|
|
850
|
+
```
|
|
851
|
+
|
|
852
|
+
#### `describeS3`
|
|
853
|
+
|
|
854
|
+
```ts
|
|
855
|
+
export const describeS3 = …
|
|
856
|
+
```
|
|
857
|
+
|
|
858
|
+
Where the storage is, without a secret: what the log may say.
|
|
859
|
+
|
|
860
|
+
#### `startupFailure`
|
|
861
|
+
|
|
862
|
+
```ts
|
|
863
|
+
export function startupFailure(findings: readonly S3Finding[], reason: string, detail?: string): string
|
|
864
|
+
```
|
|
865
|
+
|
|
866
|
+
The banner printed when the server cannot start because of storage: why storage is required, then EVERY variable by
|
|
867
|
+
name with what is wrong with it. Values of secrets are never printed.
|
|
868
|
+
|
|
716
869
|
### Store protocol routes
|
|
717
870
|
|
|
718
871
|
Source: `packages/server/src/store-api.ts`
|
|
@@ -754,6 +907,75 @@ export function storeApi(options: StoreApiOptions)
|
|
|
754
907
|
|
|
755
908
|
The Hono app of the store protocol, to be mounted at `/api/store`. Reads need role `read`; writes need `publish` through a Bearer token.
|
|
756
909
|
|
|
910
|
+
### User state in S3
|
|
911
|
+
|
|
912
|
+
Source: `packages/server/src/user-state.ts`
|
|
913
|
+
|
|
914
|
+
Reads and writes the user state document (`UserStateSchema`: the garbage collection policy and the saved settings) as
|
|
915
|
+
one JSON object at `server/state.json` in the object store, which in a running server is the S3 bucket. This is the
|
|
916
|
+
only place user state is kept: not a file, not a database, not the browser. A missing object is an empty state
|
|
917
|
+
("nothing chosen yet"); an object that cannot be read or does not match the schema is an error that stops the
|
|
918
|
+
server from starting and is never overwritten, so a transient or corrupt read can never silently reset anyone's choices.
|
|
919
|
+
Saving refuses a stale revision (the SHA-256 of the stored text), serialised within this process. An object store has
|
|
920
|
+
no conditional writes, so two servers saving at the same instant could still race; run one server per bucket.
|
|
921
|
+
|
|
922
|
+
See docs/server.md#user-state-and-garbage-collection
|
|
923
|
+
|
|
924
|
+
#### `USER_STATE_KEY`
|
|
925
|
+
|
|
926
|
+
```ts
|
|
927
|
+
export const USER_STATE_KEY = …
|
|
928
|
+
```
|
|
929
|
+
|
|
930
|
+
Where the user state lives, relative to the store (the bucket plus `ATLAS_S3_PREFIX`).
|
|
931
|
+
|
|
932
|
+
#### `StoredUserState`
|
|
933
|
+
|
|
934
|
+
```ts
|
|
935
|
+
export type StoredUserState = { readonly state: UserState; readonly revision: string; readonly exists: boolean };
|
|
936
|
+
```
|
|
937
|
+
|
|
938
|
+
The user state as stored: the document, its `revision` and whether the object `exists` yet.
|
|
939
|
+
|
|
940
|
+
#### `UserStateError`
|
|
941
|
+
|
|
942
|
+
```ts
|
|
943
|
+
export class UserStateError extends Error {}
|
|
944
|
+
```
|
|
945
|
+
|
|
946
|
+
Raised when the stored user state cannot be trusted. The message says where it is and what to do.
|
|
947
|
+
|
|
948
|
+
#### `UserStateStore`
|
|
949
|
+
|
|
950
|
+
```ts
|
|
951
|
+
export type UserStateStore = {
|
|
952
|
+
/** Reads the document. Throws {@link UserStateError} when it exists but is unreadable or invalid. */
|
|
953
|
+
read(): Promise<StoredUserState>;
|
|
954
|
+
/**
|
|
955
|
+
* Applies `change` to the stored state and writes the result, if `revision` is still the stored one. Throws a
|
|
956
|
+
* `UserStateError` starting with "Settings changed" for a stale revision.
|
|
957
|
+
*/
|
|
958
|
+
update(revision: string, change: (state: UserState) => UserState): Promise<StoredUserState>;
|
|
959
|
+
};
|
|
960
|
+
```
|
|
961
|
+
|
|
962
|
+
The user state of one server: read it, or change it against the revision that was read.
|
|
963
|
+
|
|
964
|
+
#### `userStateStore`
|
|
965
|
+
|
|
966
|
+
```ts
|
|
967
|
+
export function userStateStore(store: ObjectStore): UserStateStore
|
|
968
|
+
```
|
|
969
|
+
|
|
970
|
+
The user state kept in `store` at {@link USER_STATE_KEY}.
|
|
971
|
+
|
|
972
|
+
**Example**
|
|
973
|
+
|
|
974
|
+
```ts
|
|
975
|
+
const userState = userStateStore(s3Backend.store);
|
|
976
|
+
const { state } = await userState.read();
|
|
977
|
+
```
|
|
978
|
+
|
|
757
979
|
### Upload validation
|
|
758
980
|
|
|
759
981
|
Source: `packages/server/src/validate.ts`
|