@mostra/runtime 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +2879 -8
- package/dist/bin.js.map +1 -1
- package/dist/builder/child/main.js +2 -2
- package/dist/{child-Chiavhdx.js → child-BmNZ4a70.js} +2 -2
- package/dist/{child-Chiavhdx.js.map → child-BmNZ4a70.js.map} +1 -1
- package/dist/deps/child/install.js +1 -1
- package/dist/{directories-mJ1qlVht.js → directories-BLJjeSPD.js} +5 -5
- package/dist/directories-BLJjeSPD.js.map +1 -0
- package/dist/{files-FygSJIzK.js → files-DGeWuE4I.js} +2 -2
- package/dist/{files-FygSJIzK.js.map → files-DGeWuE4I.js.map} +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{instance-EnaZPs14.js → instance-FExcyENK.js} +3341 -396
- package/dist/instance-FExcyENK.js.map +1 -0
- package/package.json +7 -5
- package/src/actions/index.ts +163 -10
- package/src/bin.ts +78 -1
- package/src/gateway/actions.ts +20 -4
- package/src/gateway/browser.ts +118 -95
- package/src/gateway/capabilities.ts +15 -6
- package/src/gateway/index.ts +112 -15
- package/src/gateway/routes.ts +15 -7
- package/src/gc/index.ts +14 -0
- package/src/instance/index.ts +41 -2
- package/src/interaction/ephemeral.ts +32 -23
- package/src/interaction/events.ts +21 -1
- package/src/interaction/imports.ts +317 -0
- package/src/interaction/index.ts +172 -19
- package/src/interaction/outbox.ts +9 -0
- package/src/interaction/results.ts +8 -1
- package/src/interaction/runs.ts +52 -2
- package/src/interaction/submissions.ts +17 -1
- package/src/interaction/tally.ts +129 -0
- package/src/ledger/index.ts +104 -9
- package/src/ledger/migrations.ts +183 -0
- package/src/ledger/types.ts +8 -0
- package/src/link/binding.ts +289 -0
- package/src/link/hosted.ts +65 -0
- package/src/link/http.ts +136 -0
- package/src/link/index.ts +8 -0
- package/src/link/manager.ts +1385 -0
- package/src/link/relay.ts +512 -0
- package/src/live/index.ts +112 -2
- package/src/platform/filesystems.ts +78 -0
- package/src/platform/index.ts +1 -0
- package/src/protocol-server/index.ts +18 -1
- package/src/protocol-server/server.ts +38 -7
- package/src/protocol-server/websocket.ts +161 -0
- package/src/publish/bundle.ts +398 -0
- package/src/publish/manifest.ts +9 -0
- package/src/publish/methods.ts +12 -1
- package/src/publish/publisher.ts +127 -0
- package/src/registry/index.ts +35 -14
- package/src/server/api.ts +439 -0
- package/src/server/auth.ts +613 -0
- package/src/server/config.ts +221 -0
- package/src/server/data.ts +369 -0
- package/src/server/directory.ts +141 -0
- package/src/server/hosted.ts +822 -0
- package/src/server/index.ts +368 -0
- package/src/server/relay.ts +1163 -0
- package/src/share/index.ts +110 -6
- package/dist/directories-mJ1qlVht.js.map +0 -1
- package/dist/instance-EnaZPs14.js.map +0 -1
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The remote server's configuration (INTERFACES §18.1): a JSON file (`--config`) and
|
|
3
|
+
* `MOSTRA_SERVER_<KEY>` variables, read before the server serves anything. A configuration it
|
|
4
|
+
* can't use stops it, naming the key (`MOSTRA_REMOTE_CONFIG_INVALID`). Domains are configuration,
|
|
5
|
+
* never code (D137, D165 item 0a): the dossier and this code name only `example` hosts.
|
|
6
|
+
*/
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
import { createDiagnostic, Duration, RpcError } from "@mostra/protocol";
|
|
9
|
+
import * as z from "zod";
|
|
10
|
+
import type { Environment } from "../log.ts";
|
|
11
|
+
|
|
12
|
+
/** A rate: `count` per `per` (a duration such as `1m`, as links' lifetimes are written). */
|
|
13
|
+
const RateSchema: z.ZodType<Rate> = z.object({
|
|
14
|
+
count: z.int().min(1),
|
|
15
|
+
per: Duration,
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
/** A host name a link can use: lower-case labels of letters, digits and hyphens. */
|
|
19
|
+
const HostName: z.ZodString = z
|
|
20
|
+
.string()
|
|
21
|
+
.regex(
|
|
22
|
+
/^(?=.{1,253}$)([a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)(\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*(:\d{1,5})?$/,
|
|
23
|
+
{
|
|
24
|
+
message: "must be a lower-case host name, with a port only for a local test",
|
|
25
|
+
},
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* An origin the server is reached at: `https:`, or `http:` for loopback and `*.localhost` (tests
|
|
30
|
+
* and a self-hoster's first run).
|
|
31
|
+
*/
|
|
32
|
+
const Origin: z.ZodType<string> = z.string().refine(
|
|
33
|
+
(value) => {
|
|
34
|
+
try {
|
|
35
|
+
const url = new URL(value);
|
|
36
|
+
if (url.origin !== value) return false;
|
|
37
|
+
if (url.protocol === "https:") return true;
|
|
38
|
+
return (
|
|
39
|
+
url.protocol === "http:" &&
|
|
40
|
+
(url.hostname === "127.0.0.1" ||
|
|
41
|
+
url.hostname === "localhost" ||
|
|
42
|
+
url.hostname.endsWith(".localhost") ||
|
|
43
|
+
url.hostname === "[::1]")
|
|
44
|
+
);
|
|
45
|
+
} catch {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
{ message: "must be an https: origin (http: only on loopback), with no path" },
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
/** A rate limit: `count` per `per`. */
|
|
53
|
+
export interface Rate {
|
|
54
|
+
readonly count: number;
|
|
55
|
+
readonly per: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** An identity provider sign-in goes through (§18.5); WP-30 adds `oidc` and `github`. */
|
|
59
|
+
export interface IdentityProviderConfig {
|
|
60
|
+
readonly id: string;
|
|
61
|
+
readonly kind: "development" | "oidc" | "github";
|
|
62
|
+
readonly label?: string | undefined;
|
|
63
|
+
readonly issuer?: string | undefined;
|
|
64
|
+
readonly clientId?: string | undefined;
|
|
65
|
+
readonly clientSecretFile?: string | undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The configuration a remote server runs with (INTERFACES §18.1). */
|
|
69
|
+
export interface RemoteServerConfig {
|
|
70
|
+
readonly apiOrigin: string;
|
|
71
|
+
readonly domains: { readonly active: string; readonly spares?: string[] | undefined };
|
|
72
|
+
readonly scheme?: "https" | "http" | undefined;
|
|
73
|
+
readonly trustedProxies?: string[] | undefined;
|
|
74
|
+
readonly identityProviders?: IdentityProviderConfig[] | undefined;
|
|
75
|
+
readonly rateLimits?:
|
|
76
|
+
| {
|
|
77
|
+
readonly perIp?:
|
|
78
|
+
| { readonly submissions?: Rate | undefined; readonly requests?: Rate | undefined }
|
|
79
|
+
| undefined;
|
|
80
|
+
readonly perLink?:
|
|
81
|
+
| { readonly submissions?: Rate | undefined; readonly relayedRuns?: Rate | undefined }
|
|
82
|
+
| undefined;
|
|
83
|
+
readonly ipv6Prefix?: number | undefined;
|
|
84
|
+
}
|
|
85
|
+
| undefined;
|
|
86
|
+
readonly ledger?: { readonly walAutocheckpoint?: number | undefined } | undefined;
|
|
87
|
+
readonly relay?:
|
|
88
|
+
| {
|
|
89
|
+
readonly waitingRunsPerArtifact?: number | undefined;
|
|
90
|
+
readonly drainPerSecond?: number | undefined;
|
|
91
|
+
}
|
|
92
|
+
| undefined;
|
|
93
|
+
readonly homes?: { readonly idleCloseMinutes?: number | undefined } | undefined;
|
|
94
|
+
readonly framing?: { readonly frameAncestors: "'none'" } | undefined;
|
|
95
|
+
readonly operator?: string | undefined;
|
|
96
|
+
readonly chatHosts?: Record<string, unknown> | undefined;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The schema of {@link RemoteServerConfig}. */
|
|
100
|
+
const Config: z.ZodType<RemoteServerConfig> & { readonly shape: Record<string, unknown> } =
|
|
101
|
+
z.strictObject({
|
|
102
|
+
apiOrigin: Origin,
|
|
103
|
+
domains: z.strictObject({
|
|
104
|
+
active: HostName,
|
|
105
|
+
spares: z.array(HostName).optional(),
|
|
106
|
+
}),
|
|
107
|
+
/**
|
|
108
|
+
* The scheme the user-content domain is served on: `https` behind a TLS terminator, `http`
|
|
109
|
+
* for a loopback test. Default `https`.
|
|
110
|
+
*/
|
|
111
|
+
scheme: z.enum(["https", "http"]).optional(),
|
|
112
|
+
trustedProxies: z.array(z.string()).optional(),
|
|
113
|
+
identityProviders: z
|
|
114
|
+
.array(
|
|
115
|
+
z.strictObject({
|
|
116
|
+
id: z.string().regex(/^[a-z][a-z0-9-]{0,31}$/),
|
|
117
|
+
kind: z.enum(["development", "oidc", "github"]),
|
|
118
|
+
label: z.string().min(1).optional(),
|
|
119
|
+
issuer: z.string().optional(),
|
|
120
|
+
clientId: z.string().optional(),
|
|
121
|
+
clientSecretFile: z.string().optional(),
|
|
122
|
+
}),
|
|
123
|
+
)
|
|
124
|
+
.optional(),
|
|
125
|
+
rateLimits: z
|
|
126
|
+
.strictObject({
|
|
127
|
+
perIp: z
|
|
128
|
+
.strictObject({ submissions: RateSchema.optional(), requests: RateSchema.optional() })
|
|
129
|
+
.optional(),
|
|
130
|
+
perLink: z
|
|
131
|
+
.strictObject({ submissions: RateSchema.optional(), relayedRuns: RateSchema.optional() })
|
|
132
|
+
.optional(),
|
|
133
|
+
ipv6Prefix: z.int().min(1).max(128).optional(),
|
|
134
|
+
})
|
|
135
|
+
.optional(),
|
|
136
|
+
ledger: z.strictObject({ walAutocheckpoint: z.int().min(0).optional() }).optional(),
|
|
137
|
+
relay: z
|
|
138
|
+
.strictObject({
|
|
139
|
+
waitingRunsPerArtifact: z.int().min(1).optional(),
|
|
140
|
+
drainPerSecond: z.number().positive().optional(),
|
|
141
|
+
})
|
|
142
|
+
.optional(),
|
|
143
|
+
homes: z.strictObject({ idleCloseMinutes: z.number().positive().optional() }).optional(),
|
|
144
|
+
framing: z.strictObject({ frameAncestors: z.literal("'none'") }).optional(),
|
|
145
|
+
operator: z.string().optional(),
|
|
146
|
+
chatHosts: z.record(z.string(), z.unknown()).optional(),
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
/** The configuration keys, and the variable each also comes from. */
|
|
150
|
+
const KEYS = Object.keys(Config.shape) as (keyof RemoteServerConfig)[];
|
|
151
|
+
|
|
152
|
+
/** `apiOrigin` → `MOSTRA_SERVER_API_ORIGIN`. */
|
|
153
|
+
function variableOf(key: string): string {
|
|
154
|
+
return `MOSTRA_SERVER_${key.replace(/[A-Z]/g, (letter) => `_${letter}`).toUpperCase()}`;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function invalid(message: string): RpcError {
|
|
158
|
+
return new RpcError(createDiagnostic("MOSTRA_REMOTE_CONFIG_INVALID", { message }));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Reads the configuration: the file, then each `MOSTRA_SERVER_<KEY>` (a string for a string key,
|
|
163
|
+
* JSON for the others) over it. Throws `MOSTRA_REMOTE_CONFIG_INVALID` naming the first key it
|
|
164
|
+
* can't use.
|
|
165
|
+
*/
|
|
166
|
+
export function readServerConfig(file: string | undefined, env: Environment): RemoteServerConfig {
|
|
167
|
+
let raw: Record<string, unknown> = {};
|
|
168
|
+
if (file !== undefined) {
|
|
169
|
+
let text: string;
|
|
170
|
+
try {
|
|
171
|
+
text = readFileSync(file, "utf8");
|
|
172
|
+
} catch (error) {
|
|
173
|
+
throw invalid(
|
|
174
|
+
`The configuration file ${file} can't be read (${(error as NodeJS.ErrnoException).code ?? "error"}).`,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
try {
|
|
178
|
+
const parsed = JSON.parse(text) as unknown;
|
|
179
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
180
|
+
throw new Error("not an object");
|
|
181
|
+
}
|
|
182
|
+
raw = parsed as Record<string, unknown>;
|
|
183
|
+
} catch {
|
|
184
|
+
throw invalid(`The configuration file ${file} isn't a JSON object.`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
for (const key of KEYS) {
|
|
188
|
+
const variable = variableOf(key);
|
|
189
|
+
const value = env[variable];
|
|
190
|
+
if (value === undefined || value === "") continue;
|
|
191
|
+
const stringKey = key === "apiOrigin" || key === "operator" || key === "scheme";
|
|
192
|
+
if (stringKey) raw[key] = value;
|
|
193
|
+
else {
|
|
194
|
+
try {
|
|
195
|
+
raw[key] = JSON.parse(value) as unknown;
|
|
196
|
+
} catch {
|
|
197
|
+
throw invalid(`${variable} isn't JSON.`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const checked = Config.safeParse(raw);
|
|
202
|
+
if (!checked.success) {
|
|
203
|
+
const issue = checked.error.issues[0];
|
|
204
|
+
const path = issue?.path.join(".") || "(configuration)";
|
|
205
|
+
throw invalid(
|
|
206
|
+
`The configuration key \`${path}\` can't be used: ${issue?.message ?? "invalid"}.`,
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
const config = checked.data;
|
|
210
|
+
const domains = [config.domains.active, ...(config.domains.spares ?? [])];
|
|
211
|
+
const api = new URL(config.apiOrigin).host;
|
|
212
|
+
if (domains.some((domain) => domain === api || api.endsWith(`.${domain}`))) {
|
|
213
|
+
throw invalid(
|
|
214
|
+
"The configuration key `apiOrigin` can't be on a user-content domain: sign-in and the binding live apart from tenant pages (D165 item 7).",
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
if (new Set(domains).size !== domains.length) {
|
|
218
|
+
throw invalid("The configuration key `domains` names one domain twice.");
|
|
219
|
+
}
|
|
220
|
+
return config;
|
|
221
|
+
}
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A remote server's data directory (DATA_MODEL §7.1) and its hosted homes (§7.2, D165 items 0b,
|
|
3
|
+
* 6): the directory is refused on a network filesystem, held by `server.lock` so a second server
|
|
4
|
+
* refuses it, and its directory ledger `server.sqlite` opens in WAL with normal locking. Each
|
|
5
|
+
* hosted home opens on its first request (its own `home.lock`, its ledger in normal locking, so a
|
|
6
|
+
* backup replicator can read it) and closes after it has been idle a while. Opening and closing one
|
|
7
|
+
* home are serialized, so a home never has two writers in one process, and a request that meets a
|
|
8
|
+
* home being closed waits for the close and opens it again (FAILURE_MODEL §11.2).
|
|
9
|
+
*/
|
|
10
|
+
import { randomBytes } from "node:crypto";
|
|
11
|
+
import { mkdirSync } from "node:fs";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { createDiagnostic, RpcError } from "@mostra/protocol";
|
|
14
|
+
import {
|
|
15
|
+
type HeldLock,
|
|
16
|
+
holdLock,
|
|
17
|
+
type Ledger,
|
|
18
|
+
type LedgerOwner,
|
|
19
|
+
openLedger,
|
|
20
|
+
} from "../ledger/index.ts";
|
|
21
|
+
import type { Environment, Log } from "../log.ts";
|
|
22
|
+
import { networkFilesystem, ownedDirectory } from "../platform/index.ts";
|
|
23
|
+
import { SERVER_MIGRATIONS } from "./directory.ts";
|
|
24
|
+
|
|
25
|
+
/** An open hosted home: its ledger, its lock, and whatever the server assembled over them. */
|
|
26
|
+
export interface OpenedHome<T> {
|
|
27
|
+
readonly id: string;
|
|
28
|
+
readonly dir: string;
|
|
29
|
+
readonly ledger: Ledger;
|
|
30
|
+
readonly incarnation: string;
|
|
31
|
+
/** What the server built over the ledger (interaction, gateway routes, sessions). */
|
|
32
|
+
readonly assembled: T;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** What opens and closes the parts of a hosted home that live above its ledger. */
|
|
36
|
+
export interface HomeAssembler<T> {
|
|
37
|
+
/** Builds a home's parts over its ledger, once it is open. */
|
|
38
|
+
assemble(home: {
|
|
39
|
+
readonly id: string;
|
|
40
|
+
readonly dir: string;
|
|
41
|
+
readonly ledger: Ledger;
|
|
42
|
+
readonly incarnation: string;
|
|
43
|
+
}): Promise<T>;
|
|
44
|
+
/** Stops them, before the ledger closes. */
|
|
45
|
+
disassemble(assembled: T): Promise<void>;
|
|
46
|
+
/** Whether a home is in use now: a connection, a stream, a poll, a pending delivery, a waiting run. */
|
|
47
|
+
busy(assembled: T): boolean;
|
|
48
|
+
/** The home's incarnation now: a hosted home rotates it when its owner shows it was restored. */
|
|
49
|
+
incarnation?(assembled: T): string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The data directory, held. */
|
|
53
|
+
export interface DataDirectory<T> {
|
|
54
|
+
readonly dir: string;
|
|
55
|
+
readonly directory: Ledger;
|
|
56
|
+
/** Opens a hosted home, or returns it when open; a request waits for a close in progress. */
|
|
57
|
+
home(id: string): Promise<OpenedHome<T>>;
|
|
58
|
+
/** The homes open now. */
|
|
59
|
+
open(): readonly string[];
|
|
60
|
+
/** Notes that a home was used now, so it isn't closed while used. */
|
|
61
|
+
touch(id: string): void;
|
|
62
|
+
/** Closes every home, then the directory, then releases the lock. */
|
|
63
|
+
close(): Promise<void>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Options for {@link openDataDirectory}. */
|
|
67
|
+
export interface DataDirectoryOptions<T> {
|
|
68
|
+
readonly dir: string;
|
|
69
|
+
readonly runtimeVersion: string;
|
|
70
|
+
readonly owner: LedgerOwner;
|
|
71
|
+
readonly log: Log;
|
|
72
|
+
readonly env: Environment;
|
|
73
|
+
readonly assembler: HomeAssembler<T>;
|
|
74
|
+
/** Close a home this long after it was last used and nothing is busy. Default 10 minutes. */
|
|
75
|
+
readonly idleCloseMs?: number;
|
|
76
|
+
readonly walAutocheckpoint?: number;
|
|
77
|
+
/**
|
|
78
|
+
* Homes restored from a backup by hand (`--restored <homeId|all>`): each rotates its incarnation
|
|
79
|
+
* when it first opens (INTERFACES §18.2).
|
|
80
|
+
*/
|
|
81
|
+
readonly restored?: readonly string[];
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** A home's head and relay high-water mark, as `home_cursor` records them (DATA_MODEL §7.1). */
|
|
85
|
+
function marksOf(ledger: Ledger): { readonly head: number; readonly relayHigh: number } {
|
|
86
|
+
return ledger.read((db) => ({
|
|
87
|
+
head: db.get<{ head: number | null }>("SELECT MAX(cursor) AS head FROM event")?.head ?? 0,
|
|
88
|
+
relayHigh:
|
|
89
|
+
db.get<{ high: number | null }>("SELECT MAX(relay_seq) AS high FROM relay_run")?.high ?? 0,
|
|
90
|
+
}));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** A fresh incarnation, written in the one transaction that rotates it (DATA_MODEL §7.2). */
|
|
94
|
+
export function rotateIncarnation(ledger: Ledger): string {
|
|
95
|
+
const fresh = randomBytes(16).toString("hex");
|
|
96
|
+
ledger.write((db) => db.run("UPDATE home_meta SET incarnation = ? WHERE id = 1", fresh));
|
|
97
|
+
return fresh;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function locked(message: string): RpcError {
|
|
101
|
+
return new RpcError(createDiagnostic("MOSTRA_HOME_LOCKED", { message }));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Refuses a data directory on a network filesystem (`MOSTRA_HOME_NETWORK_FILESYSTEM`), before
|
|
106
|
+
* anything in it is opened. `MOSTRA_TEST_NETWORK_FILESYSTEM` names a directory tests treat as one,
|
|
107
|
+
* since no test machine mounts NFS.
|
|
108
|
+
*/
|
|
109
|
+
function refuseNetworkFilesystem(dir: string, env: Environment): void {
|
|
110
|
+
const testing = env["MOSTRA_TEST_NETWORK_FILESYSTEM"];
|
|
111
|
+
const found =
|
|
112
|
+
testing !== undefined && testing !== "" && dir.startsWith(testing)
|
|
113
|
+
? "nfs (test)"
|
|
114
|
+
: networkFilesystem(dir);
|
|
115
|
+
if (found !== undefined) {
|
|
116
|
+
throw new RpcError(
|
|
117
|
+
createDiagnostic("MOSTRA_HOME_NETWORK_FILESYSTEM", {
|
|
118
|
+
message: `The data directory ${dir} is on a network filesystem (${found}); a remote server's homes need a local disk.`,
|
|
119
|
+
}),
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Opens and holds a server's data directory. Throws `MOSTRA_HOME_LOCKED` when another server holds it. */
|
|
125
|
+
export async function openDataDirectory<T>(
|
|
126
|
+
options: DataDirectoryOptions<T>,
|
|
127
|
+
): Promise<DataDirectory<T>> {
|
|
128
|
+
const { dir, log } = options;
|
|
129
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
130
|
+
ownedDirectory(dir);
|
|
131
|
+
refuseNetworkFilesystem(dir, options.env);
|
|
132
|
+
const serverLock = holdLock(join(dir, "server.lock"), options.owner);
|
|
133
|
+
if (!serverLock) {
|
|
134
|
+
throw locked(`Another server process holds the data directory ${dir} (server.lock).`);
|
|
135
|
+
}
|
|
136
|
+
let directory: Ledger;
|
|
137
|
+
try {
|
|
138
|
+
const opening = openLedger({
|
|
139
|
+
storeDir: dir,
|
|
140
|
+
fileName: "server.sqlite",
|
|
141
|
+
runtimeVersion: options.runtimeVersion,
|
|
142
|
+
owner: options.owner,
|
|
143
|
+
log,
|
|
144
|
+
migrations: SERVER_MIGRATIONS,
|
|
145
|
+
locking: "normal",
|
|
146
|
+
...(options.walAutocheckpoint !== undefined
|
|
147
|
+
? { walAutocheckpoint: options.walAutocheckpoint }
|
|
148
|
+
: {}),
|
|
149
|
+
});
|
|
150
|
+
if (!opening.ok) throw locked(`server.sqlite in ${dir} is locked by another process.`);
|
|
151
|
+
directory = opening.ledger;
|
|
152
|
+
} catch (error) {
|
|
153
|
+
serverLock.release();
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
mkdirSync(join(dir, "homes"), { recursive: true, mode: 0o700 });
|
|
157
|
+
mkdirSync(join(dir, "files"), { recursive: true, mode: 0o700 });
|
|
158
|
+
mkdirSync(join(dir, "keys"), { recursive: true, mode: 0o700 });
|
|
159
|
+
|
|
160
|
+
interface Entry {
|
|
161
|
+
opening: Promise<OpenedHome<T>>;
|
|
162
|
+
lock?: HeldLock;
|
|
163
|
+
ledger?: Ledger;
|
|
164
|
+
incarnation?: () => string;
|
|
165
|
+
lastUsed: number;
|
|
166
|
+
closing?: Promise<void>;
|
|
167
|
+
}
|
|
168
|
+
const entries = new Map<string, Entry>();
|
|
169
|
+
/** `--restored`'s homes not opened since this server started. */
|
|
170
|
+
const restores = new Set(options.restored ?? []);
|
|
171
|
+
|
|
172
|
+
/** Writes a home's high-water marks, which a home opened below them was restored. */
|
|
173
|
+
const recordCursor = (id: string, ledger: Ledger, incarnation: string): void => {
|
|
174
|
+
const { head, relayHigh } = marksOf(ledger);
|
|
175
|
+
directory.write((db) =>
|
|
176
|
+
db.run(
|
|
177
|
+
`INSERT INTO home_cursor (home_id, incarnation, high_cursor, relay_high, updated_at)
|
|
178
|
+
VALUES (?, ?, ?, ?, ?)
|
|
179
|
+
ON CONFLICT(home_id) DO UPDATE SET incarnation = excluded.incarnation,
|
|
180
|
+
high_cursor = excluded.high_cursor, relay_high = excluded.relay_high,
|
|
181
|
+
updated_at = excluded.updated_at`,
|
|
182
|
+
id,
|
|
183
|
+
incarnation,
|
|
184
|
+
head,
|
|
185
|
+
relayHigh,
|
|
186
|
+
new Date().toISOString(),
|
|
187
|
+
),
|
|
188
|
+
);
|
|
189
|
+
};
|
|
190
|
+
const idleCloseMs = options.idleCloseMs ?? 10 * 60_000;
|
|
191
|
+
let closed = false;
|
|
192
|
+
|
|
193
|
+
const openHome = async (id: string): Promise<OpenedHome<T>> => {
|
|
194
|
+
const row = directory.read((db) =>
|
|
195
|
+
db.get<{ data_path: string }>("SELECT data_path FROM home WHERE id = ?", id),
|
|
196
|
+
);
|
|
197
|
+
if (!row) {
|
|
198
|
+
throw new RpcError(
|
|
199
|
+
createDiagnostic("MOSTRA_HOST_INVALID_PARAMS", { message: `No hosted home ${id}.` }),
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
const homeDir = join(dir, row.data_path);
|
|
203
|
+
mkdirSync(homeDir, { recursive: true, mode: 0o700 });
|
|
204
|
+
const lock = holdLock(join(homeDir, "home.lock"), options.owner);
|
|
205
|
+
if (!lock) throw locked(`Another process holds the hosted home ${id} (home.lock).`);
|
|
206
|
+
try {
|
|
207
|
+
const opening = openLedger({
|
|
208
|
+
storeDir: join(homeDir, "store"),
|
|
209
|
+
runtimeVersion: options.runtimeVersion,
|
|
210
|
+
owner: options.owner,
|
|
211
|
+
log,
|
|
212
|
+
locking: "normal",
|
|
213
|
+
...(options.walAutocheckpoint !== undefined
|
|
214
|
+
? { walAutocheckpoint: options.walAutocheckpoint }
|
|
215
|
+
: {}),
|
|
216
|
+
});
|
|
217
|
+
if (!opening.ok) throw locked(`The hosted home ${id}'s ledger is locked by another process.`);
|
|
218
|
+
const ledger = opening.ledger;
|
|
219
|
+
const stored = ledger.write((db) => {
|
|
220
|
+
const meta = db.get<{ incarnation: string }>(
|
|
221
|
+
"SELECT incarnation FROM home_meta WHERE id = 1",
|
|
222
|
+
);
|
|
223
|
+
if (meta) return meta.incarnation;
|
|
224
|
+
const fresh = randomBytes(16).toString("hex");
|
|
225
|
+
db.run(
|
|
226
|
+
"INSERT INTO home_meta (id, incarnation, created_at) VALUES (1, ?, ?)",
|
|
227
|
+
fresh,
|
|
228
|
+
new Date().toISOString(),
|
|
229
|
+
);
|
|
230
|
+
return fresh;
|
|
231
|
+
});
|
|
232
|
+
// A home restored from a backup reuses cursors its owner may have copied: it rotates its
|
|
233
|
+
// incarnation before it answers anyone, when restored by hand or when it opens below the
|
|
234
|
+
// marks it reached (INTERFACES §18.2, FAILURE_MODEL §11.2).
|
|
235
|
+
const recorded = directory.read((db) =>
|
|
236
|
+
db.get<{ high_cursor: number; relay_high: number }>(
|
|
237
|
+
"SELECT high_cursor, relay_high FROM home_cursor WHERE home_id = ?",
|
|
238
|
+
id,
|
|
239
|
+
),
|
|
240
|
+
);
|
|
241
|
+
const marks = marksOf(ledger);
|
|
242
|
+
const byHand = restores.has(id) || restores.has("all");
|
|
243
|
+
restores.delete(id);
|
|
244
|
+
const below =
|
|
245
|
+
recorded !== undefined &&
|
|
246
|
+
(marks.head < recorded.high_cursor || marks.relayHigh < recorded.relay_high);
|
|
247
|
+
const incarnation = byHand || below ? rotateIncarnation(ledger) : stored;
|
|
248
|
+
if (incarnation !== stored) {
|
|
249
|
+
log.warn("server.home_rotated", { home: id, why: byHand ? "restored" : "below-mark" });
|
|
250
|
+
}
|
|
251
|
+
recordCursor(id, ledger, incarnation);
|
|
252
|
+
try {
|
|
253
|
+
const assembled = await options.assembler.assemble({
|
|
254
|
+
id,
|
|
255
|
+
dir: homeDir,
|
|
256
|
+
ledger,
|
|
257
|
+
incarnation,
|
|
258
|
+
});
|
|
259
|
+
const entry = entries.get(id);
|
|
260
|
+
if (entry) {
|
|
261
|
+
entry.lock = lock;
|
|
262
|
+
entry.ledger = ledger;
|
|
263
|
+
entry.incarnation = () => options.assembler.incarnation?.(assembled) ?? incarnation;
|
|
264
|
+
}
|
|
265
|
+
log.info("server.home_opened", { home: id });
|
|
266
|
+
return { id, dir: homeDir, ledger, incarnation, assembled };
|
|
267
|
+
} catch (error) {
|
|
268
|
+
ledger.close();
|
|
269
|
+
throw error;
|
|
270
|
+
}
|
|
271
|
+
} catch (error) {
|
|
272
|
+
lock.release();
|
|
273
|
+
throw error;
|
|
274
|
+
}
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
const closeHome = async (id: string, entry: Entry): Promise<void> => {
|
|
278
|
+
const opened = await entry.opening.catch(() => undefined);
|
|
279
|
+
if (opened) {
|
|
280
|
+
try {
|
|
281
|
+
recordCursor(id, opened.ledger, entry.incarnation?.() ?? opened.incarnation);
|
|
282
|
+
} catch (error) {
|
|
283
|
+
log.warn("server.home_cursor_failed", { home: id, error });
|
|
284
|
+
}
|
|
285
|
+
await options.assembler
|
|
286
|
+
.disassemble(opened.assembled)
|
|
287
|
+
.catch((error: unknown) => log.warn("server.home_disassemble_failed", { home: id, error }));
|
|
288
|
+
opened.ledger.close();
|
|
289
|
+
}
|
|
290
|
+
entry.lock?.release();
|
|
291
|
+
entries.delete(id);
|
|
292
|
+
log.info("server.home_closed", { home: id });
|
|
293
|
+
};
|
|
294
|
+
|
|
295
|
+
const sweep = setInterval(
|
|
296
|
+
() => {
|
|
297
|
+
const now = Date.now();
|
|
298
|
+
for (const [id, entry] of entries) {
|
|
299
|
+
if (entry.closing || now - entry.lastUsed < idleCloseMs) continue;
|
|
300
|
+
void entry.opening.then(
|
|
301
|
+
(opened) => {
|
|
302
|
+
if (options.assembler.busy(opened.assembled) || entry.closing) return;
|
|
303
|
+
if (Date.now() - entry.lastUsed < idleCloseMs) return;
|
|
304
|
+
entry.closing = closeHome(id, entry);
|
|
305
|
+
},
|
|
306
|
+
() => {
|
|
307
|
+
entries.delete(id);
|
|
308
|
+
},
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
},
|
|
312
|
+
Math.min(30_000, Math.max(50, idleCloseMs / 4)),
|
|
313
|
+
);
|
|
314
|
+
sweep.unref();
|
|
315
|
+
// At least every 10 s while a home is open (DATA_MODEL §7.1).
|
|
316
|
+
const marker = setInterval(() => {
|
|
317
|
+
for (const [id, entry] of entries) {
|
|
318
|
+
if (!entry.ledger || entry.closing) continue;
|
|
319
|
+
try {
|
|
320
|
+
recordCursor(id, entry.ledger, entry.incarnation?.() ?? "");
|
|
321
|
+
} catch (error) {
|
|
322
|
+
log.warn("server.home_cursor_failed", { home: id, error });
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}, 10_000);
|
|
326
|
+
marker.unref();
|
|
327
|
+
|
|
328
|
+
return {
|
|
329
|
+
dir,
|
|
330
|
+
directory,
|
|
331
|
+
async home(id) {
|
|
332
|
+
if (closed) throw locked("The server is stopping.");
|
|
333
|
+
for (;;) {
|
|
334
|
+
const entry = entries.get(id);
|
|
335
|
+
if (entry?.closing) {
|
|
336
|
+
await entry.closing;
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
if (entry) {
|
|
340
|
+
entry.lastUsed = Date.now();
|
|
341
|
+
return entry.opening;
|
|
342
|
+
}
|
|
343
|
+
const fresh: Entry = { opening: openHome(id), lastUsed: Date.now() };
|
|
344
|
+
entries.set(id, fresh);
|
|
345
|
+
fresh.opening.catch(() => {
|
|
346
|
+
if (entries.get(id) === fresh) entries.delete(id);
|
|
347
|
+
});
|
|
348
|
+
return fresh.opening;
|
|
349
|
+
}
|
|
350
|
+
},
|
|
351
|
+
open: () => [...entries.keys()],
|
|
352
|
+
touch(id) {
|
|
353
|
+
const entry = entries.get(id);
|
|
354
|
+
if (entry) entry.lastUsed = Date.now();
|
|
355
|
+
},
|
|
356
|
+
async close() {
|
|
357
|
+
if (closed) return;
|
|
358
|
+
closed = true;
|
|
359
|
+
clearInterval(sweep);
|
|
360
|
+
clearInterval(marker);
|
|
361
|
+
for (const [id, entry] of [...entries]) {
|
|
362
|
+
entry.closing ??= closeHome(id, entry);
|
|
363
|
+
await entry.closing;
|
|
364
|
+
}
|
|
365
|
+
directory.close();
|
|
366
|
+
serverLock.release();
|
|
367
|
+
},
|
|
368
|
+
};
|
|
369
|
+
}
|