@parcha/pi-durable-disk 0.0.0-stage → 0.1.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +329 -0
- package/LICENSE +202 -0
- package/NOTICE +2 -0
- package/README.md +695 -2
- package/bin/archil-scoped +165 -0
- package/dist/app.d.ts +40 -0
- package/dist/app.js +47 -0
- package/dist/claim.d.ts +222 -0
- package/dist/claim.js +722 -0
- package/dist/cli.d.ts +64 -0
- package/dist/cli.js +664 -0
- package/dist/commands.d.ts +67 -0
- package/dist/commands.js +299 -0
- package/dist/confine.d.ts +91 -0
- package/dist/confine.js +341 -0
- package/dist/env.d.ts +129 -0
- package/dist/env.js +608 -0
- package/dist/errors.d.ts +47 -0
- package/dist/errors.js +55 -0
- package/dist/fork.d.ts +41 -0
- package/dist/fork.js +181 -0
- package/dist/hosts/daytona-launch.d.ts +52 -0
- package/dist/hosts/daytona-launch.js +270 -0
- package/dist/hosts/daytona.d.ts +147 -0
- package/dist/hosts/daytona.js +368 -0
- package/dist/hosts/docker.d.ts +103 -0
- package/dist/hosts/docker.js +362 -0
- package/dist/hosts/local-host.d.ts +71 -0
- package/dist/hosts/local-host.js +356 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +17 -0
- package/dist/lease.d.ts +7 -0
- package/dist/lease.js +6 -0
- package/dist/local-run.d.ts +10 -0
- package/dist/local-run.js +63 -0
- package/dist/park.d.ts +92 -0
- package/dist/park.js +229 -0
- package/dist/run.d.ts +290 -0
- package/dist/run.js +785 -0
- package/dist/serve.d.ts +101 -0
- package/dist/serve.js +393 -0
- package/dist/status.d.ts +123 -0
- package/dist/status.js +264 -0
- package/dist/store.d.ts +103 -0
- package/dist/store.js +217 -0
- package/dist/supervise.d.ts +340 -0
- package/dist/supervise.js +528 -0
- package/dist/watchdog.d.ts +39 -0
- package/dist/watchdog.js +137 -0
- package/package.json +87 -4
package/dist/status.js
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
// run.json: the run's status, generation, holder, heartbeat and seal, written by the instance that holds the
|
|
2
|
+
// claim with write-to-temp, fsync, rename, and read by the supervisor (over the S3 API or a local mount). It is a hint
|
|
3
|
+
// for the supervisor, never the source of truth: the store is. Any error writing it is a fence. The instance writes it
|
|
4
|
+
// through its claim directory (`ClaimDir`), never by path, so a write cannot land under a mountpoint whose mount is gone.
|
|
5
|
+
import { constants, readFileSync } from "node:fs";
|
|
6
|
+
import { open, readFile, rename, rm, stat } from "node:fs/promises";
|
|
7
|
+
import { hostname } from "node:os";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import { FencedError, ownWriteFenced, PdaError } from "./errors.js";
|
|
10
|
+
export const RUN_JSON = "run.json";
|
|
11
|
+
/** The temporary file every write goes through; renamed over `run.json` once it is fsynced. */
|
|
12
|
+
export const RUN_JSON_TEMP = "run.json.tmp";
|
|
13
|
+
/** Heartbeat period: the instance rewrites `heartbeatAt` this often. */
|
|
14
|
+
export const DEFAULT_HEARTBEAT_MS = 20_000;
|
|
15
|
+
/** A lease whose last heartbeat is older than this has expired; the supervisor may revoke. */
|
|
16
|
+
export const DEFAULT_LEASE_EXPIRY_MS = 90_000;
|
|
17
|
+
/** The instance fences itself this long before its lease expires, on its own monotonic clock. */
|
|
18
|
+
export const DEFAULT_LEASE_MARGIN_MS = 15_000;
|
|
19
|
+
export const RUN_STATUSES = ["running", "sleeping", "paused", "done", "failed"];
|
|
20
|
+
/** `run.json` is missing a field, has one of the wrong type, or is not JSON. Exit 65 (EX_DATAERR): a restart cannot fix it. */
|
|
21
|
+
export class RunRecordError extends PdaError {
|
|
22
|
+
constructor(message, options = {}) {
|
|
23
|
+
super("RUN_JSON_INVALID", message, { cause: options.cause, exitCode: 65 });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const isInstant = (value) => typeof value === "string" && Number.isFinite(Date.parse(value));
|
|
27
|
+
const isCount = (value) => Number.isSafeInteger(value) && value >= 0;
|
|
28
|
+
function holderOf(value) {
|
|
29
|
+
if (value === null)
|
|
30
|
+
return null;
|
|
31
|
+
const h = value;
|
|
32
|
+
if (typeof h !== "object" || h === undefined || Array.isArray(h))
|
|
33
|
+
throw new RunRecordError("holder is not an object or null");
|
|
34
|
+
if (typeof h.driver !== "string" || typeof h.host !== "string")
|
|
35
|
+
throw new RunRecordError("holder.driver and holder.host must be strings");
|
|
36
|
+
if (h.bootId !== null && typeof h.bootId !== "string")
|
|
37
|
+
throw new RunRecordError("holder.bootId must be a string or null");
|
|
38
|
+
if (!Number.isSafeInteger(h.pid) || h.pid <= 0)
|
|
39
|
+
throw new RunRecordError("holder.pid must be a positive integer");
|
|
40
|
+
if (!isInstant(h.since))
|
|
41
|
+
throw new RunRecordError("holder.since must be an ISO 8601 instant");
|
|
42
|
+
return { ...h, driver: h.driver, host: h.host, bootId: h.bootId, pid: h.pid, since: h.since };
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Parse and validate the text of a `run.json`. Unknown fields are ignored, so a newer writer stays readable. Throws
|
|
46
|
+
* RunRecordError on anything else. This is the reader the supervisor uses on the bytes it gets over the S3 API.
|
|
47
|
+
*/
|
|
48
|
+
export function parseRunRecord(text) {
|
|
49
|
+
let value;
|
|
50
|
+
try {
|
|
51
|
+
value = JSON.parse(text);
|
|
52
|
+
}
|
|
53
|
+
catch (cause) {
|
|
54
|
+
throw new RunRecordError("run.json is not JSON", { cause });
|
|
55
|
+
}
|
|
56
|
+
const r = value;
|
|
57
|
+
if (typeof r !== "object" || r === null || Array.isArray(r))
|
|
58
|
+
throw new RunRecordError("run.json is not an object");
|
|
59
|
+
if (typeof r.run !== "string" || r.run === "")
|
|
60
|
+
throw new RunRecordError("run must be a non-empty string");
|
|
61
|
+
if (!RUN_STATUSES.includes(r.status))
|
|
62
|
+
throw new RunRecordError(`status ${JSON.stringify(r.status)} is not one of ${RUN_STATUSES.join(", ")}`);
|
|
63
|
+
if (!isCount(r.generation))
|
|
64
|
+
throw new RunRecordError("generation must be a non-negative integer");
|
|
65
|
+
if (r.sealedSeq !== null && !isCount(r.sealedSeq))
|
|
66
|
+
throw new RunRecordError("sealedSeq must be a non-negative integer or null");
|
|
67
|
+
if (r.wakeAt !== null && !isInstant(r.wakeAt))
|
|
68
|
+
throw new RunRecordError("wakeAt must be an ISO 8601 instant or null");
|
|
69
|
+
if (r.heartbeatAt !== null && !isInstant(r.heartbeatAt))
|
|
70
|
+
throw new RunRecordError("heartbeatAt must be an ISO 8601 instant or null");
|
|
71
|
+
if (!isInstant(r.updatedAt))
|
|
72
|
+
throw new RunRecordError("updatedAt must be an ISO 8601 instant");
|
|
73
|
+
return {
|
|
74
|
+
run: r.run,
|
|
75
|
+
status: r.status,
|
|
76
|
+
generation: r.generation,
|
|
77
|
+
sealedSeq: r.sealedSeq,
|
|
78
|
+
wakeAt: r.wakeAt,
|
|
79
|
+
holder: holderOf(r.holder),
|
|
80
|
+
heartbeatAt: r.heartbeatAt,
|
|
81
|
+
updatedAt: r.updatedAt,
|
|
82
|
+
detail: r.detail === undefined ? null : r.detail,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/** The `run.json` under a run's root on a local mount (or in a claim directory), or undefined when the run has none yet. */
|
|
86
|
+
export async function readRunRecord(root) {
|
|
87
|
+
let text;
|
|
88
|
+
try {
|
|
89
|
+
text = await readFile(join(typeof root === "string" ? root : root.at, RUN_JSON), "utf8");
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
if (error.code === "ENOENT")
|
|
93
|
+
return undefined;
|
|
94
|
+
throw error;
|
|
95
|
+
}
|
|
96
|
+
return parseRunRecord(text);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
|
|
100
|
+
* clock skew between hosts inside the expiry.
|
|
101
|
+
*/
|
|
102
|
+
export function isLeaseFresh(record, nowMs = Date.now(), expiryMs = DEFAULT_LEASE_EXPIRY_MS) {
|
|
103
|
+
if (record.heartbeatAt === null)
|
|
104
|
+
return false;
|
|
105
|
+
return nowMs - Date.parse(record.heartbeatAt) < expiryMs;
|
|
106
|
+
}
|
|
107
|
+
/** Write to `run.json.tmp`, fsync it, rename it over `run.json`, fsync the directory. */
|
|
108
|
+
export const persistRecord = async (root, text, signal) => {
|
|
109
|
+
const temp = join(root, RUN_JSON_TEMP);
|
|
110
|
+
const file = await open(temp, "w", 0o644);
|
|
111
|
+
try {
|
|
112
|
+
await file.writeFile(text);
|
|
113
|
+
await file.sync();
|
|
114
|
+
}
|
|
115
|
+
finally {
|
|
116
|
+
await file.close();
|
|
117
|
+
}
|
|
118
|
+
// The rename is the commit point: a write that began before a fence must not publish after it. Nothing else runs
|
|
119
|
+
// between this check and the rename's dispatch.
|
|
120
|
+
if (signal?.aborted) {
|
|
121
|
+
await rm(temp, { force: true }).catch(() => undefined);
|
|
122
|
+
signal.throwIfAborted();
|
|
123
|
+
}
|
|
124
|
+
await rename(temp, join(root, RUN_JSON));
|
|
125
|
+
const dir = await open(root, "r");
|
|
126
|
+
try {
|
|
127
|
+
await dir.sync();
|
|
128
|
+
}
|
|
129
|
+
finally {
|
|
130
|
+
await dir.close();
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
/**
|
|
134
|
+
* Write `record`; any error at all is a fence (`ownWriteFenced`), whatever its errno. Into a claim directory, the write
|
|
135
|
+
* goes through the open directory, after `assertMounted` (its CLAIM_UNMOUNTED fence is rethrown as is). `signal` as in
|
|
136
|
+
* `PersistRecord`.
|
|
137
|
+
*/
|
|
138
|
+
export async function writeRunRecord(root, record, persist = persistRecord, signal) {
|
|
139
|
+
try {
|
|
140
|
+
if (typeof root !== "string")
|
|
141
|
+
await root.assertMounted();
|
|
142
|
+
await persist(typeof root === "string" ? root : root.at, `${JSON.stringify(record, null, 2)}\n`, signal);
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
if (error instanceof FencedError && error.code === CLAIM_UNMOUNTED)
|
|
146
|
+
throw error;
|
|
147
|
+
throw ownWriteFenced(join(typeof root === "string" ? root : root.root, RUN_JSON), error);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
// ---- the claim directory ----------------------------------------------------------------------------------------------
|
|
151
|
+
/** The FencedError code of a claim root that is no longer the mount the instance opened (unmounted, lazily or not). */
|
|
152
|
+
export const CLAIM_UNMOUNTED = "CLAIM_UNMOUNTED";
|
|
153
|
+
/** The file system type of an Archil mount in the mount table. */
|
|
154
|
+
export const ARCHIL_FSTYPE = "fuse.archil";
|
|
155
|
+
/** `major:minor` of a device number, as `/proc/self/mountinfo` writes it (glibc's encoding). */
|
|
156
|
+
export function deviceName(dev) {
|
|
157
|
+
const major = ((dev & 0xfff00n) >> 8n) | ((dev & 0xfffff00000000000n) >> 32n);
|
|
158
|
+
const minor = (dev & 0xffn) | ((dev & 0xffffff00000n) >> 12n);
|
|
159
|
+
return `${major}:${minor}`;
|
|
160
|
+
}
|
|
161
|
+
const unescapeMount = (s) => s.replace(/\\([0-7]{3})/g, (_, o) => String.fromCharCode(parseInt(o, 8)));
|
|
162
|
+
/** Every mount at `mountpoint` in a mountinfo table: its device and its type. */
|
|
163
|
+
function mountsAt(table, mountpoint) {
|
|
164
|
+
const found = [];
|
|
165
|
+
for (const line of table.split("\n")) {
|
|
166
|
+
const f = line.split(" ");
|
|
167
|
+
const separator = f.indexOf("-", 6);
|
|
168
|
+
if (separator < 0 || f.length < separator + 2)
|
|
169
|
+
continue;
|
|
170
|
+
if (unescapeMount(f[4]) === mountpoint)
|
|
171
|
+
found.push({ dev: f[2], fstype: f[separator + 1] });
|
|
172
|
+
}
|
|
173
|
+
return found;
|
|
174
|
+
}
|
|
175
|
+
const unmounted = (message, cause) => new FencedError(message, { code: CLAIM_UNMOUNTED, cause });
|
|
176
|
+
/**
|
|
177
|
+
* Open `root` as the claim's directory. With an `fstype` (default `fuse.archil`), the mount table must list a mount of
|
|
178
|
+
* that type at `root` on the device of the directory just opened; otherwise, and when the root cannot be opened, a
|
|
179
|
+
* FencedError CLAIM_UNMOUNTED (exit 75).
|
|
180
|
+
*/
|
|
181
|
+
export async function openClaimDir(root, options = {}) {
|
|
182
|
+
const fstype = options.fstype === undefined ? ARCHIL_FSTYPE : options.fstype;
|
|
183
|
+
let handle;
|
|
184
|
+
try {
|
|
185
|
+
handle = await open(root, constants.O_RDONLY | constants.O_DIRECTORY);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
throw unmounted(`opening the claim root ${root} failed: ${error.message}`, error);
|
|
189
|
+
}
|
|
190
|
+
let held;
|
|
191
|
+
try {
|
|
192
|
+
held = await handle.stat({ bigint: true });
|
|
193
|
+
if (fstype !== null) {
|
|
194
|
+
const table = await readFile(options.mountinfo ?? "/proc/self/mountinfo", "utf8");
|
|
195
|
+
const dev = deviceName(held.dev);
|
|
196
|
+
const mounts = mountsAt(table, root);
|
|
197
|
+
if (!mounts.some((m) => m.fstype === fstype && m.dev === dev)) {
|
|
198
|
+
const listed = mounts.map((m) => `${m.fstype} on ${m.dev}`).join(", ") || "nothing";
|
|
199
|
+
throw unmounted(`the claim root ${root} is on device ${dev}, not a ${fstype} mount there (the mount table lists ${listed})`);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
await handle.close().catch(() => undefined);
|
|
205
|
+
throw error instanceof FencedError ? error : unmounted(`checking the claim root ${root} failed: ${error.message}`, error);
|
|
206
|
+
}
|
|
207
|
+
const at = `/proc/self/fd/${handle.fd}`;
|
|
208
|
+
let closed = false;
|
|
209
|
+
const same = (path, now, claims) => {
|
|
210
|
+
if (now.dev === claims.dev && now.ino === claims.ino)
|
|
211
|
+
return;
|
|
212
|
+
throw unmounted(`${path} is device ${deviceName(now.dev)} inode ${now.ino}, not the claim's device ${deviceName(claims.dev)} inode ${claims.ino}: the claim's mount is gone`);
|
|
213
|
+
};
|
|
214
|
+
const statOf = async (path) => {
|
|
215
|
+
try {
|
|
216
|
+
return await stat(path, { bigint: true });
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
throw unmounted(`stat of ${path} failed, so the claim's mount cannot be trusted: ${error.message}`, error);
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
return {
|
|
223
|
+
root,
|
|
224
|
+
get at() {
|
|
225
|
+
if (closed)
|
|
226
|
+
throw new FencedError(`the claim directory of ${root} is closed`);
|
|
227
|
+
return at;
|
|
228
|
+
},
|
|
229
|
+
async assertMounted() {
|
|
230
|
+
if (closed)
|
|
231
|
+
throw unmounted(`the claim directory of ${root} is closed`);
|
|
232
|
+
same(root, await statOf(root), held);
|
|
233
|
+
},
|
|
234
|
+
async assertSame(name) {
|
|
235
|
+
if (closed)
|
|
236
|
+
throw unmounted(`the claim directory of ${root} is closed`);
|
|
237
|
+
const [byPath, inDir] = await Promise.all([statOf(join(root, name)), statOf(join(at, name))]);
|
|
238
|
+
if (inDir.dev !== held.dev)
|
|
239
|
+
throw unmounted(`${name} in the claim directory is on device ${deviceName(inDir.dev)}, not the claim's ${deviceName(held.dev)}`);
|
|
240
|
+
same(join(root, name), byPath, inDir);
|
|
241
|
+
},
|
|
242
|
+
async close() {
|
|
243
|
+
if (closed)
|
|
244
|
+
return;
|
|
245
|
+
closed = true;
|
|
246
|
+
await handle.close();
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
function bootId() {
|
|
251
|
+
try {
|
|
252
|
+
return readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim() || null;
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* This process as a holder: the host driver's `handle` (default driver `local`, this machine's host name and boot id)
|
|
260
|
+
* with this process's pid and `since`. A handle whose named fields have the wrong type is refused (RunRecordError).
|
|
261
|
+
*/
|
|
262
|
+
export function currentHolder(handle = {}, since = new Date()) {
|
|
263
|
+
return holderOf({ driver: "local", host: hostname(), bootId: bootId(), ...handle, pid: process.pid, since: since.toISOString() });
|
|
264
|
+
}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { SqliteStorage } from "@earendil-works/pi-durable/storage/sqlite";
|
|
2
|
+
import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "@earendil-works/pi-durable/storage/sqlite";
|
|
3
|
+
import { FencedError, HeldError } from "./errors.js";
|
|
4
|
+
/** `exclusive`: no `-shm`, one connection. `shared`: WAL with `-shm`, readers on the same mount may connect. */
|
|
5
|
+
export type Profile = "exclusive" | "shared";
|
|
6
|
+
/** Thrown by every call on a fenced handle. `cause` is the storage error that fenced it, `errcode` its SQLite code. */
|
|
7
|
+
export declare class StoreFencedError extends FencedError {
|
|
8
|
+
readonly errcode: number;
|
|
9
|
+
constructor(cause: unknown, errcode: number);
|
|
10
|
+
}
|
|
11
|
+
/** Thrown when another connection holds the database file at open (the exclusive profile's "database is locked"). */
|
|
12
|
+
export declare class StoreBusyError extends HeldError {
|
|
13
|
+
constructor(file: string, profile: Profile, cause: unknown);
|
|
14
|
+
}
|
|
15
|
+
export type PragmaMismatch = {
|
|
16
|
+
readonly pragma: string;
|
|
17
|
+
readonly expected: string | number;
|
|
18
|
+
readonly actual: unknown;
|
|
19
|
+
};
|
|
20
|
+
/** Thrown when the filesystem or SQLite did not accept the profile's pragmas; the store is never opened. */
|
|
21
|
+
export declare class StorePragmaError extends Error {
|
|
22
|
+
readonly code = "STORE_PRAGMA";
|
|
23
|
+
readonly profile: Profile;
|
|
24
|
+
readonly mismatches: readonly PragmaMismatch[];
|
|
25
|
+
constructor(profile: Profile, mismatches: readonly PragmaMismatch[]);
|
|
26
|
+
}
|
|
27
|
+
/** What each profile must read back after it is set. `synchronous` 2 is FULL. */
|
|
28
|
+
export declare const PROFILE_PRAGMAS: {
|
|
29
|
+
readonly exclusive: {
|
|
30
|
+
readonly locking_mode: "exclusive";
|
|
31
|
+
readonly journal_mode: "wal";
|
|
32
|
+
readonly synchronous: 2;
|
|
33
|
+
readonly mmap_size: 0;
|
|
34
|
+
};
|
|
35
|
+
readonly shared: {
|
|
36
|
+
readonly locking_mode: "normal";
|
|
37
|
+
readonly journal_mode: "wal";
|
|
38
|
+
readonly synchronous: 2;
|
|
39
|
+
readonly mmap_size: 0;
|
|
40
|
+
};
|
|
41
|
+
};
|
|
42
|
+
/** Read every pragma of `profile` back and refuse (StorePragmaError) when any differs. */
|
|
43
|
+
export declare function assertPragmas(database: SqliteExecutor, profile: Profile): Promise<void>;
|
|
44
|
+
/**
|
|
45
|
+
* The fenced state of one handle. The first error whose result code is in the I/O, full or not-a-database class marks it
|
|
46
|
+
* fenced, runs `onFenced` once (before the error propagates), and from then on every operation fails with the same
|
|
47
|
+
* StoreFencedError: operations still in flight when it was marked fail too, and their results are discarded.
|
|
48
|
+
*/
|
|
49
|
+
export declare class Fence {
|
|
50
|
+
#private;
|
|
51
|
+
constructor(onFenced?: (error: StoreFencedError) => void);
|
|
52
|
+
get fenced(): boolean;
|
|
53
|
+
/** The error that fenced the handle, once it is fenced. */
|
|
54
|
+
get error(): StoreFencedError | undefined;
|
|
55
|
+
assertLive(): void;
|
|
56
|
+
/** Run one storage operation under the fence. */
|
|
57
|
+
guard<T>(operation: () => Promise<T> | T): Promise<T>;
|
|
58
|
+
}
|
|
59
|
+
declare class FencedExecutor implements SqliteExecutor {
|
|
60
|
+
#private;
|
|
61
|
+
protected readonly fence: Fence;
|
|
62
|
+
constructor(fence: Fence, inner: SqliteExecutor);
|
|
63
|
+
exec(sql: string): Promise<void>;
|
|
64
|
+
run(sql: string, ...params: SqliteValue[]): Promise<void>;
|
|
65
|
+
get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined>;
|
|
66
|
+
all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]>;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* pi's `SqliteDatabase` facade with the fence applied by delegation: statements, the transaction handle's statements
|
|
70
|
+
* and the commit itself all report to the same `Fence`. A fenced handle is abandoned, not closed: `close()` fails at
|
|
71
|
+
* once without touching the database, because closing runs a checkpoint and a fenced mount may never answer. The
|
|
72
|
+
* process is expected to exit.
|
|
73
|
+
*/
|
|
74
|
+
export declare class FencedDatabase extends FencedExecutor implements SqliteDatabase {
|
|
75
|
+
#private;
|
|
76
|
+
constructor(database: SqliteDatabase, fence?: Fence);
|
|
77
|
+
get fenced(): boolean;
|
|
78
|
+
get fencedBy(): StoreFencedError | undefined;
|
|
79
|
+
transaction<T>(callback: (transaction: SqliteExecutor) => Promise<T>): Promise<T>;
|
|
80
|
+
close(): Promise<void>;
|
|
81
|
+
}
|
|
82
|
+
export type OpenOptions = {
|
|
83
|
+
/** Runs once, synchronously, when the handle is fenced and before the failing call rejects. */
|
|
84
|
+
readonly onFenced?: (error: StoreFencedError) => void;
|
|
85
|
+
/** Wraps the raw database inside the fence, for instrumentation and fault injection. */
|
|
86
|
+
readonly decorate?: (database: SqliteDatabase) => SqliteDatabase;
|
|
87
|
+
};
|
|
88
|
+
export type ArchilStore = {
|
|
89
|
+
readonly storage: SqliteStorage;
|
|
90
|
+
/** The fence-aware facade the storage runs on: `database.fenced`, `database.fencedBy`. */
|
|
91
|
+
readonly database: FencedDatabase;
|
|
92
|
+
readonly profile: Profile;
|
|
93
|
+
readonly file: string;
|
|
94
|
+
};
|
|
95
|
+
/**
|
|
96
|
+
* Open `file` as a pi-durable store in `profile` (default `exclusive`). The exclusive profile sets
|
|
97
|
+
* `locking_mode = EXCLUSIVE` before `journal_mode = WAL`, so the WAL index lives in heap and no `-shm` file exists; a
|
|
98
|
+
* second connection gets "database is locked". The shared profile is pi's own WAL opener, with `-shm`. Rejects with
|
|
99
|
+
* StorePragmaError when a pragma does not read back, StoreBusyError when another connection holds the file, and
|
|
100
|
+
* StoreFencedError when the storage is already fenced at open (that handle is abandoned, as in FencedDatabase).
|
|
101
|
+
*/
|
|
102
|
+
export declare function openArchilStore(file: string, profile?: Profile, options?: OpenOptions): Promise<ArchilStore>;
|
|
103
|
+
export {};
|
package/dist/store.js
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
// The store: pi's SqliteStorage over one database file in one of two profiles, behind a fence-aware facade.
|
|
2
|
+
// Both profiles run at synchronous = FULL with mmap off, and every pragma is read back before the store opens. After a
|
|
3
|
+
// storage error of the I/O, full or not-a-database class the handle is fenced: nothing is retried on it, nothing read
|
|
4
|
+
// from it is trusted, and every later call fails at once with StoreFencedError. The facade is the only place the
|
|
5
|
+
// package reads a storage error.
|
|
6
|
+
import { mkdir } from "node:fs/promises";
|
|
7
|
+
import { dirname } from "node:path";
|
|
8
|
+
import { DatabaseSync } from "node:sqlite";
|
|
9
|
+
import { SqliteStorage } from "@earendil-works/pi-durable/storage/sqlite";
|
|
10
|
+
import { NodeSqliteDatabase, openNodeSqliteDatabase } from "@earendil-works/pi-durable/storage/sqlite/node";
|
|
11
|
+
import { FencedError, HeldError } from "./errors.js";
|
|
12
|
+
// SQLite primary result codes; an extended code carries its primary code in the low byte.
|
|
13
|
+
const SQLITE_BUSY = 5;
|
|
14
|
+
const SQLITE_LOCKED = 6;
|
|
15
|
+
const SQLITE_IOERR = 10;
|
|
16
|
+
const SQLITE_FULL = 13;
|
|
17
|
+
const SQLITE_NOTADB = 26;
|
|
18
|
+
const FENCING_CODES = new Set([SQLITE_IOERR, SQLITE_FULL, SQLITE_NOTADB]);
|
|
19
|
+
const BUSY_CODES = new Set([SQLITE_BUSY, SQLITE_LOCKED]);
|
|
20
|
+
const primary = (errcode) => errcode & 0xff;
|
|
21
|
+
/** Every node:sqlite extended result code in an error, its causes and its aggregate members, outermost first. */
|
|
22
|
+
function resultCodes(error, seen = new Set()) {
|
|
23
|
+
if (typeof error !== "object" || error === null || seen.has(error))
|
|
24
|
+
return [];
|
|
25
|
+
seen.add(error);
|
|
26
|
+
const codes = [];
|
|
27
|
+
const { errcode, cause } = error;
|
|
28
|
+
if (typeof errcode === "number")
|
|
29
|
+
codes.push(errcode);
|
|
30
|
+
const members = error instanceof AggregateError ? error.errors : [];
|
|
31
|
+
for (const inner of [...members, cause])
|
|
32
|
+
codes.push(...resultCodes(inner, seen));
|
|
33
|
+
return codes;
|
|
34
|
+
}
|
|
35
|
+
/** Thrown by every call on a fenced handle. `cause` is the storage error that fenced it, `errcode` its SQLite code. */
|
|
36
|
+
export class StoreFencedError extends FencedError {
|
|
37
|
+
errcode;
|
|
38
|
+
constructor(cause, errcode) {
|
|
39
|
+
super(`store fenced by SQLite result code ${errcode}; the handle is unusable`, { cause, code: "STORE_FENCED" });
|
|
40
|
+
this.errcode = errcode;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/** Thrown when another connection holds the database file at open (the exclusive profile's "database is locked"). */
|
|
44
|
+
export class StoreBusyError extends HeldError {
|
|
45
|
+
constructor(file, profile, cause) {
|
|
46
|
+
super("store", `store ${file} is held by another connection (${profile} profile)`, { cause });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/** Thrown when the filesystem or SQLite did not accept the profile's pragmas; the store is never opened. */
|
|
50
|
+
export class StorePragmaError extends Error {
|
|
51
|
+
code = "STORE_PRAGMA";
|
|
52
|
+
profile;
|
|
53
|
+
mismatches;
|
|
54
|
+
constructor(profile, mismatches) {
|
|
55
|
+
const detail = mismatches.map((m) => `${m.pragma} is ${JSON.stringify(m.actual)}, expected ${JSON.stringify(m.expected)}`);
|
|
56
|
+
super(`${profile} profile not accepted: ${detail.join("; ")}`);
|
|
57
|
+
this.name = "StorePragmaError";
|
|
58
|
+
this.profile = profile;
|
|
59
|
+
this.mismatches = mismatches;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** What each profile must read back after it is set. `synchronous` 2 is FULL. */
|
|
63
|
+
export const PROFILE_PRAGMAS = {
|
|
64
|
+
exclusive: { locking_mode: "exclusive", journal_mode: "wal", synchronous: 2, mmap_size: 0 },
|
|
65
|
+
shared: { locking_mode: "normal", journal_mode: "wal", synchronous: 2, mmap_size: 0 },
|
|
66
|
+
};
|
|
67
|
+
/** Read every pragma of `profile` back and refuse (StorePragmaError) when any differs. */
|
|
68
|
+
export async function assertPragmas(database, profile) {
|
|
69
|
+
const mismatches = [];
|
|
70
|
+
for (const [pragma, expected] of Object.entries(PROFILE_PRAGMAS[profile])) {
|
|
71
|
+
const row = await database.get(`PRAGMA ${pragma}`);
|
|
72
|
+
const actual = row === undefined ? undefined : Object.values(row)[0];
|
|
73
|
+
if (actual !== expected)
|
|
74
|
+
mismatches.push({ pragma, expected, actual });
|
|
75
|
+
}
|
|
76
|
+
if (mismatches.length > 0)
|
|
77
|
+
throw new StorePragmaError(profile, mismatches);
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The fenced state of one handle. The first error whose result code is in the I/O, full or not-a-database class marks it
|
|
81
|
+
* fenced, runs `onFenced` once (before the error propagates), and from then on every operation fails with the same
|
|
82
|
+
* StoreFencedError: operations still in flight when it was marked fail too, and their results are discarded.
|
|
83
|
+
*/
|
|
84
|
+
export class Fence {
|
|
85
|
+
#error;
|
|
86
|
+
#onFenced;
|
|
87
|
+
constructor(onFenced) {
|
|
88
|
+
this.#onFenced = onFenced;
|
|
89
|
+
}
|
|
90
|
+
get fenced() {
|
|
91
|
+
return this.#error !== undefined;
|
|
92
|
+
}
|
|
93
|
+
/** The error that fenced the handle, once it is fenced. */
|
|
94
|
+
get error() {
|
|
95
|
+
return this.#error;
|
|
96
|
+
}
|
|
97
|
+
assertLive() {
|
|
98
|
+
if (this.#error !== undefined)
|
|
99
|
+
throw this.#error;
|
|
100
|
+
}
|
|
101
|
+
/** Run one storage operation under the fence. */
|
|
102
|
+
async guard(operation) {
|
|
103
|
+
this.assertLive();
|
|
104
|
+
let result;
|
|
105
|
+
try {
|
|
106
|
+
result = await operation();
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
throw this.#absorb(error);
|
|
110
|
+
}
|
|
111
|
+
this.assertLive();
|
|
112
|
+
return result;
|
|
113
|
+
}
|
|
114
|
+
#absorb(error) {
|
|
115
|
+
if (this.#error !== undefined)
|
|
116
|
+
return this.#error;
|
|
117
|
+
const errcode = resultCodes(error).find((code) => FENCING_CODES.has(primary(code)));
|
|
118
|
+
if (errcode === undefined)
|
|
119
|
+
return error;
|
|
120
|
+
const fenced = new StoreFencedError(error, errcode);
|
|
121
|
+
this.#error = fenced;
|
|
122
|
+
try {
|
|
123
|
+
this.#onFenced?.(fenced);
|
|
124
|
+
}
|
|
125
|
+
catch (listenerError) {
|
|
126
|
+
// A throwing listener must neither hide the fence nor be swallowed: it surfaces as an uncaught exception.
|
|
127
|
+
queueMicrotask(() => {
|
|
128
|
+
throw listenerError;
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
return fenced;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
class FencedExecutor {
|
|
135
|
+
fence;
|
|
136
|
+
#inner;
|
|
137
|
+
constructor(fence, inner) {
|
|
138
|
+
this.fence = fence;
|
|
139
|
+
this.#inner = inner;
|
|
140
|
+
}
|
|
141
|
+
exec(sql) {
|
|
142
|
+
return this.fence.guard(() => this.#inner.exec(sql));
|
|
143
|
+
}
|
|
144
|
+
run(sql, ...params) {
|
|
145
|
+
return this.fence.guard(() => this.#inner.run(sql, ...params));
|
|
146
|
+
}
|
|
147
|
+
get(sql, ...params) {
|
|
148
|
+
return this.fence.guard(() => this.#inner.get(sql, ...params));
|
|
149
|
+
}
|
|
150
|
+
all(sql, ...params) {
|
|
151
|
+
return this.fence.guard(() => this.#inner.all(sql, ...params));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* pi's `SqliteDatabase` facade with the fence applied by delegation: statements, the transaction handle's statements
|
|
156
|
+
* and the commit itself all report to the same `Fence`. A fenced handle is abandoned, not closed: `close()` fails at
|
|
157
|
+
* once without touching the database, because closing runs a checkpoint and a fenced mount may never answer. The
|
|
158
|
+
* process is expected to exit.
|
|
159
|
+
*/
|
|
160
|
+
export class FencedDatabase extends FencedExecutor {
|
|
161
|
+
#database;
|
|
162
|
+
constructor(database, fence = new Fence()) {
|
|
163
|
+
super(fence, database);
|
|
164
|
+
this.#database = database;
|
|
165
|
+
}
|
|
166
|
+
get fenced() {
|
|
167
|
+
return this.fence.fenced;
|
|
168
|
+
}
|
|
169
|
+
get fencedBy() {
|
|
170
|
+
return this.fence.error;
|
|
171
|
+
}
|
|
172
|
+
transaction(callback) {
|
|
173
|
+
return this.fence.guard(() => this.#database.transaction((handle) => callback(new FencedExecutor(this.fence, handle))));
|
|
174
|
+
}
|
|
175
|
+
close() {
|
|
176
|
+
return this.fence.guard(() => this.#database.close());
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Open `file` as a pi-durable store in `profile` (default `exclusive`). The exclusive profile sets
|
|
181
|
+
* `locking_mode = EXCLUSIVE` before `journal_mode = WAL`, so the WAL index lives in heap and no `-shm` file exists; a
|
|
182
|
+
* second connection gets "database is locked". The shared profile is pi's own WAL opener, with `-shm`. Rejects with
|
|
183
|
+
* StorePragmaError when a pragma does not read back, StoreBusyError when another connection holds the file, and
|
|
184
|
+
* StoreFencedError when the storage is already fenced at open (that handle is abandoned, as in FencedDatabase).
|
|
185
|
+
*/
|
|
186
|
+
export async function openArchilStore(file, profile = "exclusive", options = {}) {
|
|
187
|
+
if (profile !== "exclusive" && profile !== "shared")
|
|
188
|
+
throw new TypeError(`unknown store profile ${JSON.stringify(profile)}`);
|
|
189
|
+
const fence = new Fence(options.onFenced);
|
|
190
|
+
const decorate = options.decorate ?? ((database) => database);
|
|
191
|
+
let database;
|
|
192
|
+
try {
|
|
193
|
+
if (file !== ":memory:")
|
|
194
|
+
await mkdir(dirname(file), { recursive: true });
|
|
195
|
+
if (profile === "exclusive") {
|
|
196
|
+
const raw = await fence.guard(() => new NodeSqliteDatabase(new DatabaseSync(file, { timeout: 0 })));
|
|
197
|
+
database = new FencedDatabase(decorate(raw), fence);
|
|
198
|
+
await database.exec("PRAGMA locking_mode = EXCLUSIVE"); // before WAL: the WAL index stays in heap
|
|
199
|
+
await database.exec("PRAGMA journal_mode = WAL");
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
database = new FencedDatabase(decorate(await fence.guard(() => openNodeSqliteDatabase(file))), fence);
|
|
203
|
+
}
|
|
204
|
+
await database.exec("PRAGMA synchronous = FULL"); // fsync the WAL on every commit
|
|
205
|
+
await database.exec("PRAGMA mmap_size = 0"); // no mmap I/O of the database file on a network filesystem
|
|
206
|
+
await assertPragmas(database, profile);
|
|
207
|
+
const storage = await SqliteStorage.open(database);
|
|
208
|
+
return { storage, database, profile, file };
|
|
209
|
+
}
|
|
210
|
+
catch (error) {
|
|
211
|
+
if (database !== undefined && !fence.fenced)
|
|
212
|
+
await database.close().catch(() => undefined);
|
|
213
|
+
if (resultCodes(error).some((code) => BUSY_CODES.has(primary(code))))
|
|
214
|
+
throw new StoreBusyError(file, profile, error);
|
|
215
|
+
throw error;
|
|
216
|
+
}
|
|
217
|
+
}
|