flatsql 2.0.3 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +54 -64
- package/dist/artifacts/index.d.ts +1 -3
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +1 -3
- package/dist/artifacts/index.js.map +1 -1
- package/dist/response/artifact.d.ts +1 -2
- package/dist/response/artifact.d.ts.map +1 -1
- package/dist/response/artifact.js.map +1 -1
- package/dist/response/types.d.ts +6 -0
- package/dist/response/types.d.ts.map +1 -1
- package/dist/stress/index.d.ts +0 -1
- package/dist/stress/index.d.ts.map +1 -1
- package/dist/stress/index.js +0 -1
- package/dist/stress/index.js.map +1 -1
- package/dist/stress/native-publish-event-workload.d.ts.map +1 -1
- package/dist/stress/native-publish-event-workload.js +14 -4
- package/dist/stress/native-publish-event-workload.js.map +1 -1
- package/dist/stress/smoke-runner.d.ts.map +1 -1
- package/dist/stress/smoke-runner.js +6 -5
- package/dist/stress/smoke-runner.js.map +1 -1
- package/docs/PARTITION-STORE-WASM.md +262 -0
- package/docs/PARTITION-STORE.md +980 -0
- package/package.json +61 -22
- package/wasm/flatsql-io.js +4 -1
- package/wasm/flatsql-ps-threads.wasm +0 -0
- package/wasm/flatsql-wasi-noeh.wasm +0 -0
- package/wasm/flatsql-wasi.wasm +0 -0
- package/wasm/flatsql.js +1 -1
- package/wasm/flatsql.wasm +0 -0
- package/wasm/index.d.ts +2 -0
- package/wasm/index.js +16 -1
- package/wasm/integrity.json +28 -0
- package/wasm/ps-node-fault.d.mts +23 -0
- package/wasm/ps-node-fault.mjs +381 -0
- package/wasm/ps-node-guest.mjs +156 -0
- package/wasm/ps-node-host.d.mts +86 -0
- package/wasm/ps-node-host.mjs +431 -0
- package/wasm/ps-node-imports.mjs +847 -0
- package/wasm/ps.d.ts +21 -0
- package/wasm/ps.js +75 -0
- package/dist/artifacts/builder.d.ts +0 -31
- package/dist/artifacts/builder.d.ts.map +0 -1
- package/dist/artifacts/builder.js +0 -514
- package/dist/artifacts/builder.js.map +0 -1
- package/dist/artifacts/demo-extractors.d.ts +0 -15
- package/dist/artifacts/demo-extractors.d.ts.map +0 -1
- package/dist/artifacts/demo-extractors.js +0 -238
- package/dist/artifacts/demo-extractors.js.map +0 -1
- package/dist/artifacts/standalone-builder.d.ts +0 -62
- package/dist/artifacts/standalone-builder.d.ts.map +0 -1
- package/dist/artifacts/standalone-builder.js +0 -144
- package/dist/artifacts/standalone-builder.js.map +0 -1
- package/dist/artifacts/worker-client.d.ts +0 -32
- package/dist/artifacts/worker-client.d.ts.map +0 -1
- package/dist/artifacts/worker-client.js +0 -415
- package/dist/artifacts/worker-client.js.map +0 -1
- package/dist/core/database.d.ts +0 -86
- package/dist/core/database.d.ts.map +0 -1
- package/dist/core/database.js +0 -900
- package/dist/core/database.js.map +0 -1
- package/dist/core/index.d.ts +0 -3
- package/dist/core/index.d.ts.map +0 -1
- package/dist/core/index.js +0 -3
- package/dist/core/index.js.map +0 -1
- package/dist/core/table-store.d.ts +0 -30
- package/dist/core/table-store.d.ts.map +0 -1
- package/dist/core/table-store.js +0 -187
- package/dist/core/table-store.js.map +0 -1
- package/dist/flatbuffer-accessor.d.ts +0 -35
- package/dist/flatbuffer-accessor.d.ts.map +0 -1
- package/dist/flatbuffer-accessor.js +0 -114
- package/dist/flatbuffer-accessor.js.map +0 -1
- package/dist/index.d.ts +0 -15
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -22
- package/dist/index.js.map +0 -1
- package/dist/stress/publish-event-workload.d.ts +0 -3
- package/dist/stress/publish-event-workload.d.ts.map +0 -1
- package/dist/stress/publish-event-workload.js +0 -467
- package/dist/stress/publish-event-workload.js.map +0 -1
- package/wasm/flatsql-artifact.worker.js +0 -847
- package/wasm/standalone.d.ts +0 -122
- package/wasm/standalone.js +0 -914
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
// FlatSQL partition store: the Node host's fault overlay (design §19,
|
|
2
|
+
// 22.3a-7; docs/PARTITION-STORE-WASM.md).
|
|
3
|
+
//
|
|
4
|
+
// A SharedArrayBuffer page table every worker consults. It sits under the
|
|
5
|
+
// SDK's Node sync-fs flatsql_io provider (its `interpose` hook) and records,
|
|
6
|
+
// for every file, what a power loss could take away:
|
|
7
|
+
// - the pre-image of every 4 KiB page written since the file's last sync,
|
|
8
|
+
// and the size the file had at that sync;
|
|
9
|
+
// - files created since their parent directory's last sync.
|
|
10
|
+
// The supervisor freezes the overlay (every later mutating call fails with
|
|
11
|
+
// EIO, as from a dying process), stops the guest, then applies a crash:
|
|
12
|
+
// dropAll every unsynced page write and size change is undone, and
|
|
13
|
+
// entries created since the parent's last sync disappear;
|
|
14
|
+
// pageSubset each unsynced page independently survives or is undone
|
|
15
|
+
// (writeback in any order), and the size is the synced or the
|
|
16
|
+
// current one;
|
|
17
|
+
// keepAll kill -9 with the page cache intact: nothing is undone.
|
|
18
|
+
// Reads and syncs pass through. The overlay never touches data the guest has
|
|
19
|
+
// synced: a record acked after its durable append must survive every mode.
|
|
20
|
+
|
|
21
|
+
import fs from "node:fs";
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
|
|
24
|
+
export const PAGE = 4096;
|
|
25
|
+
const MAGIC = 0x46534c54; // "FSLT"
|
|
26
|
+
|
|
27
|
+
// Header words.
|
|
28
|
+
const H_MAGIC = 0;
|
|
29
|
+
const H_FROZEN = 1;
|
|
30
|
+
const H_OVERFLOW = 2;
|
|
31
|
+
const H_MAX_FILES = 3;
|
|
32
|
+
const H_MAX_PAGES = 4;
|
|
33
|
+
const H_NEXT_PAGE = 5;
|
|
34
|
+
const H_WRITES = 6;
|
|
35
|
+
const H_SYNCS = 7;
|
|
36
|
+
const H_WORDS = 16;
|
|
37
|
+
// File words: state, lock, flags, head page entry, pathLen, pad x3.
|
|
38
|
+
const F_WORDS = 8;
|
|
39
|
+
const F_STATE = 0;
|
|
40
|
+
const F_LOCK = 1;
|
|
41
|
+
const F_FLAGS = 2;
|
|
42
|
+
const F_HEAD = 3;
|
|
43
|
+
const F_PATHLEN = 4;
|
|
44
|
+
const F_EMPTY = 0;
|
|
45
|
+
const F_CLAIM = 1;
|
|
46
|
+
const F_LIVE = 2;
|
|
47
|
+
const FLAG_TRACKED = 1; // synced size recorded
|
|
48
|
+
const FLAG_NEW_ENTRY = 2; // created since the parent directory's last sync
|
|
49
|
+
const MAX_PATH = 1024;
|
|
50
|
+
// Page entry words: fileId, pageNo, preimage length, next.
|
|
51
|
+
const P_WORDS = 4;
|
|
52
|
+
|
|
53
|
+
const mix = (s) => {
|
|
54
|
+
let h = 0x811c9dc5;
|
|
55
|
+
for (let i = 0; i < s.length; i += 1) {
|
|
56
|
+
h ^= s.charCodeAt(i);
|
|
57
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
58
|
+
}
|
|
59
|
+
return h;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Allocate an overlay. `maxPages` bounds the unsynced pages tracked at once
|
|
64
|
+
* (4 KiB each); overflowing it is reported, never silently ignored.
|
|
65
|
+
*/
|
|
66
|
+
export function createFaultOverlay({ maxFiles = 16384, maxPages = 65536 } = {}) {
|
|
67
|
+
const bytes =
|
|
68
|
+
H_WORDS * 4 + maxFiles * F_WORDS * 4 + maxFiles * 8 + maxFiles * MAX_PATH + maxPages * P_WORDS * 4 + maxPages * PAGE;
|
|
69
|
+
const buffer = new SharedArrayBuffer(bytes);
|
|
70
|
+
const h = new Int32Array(buffer, 0, H_WORDS);
|
|
71
|
+
h[H_MAGIC] = MAGIC;
|
|
72
|
+
h[H_MAX_FILES] = maxFiles;
|
|
73
|
+
h[H_MAX_PAGES] = maxPages;
|
|
74
|
+
const v = views(buffer);
|
|
75
|
+
v.files.fill(0);
|
|
76
|
+
for (let i = 0; i < maxFiles; i += 1) v.files[i * F_WORDS + F_HEAD] = -1;
|
|
77
|
+
return { buffer, ...overlayApi(buffer) };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function views(buffer) {
|
|
81
|
+
const h = new Int32Array(buffer, 0, H_WORDS);
|
|
82
|
+
if (h[H_MAGIC] !== MAGIC) throw new TypeError("not a fault overlay");
|
|
83
|
+
const maxFiles = h[H_MAX_FILES];
|
|
84
|
+
const maxPages = h[H_MAX_PAGES];
|
|
85
|
+
let off = H_WORDS * 4;
|
|
86
|
+
const files = new Int32Array(buffer, off, maxFiles * F_WORDS);
|
|
87
|
+
off += maxFiles * F_WORDS * 4;
|
|
88
|
+
const syncedSize = new Float64Array(buffer, off, maxFiles);
|
|
89
|
+
off += maxFiles * 8;
|
|
90
|
+
const paths = new Uint8Array(buffer, off, maxFiles * MAX_PATH);
|
|
91
|
+
off += maxFiles * MAX_PATH;
|
|
92
|
+
const pages = new Int32Array(buffer, off, maxPages * P_WORDS);
|
|
93
|
+
off += maxPages * P_WORDS * 4;
|
|
94
|
+
const images = new Uint8Array(buffer, off, maxPages * PAGE);
|
|
95
|
+
return { h, maxFiles, maxPages, files, syncedSize, paths, pages, images };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function lock(v, id) {
|
|
99
|
+
const at = id * F_WORDS + F_LOCK;
|
|
100
|
+
while (Atomics.compareExchange(v.files, at, 0, 1) !== 0) Atomics.wait(v.files, at, 1, 1);
|
|
101
|
+
}
|
|
102
|
+
function unlock(v, id) {
|
|
103
|
+
const at = id * F_WORDS + F_LOCK;
|
|
104
|
+
Atomics.store(v.files, at, 0);
|
|
105
|
+
Atomics.notify(v.files, at, 1);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function pathOf(v, id) {
|
|
109
|
+
const len = v.files[id * F_WORDS + F_PATHLEN];
|
|
110
|
+
return Buffer.from(v.paths.slice(id * MAX_PATH, id * MAX_PATH + len)).toString("utf8");
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// File id for a host path (open addressing), created on first sight.
|
|
114
|
+
function fileId(v, full, create) {
|
|
115
|
+
const bytes = Buffer.from(full);
|
|
116
|
+
if (bytes.length > MAX_PATH) return -1;
|
|
117
|
+
let i = mix(full) % v.maxFiles;
|
|
118
|
+
for (let n = 0; n < v.maxFiles; n += 1, i = (i + 1) % v.maxFiles) {
|
|
119
|
+
const at = i * F_WORDS;
|
|
120
|
+
let state = Atomics.load(v.files, at + F_STATE);
|
|
121
|
+
if (state === F_EMPTY) {
|
|
122
|
+
if (!create) return -1;
|
|
123
|
+
if (Atomics.compareExchange(v.files, at + F_STATE, F_EMPTY, F_CLAIM) === F_EMPTY) {
|
|
124
|
+
v.paths.set(bytes, i * MAX_PATH);
|
|
125
|
+
v.files[at + F_PATHLEN] = bytes.length;
|
|
126
|
+
v.files[at + F_FLAGS] = 0;
|
|
127
|
+
v.files[at + F_HEAD] = -1;
|
|
128
|
+
Atomics.store(v.files, at + F_STATE, F_LIVE);
|
|
129
|
+
Atomics.notify(v.files, at + F_STATE);
|
|
130
|
+
return i;
|
|
131
|
+
}
|
|
132
|
+
state = Atomics.load(v.files, at + F_STATE);
|
|
133
|
+
}
|
|
134
|
+
while (state === F_CLAIM) {
|
|
135
|
+
Atomics.wait(v.files, at + F_STATE, F_CLAIM, 1);
|
|
136
|
+
state = Atomics.load(v.files, at + F_STATE);
|
|
137
|
+
}
|
|
138
|
+
if (v.files[at + F_PATHLEN] === bytes.length && pathOf(v, i) === full) return i;
|
|
139
|
+
}
|
|
140
|
+
Atomics.store(v.h, H_OVERFLOW, 1);
|
|
141
|
+
return -1;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function sizeOf(fd) {
|
|
145
|
+
return fs.fstatSync(fd).size;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Record the synced size the first time a file is written after a sync.
|
|
149
|
+
function track(v, id, fd) {
|
|
150
|
+
const at = id * F_WORDS + F_FLAGS;
|
|
151
|
+
if (!(v.files[at] & FLAG_TRACKED)) {
|
|
152
|
+
v.syncedSize[id] = sizeOf(fd);
|
|
153
|
+
v.files[at] |= FLAG_TRACKED;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Save the pre-image of page `p` unless it is already saved since the last
|
|
158
|
+
// sync. Pages wholly past the synced size need none: a crash cuts the file
|
|
159
|
+
// back to that size, or (pageSubset keeping the size) leaves new or zero bytes
|
|
160
|
+
// there, both of which a real disk can produce.
|
|
161
|
+
function savePage(v, id, fd, p) {
|
|
162
|
+
if (p * PAGE >= v.syncedSize[id]) return true;
|
|
163
|
+
for (let e = v.files[id * F_WORDS + F_HEAD]; e >= 0; e = v.pages[e * P_WORDS + 3]) {
|
|
164
|
+
if (v.pages[e * P_WORDS + 1] === p) return true;
|
|
165
|
+
}
|
|
166
|
+
const e = Atomics.add(v.h, H_NEXT_PAGE, 1);
|
|
167
|
+
if (e >= v.maxPages) {
|
|
168
|
+
Atomics.store(v.h, H_OVERFLOW, 1);
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
const img = v.images.subarray(e * PAGE, (e + 1) * PAGE);
|
|
172
|
+
const tmp = Buffer.alloc(PAGE);
|
|
173
|
+
const n = fs.readSync(fd, tmp, 0, PAGE, p * PAGE);
|
|
174
|
+
img.set(tmp);
|
|
175
|
+
v.pages[e * P_WORDS] = id;
|
|
176
|
+
v.pages[e * P_WORDS + 1] = p;
|
|
177
|
+
v.pages[e * P_WORDS + 2] = n;
|
|
178
|
+
v.pages[e * P_WORDS + 3] = v.files[id * F_WORDS + F_HEAD];
|
|
179
|
+
v.files[id * F_WORDS + F_HEAD] = e;
|
|
180
|
+
return true;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function frozenError() {
|
|
184
|
+
return Object.assign(new Error("fault overlay frozen"), { code: "EIO" });
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The `interpose` hook for one thread's SDK sync-fs provider. `root` is the
|
|
189
|
+
* provider's root (paths it opens are absolute below it).
|
|
190
|
+
*/
|
|
191
|
+
export function createFaultInterposer(buffer, _root) {
|
|
192
|
+
const v = views(buffer);
|
|
193
|
+
const fdFile = new Map(); // this thread's fds -> file id
|
|
194
|
+
const mutating = new Set(["write", "truncate", "mkdir", "unlink"]);
|
|
195
|
+
return (op, fn, args) => {
|
|
196
|
+
if (
|
|
197
|
+
Atomics.load(v.h, H_FROZEN) &&
|
|
198
|
+
(mutating.has(op) || op === "sync" || op === "syncdir" ||
|
|
199
|
+
(op === "open" && (args[1] & fs.constants.O_CREAT) !== 0))
|
|
200
|
+
) {
|
|
201
|
+
throw frozenError();
|
|
202
|
+
}
|
|
203
|
+
if (op === "open") {
|
|
204
|
+
const full = args[0];
|
|
205
|
+
const creating = (args[1] & fs.constants.O_CREAT) !== 0;
|
|
206
|
+
const existed = creating ? fs.existsSync(full) : true;
|
|
207
|
+
const fd = fn(...args);
|
|
208
|
+
const id = fileId(v, full, true);
|
|
209
|
+
if (id >= 0) {
|
|
210
|
+
fdFile.set(fd, id);
|
|
211
|
+
if (creating && !existed) {
|
|
212
|
+
lock(v, id);
|
|
213
|
+
v.files[id * F_WORDS + F_FLAGS] = (v.files[id * F_WORDS + F_FLAGS] | FLAG_NEW_ENTRY) & ~FLAG_TRACKED;
|
|
214
|
+
// A new file's synced content is nothing.
|
|
215
|
+
v.syncedSize[id] = 0;
|
|
216
|
+
v.files[id * F_WORDS + F_FLAGS] |= FLAG_TRACKED;
|
|
217
|
+
unlock(v, id);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return fd;
|
|
221
|
+
}
|
|
222
|
+
if (op === "close") {
|
|
223
|
+
fdFile.delete(args[0]);
|
|
224
|
+
return fn(...args);
|
|
225
|
+
}
|
|
226
|
+
if (op === "write") {
|
|
227
|
+
const [fd, view, off, len, pos] = args;
|
|
228
|
+
const id = fdFile.get(fd);
|
|
229
|
+
if (id === undefined) return fn(...args);
|
|
230
|
+
lock(v, id);
|
|
231
|
+
try {
|
|
232
|
+
track(v, id, fd);
|
|
233
|
+
const first = Math.floor(pos / PAGE);
|
|
234
|
+
const last = Math.floor((pos + len - 1) / PAGE);
|
|
235
|
+
for (let p = first; p <= last; p += 1) savePage(v, id, fd, p);
|
|
236
|
+
Atomics.add(v.h, H_WRITES, 1);
|
|
237
|
+
return fn(fd, view, off, len, pos);
|
|
238
|
+
} finally {
|
|
239
|
+
unlock(v, id);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
if (op === "truncate") {
|
|
243
|
+
const [fd, size] = args;
|
|
244
|
+
const id = fdFile.get(fd);
|
|
245
|
+
if (id === undefined) return fn(...args);
|
|
246
|
+
lock(v, id);
|
|
247
|
+
try {
|
|
248
|
+
track(v, id, fd);
|
|
249
|
+
const cur = sizeOf(fd);
|
|
250
|
+
for (let p = Math.floor(size / PAGE); p * PAGE < cur; p += 1) savePage(v, id, fd, p);
|
|
251
|
+
return fn(...args);
|
|
252
|
+
} finally {
|
|
253
|
+
unlock(v, id);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (op === "sync") {
|
|
257
|
+
const [fd] = args;
|
|
258
|
+
const id = fdFile.get(fd);
|
|
259
|
+
const r = fn(...args);
|
|
260
|
+
if (id !== undefined) {
|
|
261
|
+
lock(v, id);
|
|
262
|
+
// Everything written before this sync returned is durable.
|
|
263
|
+
v.files[id * F_WORDS + F_HEAD] = -1;
|
|
264
|
+
v.files[id * F_WORDS + F_FLAGS] &= ~FLAG_TRACKED;
|
|
265
|
+
unlock(v, id);
|
|
266
|
+
Atomics.add(v.h, H_SYNCS, 1);
|
|
267
|
+
}
|
|
268
|
+
return r;
|
|
269
|
+
}
|
|
270
|
+
if (op === "syncdir") {
|
|
271
|
+
const r = fn(...args);
|
|
272
|
+
const dir = path.resolve(args[0]);
|
|
273
|
+
for (let id = 0; id < v.maxFiles; id += 1) {
|
|
274
|
+
if (Atomics.load(v.files, id * F_WORDS + F_STATE) !== F_LIVE) continue;
|
|
275
|
+
if (!(v.files[id * F_WORDS + F_FLAGS] & FLAG_NEW_ENTRY)) continue;
|
|
276
|
+
if (path.dirname(pathOf(v, id)) !== dir) continue;
|
|
277
|
+
lock(v, id);
|
|
278
|
+
v.files[id * F_WORDS + F_FLAGS] &= ~FLAG_NEW_ENTRY;
|
|
279
|
+
unlock(v, id);
|
|
280
|
+
}
|
|
281
|
+
return r;
|
|
282
|
+
}
|
|
283
|
+
if (op === "unlink") {
|
|
284
|
+
const full = args[0];
|
|
285
|
+
const r = fn(...args);
|
|
286
|
+
const id = fileId(v, full, false);
|
|
287
|
+
if (id >= 0) {
|
|
288
|
+
lock(v, id);
|
|
289
|
+
v.files[id * F_WORDS + F_FLAGS] = 0;
|
|
290
|
+
v.files[id * F_WORDS + F_HEAD] = -1;
|
|
291
|
+
unlock(v, id);
|
|
292
|
+
}
|
|
293
|
+
return r;
|
|
294
|
+
}
|
|
295
|
+
return fn(...args);
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function overlayApi(buffer) {
|
|
300
|
+
const v = views(buffer);
|
|
301
|
+
return {
|
|
302
|
+
/** Every later mutating call fails with EIO (the process is dying). */
|
|
303
|
+
freeze() {
|
|
304
|
+
Atomics.store(v.h, H_FROZEN, 1);
|
|
305
|
+
},
|
|
306
|
+
stats() {
|
|
307
|
+
return {
|
|
308
|
+
writes: Atomics.load(v.h, H_WRITES),
|
|
309
|
+
syncs: Atomics.load(v.h, H_SYNCS),
|
|
310
|
+
pagesUsed: Math.min(Atomics.load(v.h, H_NEXT_PAGE), v.maxPages),
|
|
311
|
+
overflow: Atomics.load(v.h, H_OVERFLOW) !== 0,
|
|
312
|
+
};
|
|
313
|
+
},
|
|
314
|
+
/**
|
|
315
|
+
* Apply a crash after every guest thread has stopped, then reset the
|
|
316
|
+
* overlay (unfrozen, nothing tracked). `random` returns [0, 1).
|
|
317
|
+
* @returns {{ pagesUndone: number, pagesKept: number, filesTruncated: number, entriesDropped: number }}
|
|
318
|
+
*/
|
|
319
|
+
crash(mode, random = Math.random) {
|
|
320
|
+
if (Atomics.load(v.h, H_OVERFLOW)) throw new Error("fault overlay overflowed: the crash would be incomplete");
|
|
321
|
+
const out = { pagesUndone: 0, pagesKept: 0, filesTruncated: 0, entriesDropped: 0 };
|
|
322
|
+
for (let id = 0; id < v.maxFiles; id += 1) {
|
|
323
|
+
const at = id * F_WORDS;
|
|
324
|
+
if (v.files[at + F_STATE] !== F_LIVE) continue;
|
|
325
|
+
const flags = v.files[at + F_FLAGS];
|
|
326
|
+
const full = pathOf(v, id);
|
|
327
|
+
if (mode !== "keepAll" && flags & FLAG_NEW_ENTRY) {
|
|
328
|
+
try {
|
|
329
|
+
fs.unlinkSync(full);
|
|
330
|
+
out.entriesDropped += 1;
|
|
331
|
+
} catch {
|
|
332
|
+
// already gone
|
|
333
|
+
}
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
if (mode === "keepAll" || !(flags & FLAG_TRACKED)) continue;
|
|
337
|
+
let fd;
|
|
338
|
+
try {
|
|
339
|
+
fd = fs.openSync(full, "r+");
|
|
340
|
+
} catch {
|
|
341
|
+
continue; // unlinked durably since
|
|
342
|
+
}
|
|
343
|
+
try {
|
|
344
|
+
for (let e = v.files[at + F_HEAD]; e >= 0; e = v.pages[e * P_WORDS + 3]) {
|
|
345
|
+
const undo = mode === "dropAll" || random() < 0.5;
|
|
346
|
+
if (!undo) {
|
|
347
|
+
out.pagesKept += 1;
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
const p = v.pages[e * P_WORDS + 1];
|
|
351
|
+
const n = v.pages[e * P_WORDS + 2];
|
|
352
|
+
if (n > 0) fs.writeSync(fd, v.images.subarray(e * PAGE, e * PAGE + n), 0, n, p * PAGE);
|
|
353
|
+
out.pagesUndone += 1;
|
|
354
|
+
}
|
|
355
|
+
const synced = v.syncedSize[id];
|
|
356
|
+
const size = fs.fstatSync(fd).size;
|
|
357
|
+
const keepSize = mode === "pageSubset" && random() < 0.5;
|
|
358
|
+
if (!keepSize && size !== synced) {
|
|
359
|
+
fs.ftruncateSync(fd, synced);
|
|
360
|
+
out.filesTruncated += 1;
|
|
361
|
+
}
|
|
362
|
+
} finally {
|
|
363
|
+
fs.closeSync(fd);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
// Reset.
|
|
367
|
+
for (let id = 0; id < v.maxFiles; id += 1) {
|
|
368
|
+
v.files[id * F_WORDS + F_FLAGS] = 0;
|
|
369
|
+
v.files[id * F_WORDS + F_HEAD] = -1;
|
|
370
|
+
}
|
|
371
|
+
Atomics.store(v.h, H_NEXT_PAGE, 0);
|
|
372
|
+
Atomics.store(v.h, H_FROZEN, 0);
|
|
373
|
+
return out;
|
|
374
|
+
},
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** Re-attach to an overlay buffer (for example in another thread). */
|
|
379
|
+
export function attachFaultOverlay(buffer) {
|
|
380
|
+
return { buffer, ...overlayApi(buffer) };
|
|
381
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// FlatSQL partition store: a guest thread's worker under the Node host
|
|
2
|
+
// (ps-node-host.mjs). Three modes:
|
|
3
|
+
//
|
|
4
|
+
// command the guest's own thread: runs `_start` (the wasm test commands)
|
|
5
|
+
// and reports its exit;
|
|
6
|
+
// reactor the exec thread of one flatsql-ps-threads.wasm instance: runs
|
|
7
|
+
// `_initialize`, then serves the instance's C ABI calls (init,
|
|
8
|
+
// start, stats, stop) posted by the supervisor;
|
|
9
|
+
// pool a pooled thread for wasi.thread-spawn: waits on its slot of the
|
|
10
|
+
// pool's SharedArrayBuffer, instantiates the module over the shared
|
|
11
|
+
// memory, runs wasi_thread_start(tid, arg), and returns to idle.
|
|
12
|
+
//
|
|
13
|
+
// Every mode imports the same host (ps-node-imports.mjs), including
|
|
14
|
+
// wasi.thread-spawn, so any guest thread may spawn threads. The supervisor
|
|
15
|
+
// on the main thread starts pool workers and is never blocked by the guest.
|
|
16
|
+
|
|
17
|
+
import { parentPort, threadId, workerData } from "node:worker_threads";
|
|
18
|
+
import { writeSync } from "node:fs";
|
|
19
|
+
|
|
20
|
+
import {
|
|
21
|
+
CONTROL_EXITED,
|
|
22
|
+
CONTROL_EXIT_CODE,
|
|
23
|
+
CONTROL_FAULT,
|
|
24
|
+
CONTROL_FAULT_TID,
|
|
25
|
+
POOL_GEN,
|
|
26
|
+
POOL_HEADER,
|
|
27
|
+
POOL_MAX_RUNNING,
|
|
28
|
+
POOL_RUNNING,
|
|
29
|
+
POOL_SLOT_WORDS,
|
|
30
|
+
SLOT_ASSIGNED,
|
|
31
|
+
SLOT_DEAD,
|
|
32
|
+
SLOT_IDLE,
|
|
33
|
+
SLOT_RUNNING,
|
|
34
|
+
WasiExitError,
|
|
35
|
+
createThreadImports,
|
|
36
|
+
poolThreadSpawn,
|
|
37
|
+
registerThread,
|
|
38
|
+
} from "./ps-node-imports.mjs";
|
|
39
|
+
|
|
40
|
+
const { mode, wasmModule, memory, cfg } = workerData;
|
|
41
|
+
// A pool worker counts as a guest OS thread once it runs a guest thread.
|
|
42
|
+
let control = mode === "pool" ? new Int32Array(cfg.control) : registerThread(cfg.control);
|
|
43
|
+
const pool = new Int32Array(cfg.pool);
|
|
44
|
+
|
|
45
|
+
const built = createThreadImports(cfg, () => memory);
|
|
46
|
+
const imports = {
|
|
47
|
+
...built.imports,
|
|
48
|
+
env: { ...built.imports.env, memory },
|
|
49
|
+
wasi: { "thread-spawn": poolThreadSpawn(cfg.pool, cfg.spawnWaitMs) },
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const isExit = (error) => error instanceof WasiExitError || error?.name === "WasiExitError";
|
|
53
|
+
|
|
54
|
+
function exitProcess(code) {
|
|
55
|
+
if (Atomics.compareExchange(control, CONTROL_EXITED, 0, 1) === 0) {
|
|
56
|
+
Atomics.store(control, CONTROL_EXIT_CODE, code);
|
|
57
|
+
}
|
|
58
|
+
Atomics.notify(control, CONTROL_EXITED);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function fault(tid, error) {
|
|
62
|
+
// Written here, synchronously: the thread that would join this one may be
|
|
63
|
+
// blocked in the guest forever.
|
|
64
|
+
try {
|
|
65
|
+
writeSync(2, `[ps-node-host] guest thread ${tid} trapped: ${error?.stack ?? error}\n`);
|
|
66
|
+
} catch {
|
|
67
|
+
// stderr gone
|
|
68
|
+
}
|
|
69
|
+
Atomics.store(control, CONTROL_FAULT_TID, tid);
|
|
70
|
+
Atomics.store(control, CONTROL_FAULT, 1);
|
|
71
|
+
Atomics.notify(control, CONTROL_FAULT);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (mode === "pool") {
|
|
75
|
+
const slot = POOL_HEADER + workerData.slot * POOL_SLOT_WORDS;
|
|
76
|
+
Atomics.store(pool, slot + 3, threadId);
|
|
77
|
+
let registered = false;
|
|
78
|
+
for (;;) {
|
|
79
|
+
Atomics.store(pool, slot, SLOT_IDLE);
|
|
80
|
+
Atomics.add(pool, POOL_GEN, 1);
|
|
81
|
+
Atomics.notify(pool, POOL_GEN);
|
|
82
|
+
for (;;) {
|
|
83
|
+
const state = Atomics.load(pool, slot);
|
|
84
|
+
if (state === SLOT_ASSIGNED) break;
|
|
85
|
+
Atomics.wait(pool, slot, state, 1000);
|
|
86
|
+
}
|
|
87
|
+
const tid = Atomics.load(pool, slot + 1);
|
|
88
|
+
const arg = Atomics.load(pool, slot + 2);
|
|
89
|
+
if (!registered) {
|
|
90
|
+
registerThread(cfg.control);
|
|
91
|
+
registered = true;
|
|
92
|
+
}
|
|
93
|
+
Atomics.store(pool, slot, SLOT_RUNNING);
|
|
94
|
+
const running = Atomics.add(pool, POOL_RUNNING, 1) + 1;
|
|
95
|
+
for (;;) {
|
|
96
|
+
const max = Atomics.load(pool, POOL_MAX_RUNNING);
|
|
97
|
+
if (running <= max || Atomics.compareExchange(pool, POOL_MAX_RUNNING, max, running) === max) break;
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
const instance = new WebAssembly.Instance(wasmModule, imports);
|
|
101
|
+
instance.exports.wasi_thread_start(tid, arg);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
Atomics.sub(pool, POOL_RUNNING, 1);
|
|
104
|
+
Atomics.store(pool, slot, SLOT_DEAD);
|
|
105
|
+
if (isExit(error)) exitProcess(error.code);
|
|
106
|
+
else fault(tid, error);
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
Atomics.sub(pool, POOL_RUNNING, 1);
|
|
110
|
+
}
|
|
111
|
+
built.close();
|
|
112
|
+
} else if (mode === "command") {
|
|
113
|
+
const instance = new WebAssembly.Instance(wasmModule, imports);
|
|
114
|
+
let code = 0;
|
|
115
|
+
let error = null;
|
|
116
|
+
try {
|
|
117
|
+
instance.exports._start();
|
|
118
|
+
} catch (e) {
|
|
119
|
+
if (isExit(e)) code = e.code;
|
|
120
|
+
else {
|
|
121
|
+
code = 134;
|
|
122
|
+
error = String(e?.stack ?? e);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
exitProcess(code);
|
|
126
|
+
parentPort.postMessage({ t: "done", code: Atomics.load(control, CONTROL_EXIT_CODE), error });
|
|
127
|
+
} else {
|
|
128
|
+
const instance = new WebAssembly.Instance(wasmModule, imports);
|
|
129
|
+
instance.exports._initialize?.();
|
|
130
|
+
const mem = () => new Uint8Array(memory.buffer);
|
|
131
|
+
parentPort.on("message", (msg) => {
|
|
132
|
+
const reply = (value, err) =>
|
|
133
|
+
parentPort.postMessage({ t: "reply", id: msg.id, value, error: err ? String(err?.stack ?? err) : null });
|
|
134
|
+
try {
|
|
135
|
+
if (msg.op === "call") {
|
|
136
|
+
const fn = instance.exports[msg.fn];
|
|
137
|
+
if (typeof fn !== "function") throw new Error(`no export ${msg.fn}`);
|
|
138
|
+
reply(fn(...(msg.args ?? [])));
|
|
139
|
+
} else if (msg.op === "write") {
|
|
140
|
+
const n = msg.bytes.length;
|
|
141
|
+
const ptr = instance.exports.flatsql_ps_alloc(Math.max(n, 1));
|
|
142
|
+
if (!ptr) throw new Error("flatsql_ps_alloc failed");
|
|
143
|
+
mem().set(msg.bytes, ptr);
|
|
144
|
+
reply(ptr);
|
|
145
|
+
} else if (msg.op === "read") {
|
|
146
|
+
reply(mem().slice(msg.ptr, msg.ptr + msg.len));
|
|
147
|
+
} else {
|
|
148
|
+
throw new Error(`unknown op ${msg.op}`);
|
|
149
|
+
}
|
|
150
|
+
} catch (err) {
|
|
151
|
+
if (isExit(err)) exitProcess(err.code);
|
|
152
|
+
reply(undefined, err);
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
parentPort.postMessage({ t: "ready" });
|
|
156
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// Types for wasm/ps-node-host.mjs (the partition store's Node wasi-threads host).
|
|
2
|
+
|
|
3
|
+
export interface PsThreadReport {
|
|
4
|
+
/** wasi.thread-spawn calls that started a guest thread. */
|
|
5
|
+
guestThreadsSpawned: number;
|
|
6
|
+
/** Most guest threads running at once, the guest's own thread included. */
|
|
7
|
+
maxConcurrentGuestThreads: number;
|
|
8
|
+
runningGuestThreads: number;
|
|
9
|
+
spawnsDeclined: number;
|
|
10
|
+
/** Node worker threads (OS threads) that ran guest code. */
|
|
11
|
+
workerThreads: number;
|
|
12
|
+
distinctWorkerThreadIds: number;
|
|
13
|
+
memoryBytes: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface PsMount {
|
|
17
|
+
guest: string;
|
|
18
|
+
host: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface PsHostOptions {
|
|
22
|
+
/** Host directory the guest sees as "/" (WASI and flatsql_io). Default: a private temporary directory, removed afterwards. */
|
|
23
|
+
root?: string;
|
|
24
|
+
keepRoot?: boolean;
|
|
25
|
+
/** Further preopened directories. */
|
|
26
|
+
mounts?: PsMount[];
|
|
27
|
+
env?: Record<string, string>;
|
|
28
|
+
/** Pool cap on guest threads (default 512). */
|
|
29
|
+
maxThreads?: number;
|
|
30
|
+
/** Pool workers started up front (default 16). */
|
|
31
|
+
warmThreads?: number;
|
|
32
|
+
/** How long a spawn waits for a free worker before failing with EAGAIN (default 60000). */
|
|
33
|
+
spawnWaitMs?: number;
|
|
34
|
+
/** Fault overlay (ps-node-fault.mjs createFaultOverlay). */
|
|
35
|
+
fault?: { buffer: SharedArrayBuffer };
|
|
36
|
+
stdoutFd?: number;
|
|
37
|
+
stderrFd?: number;
|
|
38
|
+
/** Seeded random_get. */
|
|
39
|
+
deterministic?: { randomSeed?: bigint };
|
|
40
|
+
initialPages?: number;
|
|
41
|
+
maximumPages?: number;
|
|
42
|
+
maxHandles?: number;
|
|
43
|
+
instanceId?: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface PsCommandOptions extends PsHostOptions {
|
|
47
|
+
args?: string[];
|
|
48
|
+
timeoutMs?: number;
|
|
49
|
+
/** Kill the guest after this long: a crash (result.killed), not a failure. */
|
|
50
|
+
killAfterMs?: number;
|
|
51
|
+
beforeKill?: () => void;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface PsCommandResult {
|
|
55
|
+
exitCode: number;
|
|
56
|
+
killed: boolean;
|
|
57
|
+
timedOut: boolean;
|
|
58
|
+
fault: boolean;
|
|
59
|
+
error: string | null;
|
|
60
|
+
elapsedMs: number;
|
|
61
|
+
threads: PsThreadReport;
|
|
62
|
+
root: string;
|
|
63
|
+
poolErrors: string[];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function runPsCommand(wasm: string | Uint8Array, options?: PsCommandOptions): Promise<PsCommandResult>;
|
|
67
|
+
|
|
68
|
+
export interface PsNodeInstance {
|
|
69
|
+
memory: WebAssembly.Memory;
|
|
70
|
+
root: string;
|
|
71
|
+
/** Call an export of the instance on its exec thread. */
|
|
72
|
+
call(fn: string, ...args: number[]): Promise<number>;
|
|
73
|
+
/** Copy bytes into instance memory (flatsql_ps_alloc); returns the pointer. */
|
|
74
|
+
write(bytes: Uint8Array | ArrayBuffer): Promise<number>;
|
|
75
|
+
read(ptr: number, len: number): Promise<Uint8Array>;
|
|
76
|
+
threads(): PsThreadReport;
|
|
77
|
+
faulted(): boolean;
|
|
78
|
+
close(): Promise<void>;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** flatsql-ps-threads.wasm as one instance (writer or reader) with its own exec thread and thread pool. */
|
|
82
|
+
export function createPsNodeInstance(wasm: string | Uint8Array, options?: PsHostOptions): Promise<PsNodeInstance>;
|
|
83
|
+
|
|
84
|
+
export function importedMemoryLimits(
|
|
85
|
+
bytes: Uint8Array | ArrayBuffer,
|
|
86
|
+
): { module: string; name: string; initial: number; maximum?: number; shared: boolean } | null;
|