flatsql 3.11.1 → 3.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/README.md +61 -795
  2. package/bin/flatsql.mjs +76 -0
  3. package/cpp/README.md +18 -0
  4. package/cpp/include/flatsql/flatsql.h +162 -0
  5. package/cpp/include/flatsql/flatsql_io.h +159 -0
  6. package/docs/API-3.12.md +126 -0
  7. package/package.json +28 -123
  8. package/wasm/flatsql.wasm +0 -0
  9. package/wasm/index.d.ts +61 -548
  10. package/wasm/index.js +280 -1281
  11. package/wasm/integrity.json +6 -26
  12. package/wasm/runtime/LICENSE.sdk.txt +190 -0
  13. package/wasm/runtime/NOTICE.txt +2 -0
  14. package/wasm/runtime/browser-guest.mjs +1 -0
  15. package/wasm/runtime/browser-host.mjs +1 -0
  16. package/wasm/runtime/node-guest.mjs +2 -0
  17. package/wasm/runtime/node-host.mjs +5 -0
  18. package/wasm/runtime/opfsIoWorker.mjs +1 -0
  19. package/wasm/runtime/wasiThreadBrowserWorker.mjs +1 -0
  20. package/dist/artifacts/index.d.ts +0 -3
  21. package/dist/artifacts/index.d.ts.map +0 -1
  22. package/dist/artifacts/index.js +0 -3
  23. package/dist/artifacts/index.js.map +0 -1
  24. package/dist/artifacts/transport.d.ts +0 -5
  25. package/dist/artifacts/transport.d.ts.map +0 -1
  26. package/dist/artifacts/transport.js +0 -45
  27. package/dist/artifacts/transport.js.map +0 -1
  28. package/dist/artifacts/types.d.ts +0 -34
  29. package/dist/artifacts/types.d.ts.map +0 -1
  30. package/dist/artifacts/types.js +0 -2
  31. package/dist/artifacts/types.js.map +0 -1
  32. package/dist/btree/btree.d.ts +0 -34
  33. package/dist/btree/btree.d.ts.map +0 -1
  34. package/dist/btree/btree.js +0 -234
  35. package/dist/btree/btree.js.map +0 -1
  36. package/dist/btree/index.d.ts +0 -3
  37. package/dist/btree/index.d.ts.map +0 -1
  38. package/dist/btree/index.js +0 -3
  39. package/dist/btree/index.js.map +0 -1
  40. package/dist/btree/types.d.ts +0 -36
  41. package/dist/btree/types.d.ts.map +0 -1
  42. package/dist/btree/types.js +0 -55
  43. package/dist/btree/types.js.map +0 -1
  44. package/dist/cluster/index.d.ts +0 -11
  45. package/dist/cluster/index.d.ts.map +0 -1
  46. package/dist/cluster/index.js +0 -59
  47. package/dist/cluster/index.js.map +0 -1
  48. package/dist/response/artifact.d.ts +0 -4
  49. package/dist/response/artifact.d.ts.map +0 -1
  50. package/dist/response/artifact.js +0 -102
  51. package/dist/response/artifact.js.map +0 -1
  52. package/dist/response/cache-key.d.ts +0 -6
  53. package/dist/response/cache-key.d.ts.map +0 -1
  54. package/dist/response/cache-key.js +0 -114
  55. package/dist/response/cache-key.js.map +0 -1
  56. package/dist/response/index.d.ts +0 -5
  57. package/dist/response/index.d.ts.map +0 -1
  58. package/dist/response/index.js +0 -5
  59. package/dist/response/index.js.map +0 -1
  60. package/dist/response/memory-cache.d.ts +0 -17
  61. package/dist/response/memory-cache.d.ts.map +0 -1
  62. package/dist/response/memory-cache.js +0 -76
  63. package/dist/response/memory-cache.js.map +0 -1
  64. package/dist/response/types.d.ts +0 -57
  65. package/dist/response/types.d.ts.map +0 -1
  66. package/dist/response/types.js +0 -2
  67. package/dist/response/types.js.map +0 -1
  68. package/dist/schema/index.d.ts +0 -3
  69. package/dist/schema/index.d.ts.map +0 -1
  70. package/dist/schema/index.js +0 -3
  71. package/dist/schema/index.js.map +0 -1
  72. package/dist/schema/parser.d.ts +0 -25
  73. package/dist/schema/parser.d.ts.map +0 -1
  74. package/dist/schema/parser.js +0 -474
  75. package/dist/schema/parser.js.map +0 -1
  76. package/dist/schema/types.d.ts +0 -33
  77. package/dist/schema/types.d.ts.map +0 -1
  78. package/dist/schema/types.js +0 -29
  79. package/dist/schema/types.js.map +0 -1
  80. package/dist/standalone/index.d.ts +0 -3
  81. package/dist/standalone/index.d.ts.map +0 -1
  82. package/dist/standalone/index.js +0 -3
  83. package/dist/standalone/index.js.map +0 -1
  84. package/dist/standalone/process-client.d.ts +0 -85
  85. package/dist/standalone/process-client.d.ts.map +0 -1
  86. package/dist/standalone/process-client.js +0 -537
  87. package/dist/standalone/process-client.js.map +0 -1
  88. package/dist/standalone/wasmedge-runner.d.ts +0 -19
  89. package/dist/standalone/wasmedge-runner.d.ts.map +0 -1
  90. package/dist/standalone/wasmedge-runner.js +0 -108
  91. package/dist/standalone/wasmedge-runner.js.map +0 -1
  92. package/dist/storage/index.d.ts +0 -3
  93. package/dist/storage/index.d.ts.map +0 -1
  94. package/dist/storage/index.js +0 -3
  95. package/dist/storage/index.js.map +0 -1
  96. package/dist/storage/stacked-flatbuffers.d.ts +0 -38
  97. package/dist/storage/stacked-flatbuffers.d.ts.map +0 -1
  98. package/dist/storage/stacked-flatbuffers.js +0 -253
  99. package/dist/storage/stacked-flatbuffers.js.map +0 -1
  100. package/dist/storage/types.d.ts +0 -23
  101. package/dist/storage/types.d.ts.map +0 -1
  102. package/dist/storage/types.js +0 -27
  103. package/dist/storage/types.js.map +0 -1
  104. package/dist/stress/aggregate.d.ts +0 -3
  105. package/dist/stress/aggregate.d.ts.map +0 -1
  106. package/dist/stress/aggregate.js +0 -61
  107. package/dist/stress/aggregate.js.map +0 -1
  108. package/dist/stress/docker-compose.d.ts +0 -17
  109. package/dist/stress/docker-compose.d.ts.map +0 -1
  110. package/dist/stress/docker-compose.js +0 -156
  111. package/dist/stress/docker-compose.js.map +0 -1
  112. package/dist/stress/index.d.ts +0 -10
  113. package/dist/stress/index.d.ts.map +0 -1
  114. package/dist/stress/index.js +0 -10
  115. package/dist/stress/index.js.map +0 -1
  116. package/dist/stress/metrics.d.ts +0 -3
  117. package/dist/stress/metrics.d.ts.map +0 -1
  118. package/dist/stress/metrics.js +0 -83
  119. package/dist/stress/metrics.js.map +0 -1
  120. package/dist/stress/native-publish-event-workload.d.ts +0 -3
  121. package/dist/stress/native-publish-event-workload.d.ts.map +0 -1
  122. package/dist/stress/native-publish-event-workload.js +0 -465
  123. package/dist/stress/native-publish-event-workload.js.map +0 -1
  124. package/dist/stress/production-gate.d.ts +0 -4
  125. package/dist/stress/production-gate.d.ts.map +0 -1
  126. package/dist/stress/production-gate.js +0 -12
  127. package/dist/stress/production-gate.js.map +0 -1
  128. package/dist/stress/report.d.ts +0 -3
  129. package/dist/stress/report.d.ts.map +0 -1
  130. package/dist/stress/report.js +0 -174
  131. package/dist/stress/report.js.map +0 -1
  132. package/dist/stress/sds-discovery.d.ts +0 -4
  133. package/dist/stress/sds-discovery.d.ts.map +0 -1
  134. package/dist/stress/sds-discovery.js +0 -40
  135. package/dist/stress/sds-discovery.js.map +0 -1
  136. package/dist/stress/smoke-runner.d.ts +0 -3
  137. package/dist/stress/smoke-runner.d.ts.map +0 -1
  138. package/dist/stress/smoke-runner.js +0 -64
  139. package/dist/stress/smoke-runner.js.map +0 -1
  140. package/dist/stress/types.d.ts +0 -219
  141. package/dist/stress/types.d.ts.map +0 -1
  142. package/dist/stress/types.js +0 -2
  143. package/dist/stress/types.js.map +0 -1
  144. package/dist/stress/workload.d.ts +0 -6
  145. package/dist/stress/workload.d.ts.map +0 -1
  146. package/dist/stress/workload.js +0 -144
  147. package/dist/stress/workload.js.map +0 -1
  148. package/docs/PARTITION-STORE-WASM.md +0 -338
  149. package/docs/PARTITION-STORE.md +0 -2380
  150. package/docs/agents/response-artifacts.md +0 -65
  151. package/docs/response-artifacts.md +0 -86
  152. package/scripts/stress-docker.sh +0 -279
  153. package/sdm/flatsql-spatial.wasm +0 -0
  154. package/sdm/index.d.ts +0 -88
  155. package/sdm/index.js +0 -253
  156. package/sdm/manifest.json +0 -402
  157. package/stress/docker/Dockerfile +0 -11
  158. package/stress/docker/docker-compose.yml +0 -51
  159. package/stress/sds/aggregate.mjs +0 -97
  160. package/stress/sds/run.mjs +0 -105
  161. package/stress/sds/write-compose.mjs +0 -96
  162. package/wasm/flatsql-client.js +0 -280
  163. package/wasm/flatsql-io.d.ts +0 -97
  164. package/wasm/flatsql-io.js +0 -625
  165. package/wasm/flatsql-p4-threads.wasm +0 -0
  166. package/wasm/flatsql-ps-threads.wasm +0 -0
  167. package/wasm/flatsql-wasi-noeh.wasm +0 -0
  168. package/wasm/flatsql-wasi.wasm +0 -0
  169. package/wasm/flatsql.js +0 -2
  170. package/wasm/flatsql.worker.js +0 -209
  171. package/wasm/native/flatsql_wasmedge_runner.cpp +0 -858
  172. package/wasm/ps-node-fault.d.mts +0 -23
  173. package/wasm/ps-node-fault.mjs +0 -381
  174. package/wasm/ps-node-guest.mjs +0 -173
  175. package/wasm/ps-node-host.d.mts +0 -95
  176. package/wasm/ps-node-host.mjs +0 -476
  177. package/wasm/ps-node-imports.mjs +0 -847
  178. package/wasm/ps.d.ts +0 -21
  179. package/wasm/ps.js +0 -75
  180. package/wasm/wasi.d.ts +0 -11
  181. package/wasm/wasi.js +0 -51
@@ -1,23 +0,0 @@
1
- // Types for wasm/ps-node-fault.mjs (the Node host's fault overlay).
2
-
3
- export type PsCrashMode = 'dropAll' | 'pageSubset' | 'keepAll';
4
-
5
- export interface PsFaultOverlay {
6
- buffer: SharedArrayBuffer;
7
- /** Every later mutating call fails with EIO (the process is dying). */
8
- freeze(): void;
9
- stats(): { writes: number; syncs: number; pagesUsed: number; overflow: boolean };
10
- /** Apply a crash once every guest thread has stopped, then reset the overlay. */
11
- crash(
12
- mode: PsCrashMode,
13
- random?: () => number,
14
- ): { pagesUndone: number; pagesKept: number; filesTruncated: number; entriesDropped: number };
15
- }
16
-
17
- export const PAGE: number;
18
- export function createFaultOverlay(options?: { maxFiles?: number; maxPages?: number }): PsFaultOverlay;
19
- export function attachFaultOverlay(buffer: SharedArrayBuffer): PsFaultOverlay;
20
- export function createFaultInterposer(
21
- buffer: SharedArrayBuffer,
22
- root?: string,
23
- ): (op: string, fn: (...args: unknown[]) => unknown, args: unknown[]) => unknown;
@@ -1,381 +0,0 @@
1
- // FlatSQL partition store: the Node host's fault overlay (design §19,
2
- // 22.3a-7; docs/PARTITION-STORE-WASM.md).
3
- //
4
- // A SharedArrayBuffer page table every worker consults. It sits under the
5
- // SDK's Node sync-fs flatsql_io provider (its `interpose` hook) and records,
6
- // for every file, what a power loss could take away:
7
- // - the pre-image of every 4 KiB page written since the file's last sync,
8
- // and the size the file had at that sync;
9
- // - files created since their parent directory's last sync.
10
- // The supervisor freezes the overlay (every later mutating call fails with
11
- // EIO, as from a dying process), stops the guest, then applies a crash:
12
- // dropAll every unsynced page write and size change is undone, and
13
- // entries created since the parent's last sync disappear;
14
- // pageSubset each unsynced page independently survives or is undone
15
- // (writeback in any order), and the size is the synced or the
16
- // current one;
17
- // keepAll kill -9 with the page cache intact: nothing is undone.
18
- // Reads and syncs pass through. The overlay never touches data the guest has
19
- // synced: a record acked after its durable append must survive every mode.
20
-
21
- import fs from "node:fs";
22
- import path from "node:path";
23
-
24
- export const PAGE = 4096;
25
- const MAGIC = 0x46534c54; // "FSLT"
26
-
27
- // Header words.
28
- const H_MAGIC = 0;
29
- const H_FROZEN = 1;
30
- const H_OVERFLOW = 2;
31
- const H_MAX_FILES = 3;
32
- const H_MAX_PAGES = 4;
33
- const H_NEXT_PAGE = 5;
34
- const H_WRITES = 6;
35
- const H_SYNCS = 7;
36
- const H_WORDS = 16;
37
- // File words: state, lock, flags, head page entry, pathLen, pad x3.
38
- const F_WORDS = 8;
39
- const F_STATE = 0;
40
- const F_LOCK = 1;
41
- const F_FLAGS = 2;
42
- const F_HEAD = 3;
43
- const F_PATHLEN = 4;
44
- const F_EMPTY = 0;
45
- const F_CLAIM = 1;
46
- const F_LIVE = 2;
47
- const FLAG_TRACKED = 1; // synced size recorded
48
- const FLAG_NEW_ENTRY = 2; // created since the parent directory's last sync
49
- const MAX_PATH = 1024;
50
- // Page entry words: fileId, pageNo, preimage length, next.
51
- const P_WORDS = 4;
52
-
53
- const mix = (s) => {
54
- let h = 0x811c9dc5;
55
- for (let i = 0; i < s.length; i += 1) {
56
- h ^= s.charCodeAt(i);
57
- h = Math.imul(h, 0x01000193) >>> 0;
58
- }
59
- return h;
60
- };
61
-
62
- /**
63
- * Allocate an overlay. `maxPages` bounds the unsynced pages tracked at once
64
- * (4 KiB each); overflowing it is reported, never silently ignored.
65
- */
66
- export function createFaultOverlay({ maxFiles = 16384, maxPages = 65536 } = {}) {
67
- const bytes =
68
- H_WORDS * 4 + maxFiles * F_WORDS * 4 + maxFiles * 8 + maxFiles * MAX_PATH + maxPages * P_WORDS * 4 + maxPages * PAGE;
69
- const buffer = new SharedArrayBuffer(bytes);
70
- const h = new Int32Array(buffer, 0, H_WORDS);
71
- h[H_MAGIC] = MAGIC;
72
- h[H_MAX_FILES] = maxFiles;
73
- h[H_MAX_PAGES] = maxPages;
74
- const v = views(buffer);
75
- v.files.fill(0);
76
- for (let i = 0; i < maxFiles; i += 1) v.files[i * F_WORDS + F_HEAD] = -1;
77
- return { buffer, ...overlayApi(buffer) };
78
- }
79
-
80
- function views(buffer) {
81
- const h = new Int32Array(buffer, 0, H_WORDS);
82
- if (h[H_MAGIC] !== MAGIC) throw new TypeError("not a fault overlay");
83
- const maxFiles = h[H_MAX_FILES];
84
- const maxPages = h[H_MAX_PAGES];
85
- let off = H_WORDS * 4;
86
- const files = new Int32Array(buffer, off, maxFiles * F_WORDS);
87
- off += maxFiles * F_WORDS * 4;
88
- const syncedSize = new Float64Array(buffer, off, maxFiles);
89
- off += maxFiles * 8;
90
- const paths = new Uint8Array(buffer, off, maxFiles * MAX_PATH);
91
- off += maxFiles * MAX_PATH;
92
- const pages = new Int32Array(buffer, off, maxPages * P_WORDS);
93
- off += maxPages * P_WORDS * 4;
94
- const images = new Uint8Array(buffer, off, maxPages * PAGE);
95
- return { h, maxFiles, maxPages, files, syncedSize, paths, pages, images };
96
- }
97
-
98
- function lock(v, id) {
99
- const at = id * F_WORDS + F_LOCK;
100
- while (Atomics.compareExchange(v.files, at, 0, 1) !== 0) Atomics.wait(v.files, at, 1, 1);
101
- }
102
- function unlock(v, id) {
103
- const at = id * F_WORDS + F_LOCK;
104
- Atomics.store(v.files, at, 0);
105
- Atomics.notify(v.files, at, 1);
106
- }
107
-
108
- function pathOf(v, id) {
109
- const len = v.files[id * F_WORDS + F_PATHLEN];
110
- return Buffer.from(v.paths.slice(id * MAX_PATH, id * MAX_PATH + len)).toString("utf8");
111
- }
112
-
113
- // File id for a host path (open addressing), created on first sight.
114
- function fileId(v, full, create) {
115
- const bytes = Buffer.from(full);
116
- if (bytes.length > MAX_PATH) return -1;
117
- let i = mix(full) % v.maxFiles;
118
- for (let n = 0; n < v.maxFiles; n += 1, i = (i + 1) % v.maxFiles) {
119
- const at = i * F_WORDS;
120
- let state = Atomics.load(v.files, at + F_STATE);
121
- if (state === F_EMPTY) {
122
- if (!create) return -1;
123
- if (Atomics.compareExchange(v.files, at + F_STATE, F_EMPTY, F_CLAIM) === F_EMPTY) {
124
- v.paths.set(bytes, i * MAX_PATH);
125
- v.files[at + F_PATHLEN] = bytes.length;
126
- v.files[at + F_FLAGS] = 0;
127
- v.files[at + F_HEAD] = -1;
128
- Atomics.store(v.files, at + F_STATE, F_LIVE);
129
- Atomics.notify(v.files, at + F_STATE);
130
- return i;
131
- }
132
- state = Atomics.load(v.files, at + F_STATE);
133
- }
134
- while (state === F_CLAIM) {
135
- Atomics.wait(v.files, at + F_STATE, F_CLAIM, 1);
136
- state = Atomics.load(v.files, at + F_STATE);
137
- }
138
- if (v.files[at + F_PATHLEN] === bytes.length && pathOf(v, i) === full) return i;
139
- }
140
- Atomics.store(v.h, H_OVERFLOW, 1);
141
- return -1;
142
- }
143
-
144
- function sizeOf(fd) {
145
- return fs.fstatSync(fd).size;
146
- }
147
-
148
- // Record the synced size the first time a file is written after a sync.
149
- function track(v, id, fd) {
150
- const at = id * F_WORDS + F_FLAGS;
151
- if (!(v.files[at] & FLAG_TRACKED)) {
152
- v.syncedSize[id] = sizeOf(fd);
153
- v.files[at] |= FLAG_TRACKED;
154
- }
155
- }
156
-
157
- // Save the pre-image of page `p` unless it is already saved since the last
158
- // sync. Pages wholly past the synced size need none: a crash cuts the file
159
- // back to that size, or (pageSubset keeping the size) leaves new or zero bytes
160
- // there, both of which a real disk can produce.
161
- function savePage(v, id, fd, p) {
162
- if (p * PAGE >= v.syncedSize[id]) return true;
163
- for (let e = v.files[id * F_WORDS + F_HEAD]; e >= 0; e = v.pages[e * P_WORDS + 3]) {
164
- if (v.pages[e * P_WORDS + 1] === p) return true;
165
- }
166
- const e = Atomics.add(v.h, H_NEXT_PAGE, 1);
167
- if (e >= v.maxPages) {
168
- Atomics.store(v.h, H_OVERFLOW, 1);
169
- return false;
170
- }
171
- const img = v.images.subarray(e * PAGE, (e + 1) * PAGE);
172
- const tmp = Buffer.alloc(PAGE);
173
- const n = fs.readSync(fd, tmp, 0, PAGE, p * PAGE);
174
- img.set(tmp);
175
- v.pages[e * P_WORDS] = id;
176
- v.pages[e * P_WORDS + 1] = p;
177
- v.pages[e * P_WORDS + 2] = n;
178
- v.pages[e * P_WORDS + 3] = v.files[id * F_WORDS + F_HEAD];
179
- v.files[id * F_WORDS + F_HEAD] = e;
180
- return true;
181
- }
182
-
183
- function frozenError() {
184
- return Object.assign(new Error("fault overlay frozen"), { code: "EIO" });
185
- }
186
-
187
- /**
188
- * The `interpose` hook for one thread's SDK sync-fs provider. `root` is the
189
- * provider's root (paths it opens are absolute below it).
190
- */
191
- export function createFaultInterposer(buffer, _root) {
192
- const v = views(buffer);
193
- const fdFile = new Map(); // this thread's fds -> file id
194
- const mutating = new Set(["write", "truncate", "mkdir", "unlink"]);
195
- return (op, fn, args) => {
196
- if (
197
- Atomics.load(v.h, H_FROZEN) &&
198
- (mutating.has(op) || op === "sync" || op === "syncdir" ||
199
- (op === "open" && (args[1] & fs.constants.O_CREAT) !== 0))
200
- ) {
201
- throw frozenError();
202
- }
203
- if (op === "open") {
204
- const full = args[0];
205
- const creating = (args[1] & fs.constants.O_CREAT) !== 0;
206
- const existed = creating ? fs.existsSync(full) : true;
207
- const fd = fn(...args);
208
- const id = fileId(v, full, true);
209
- if (id >= 0) {
210
- fdFile.set(fd, id);
211
- if (creating && !existed) {
212
- lock(v, id);
213
- v.files[id * F_WORDS + F_FLAGS] = (v.files[id * F_WORDS + F_FLAGS] | FLAG_NEW_ENTRY) & ~FLAG_TRACKED;
214
- // A new file's synced content is nothing.
215
- v.syncedSize[id] = 0;
216
- v.files[id * F_WORDS + F_FLAGS] |= FLAG_TRACKED;
217
- unlock(v, id);
218
- }
219
- }
220
- return fd;
221
- }
222
- if (op === "close") {
223
- fdFile.delete(args[0]);
224
- return fn(...args);
225
- }
226
- if (op === "write") {
227
- const [fd, view, off, len, pos] = args;
228
- const id = fdFile.get(fd);
229
- if (id === undefined) return fn(...args);
230
- lock(v, id);
231
- try {
232
- track(v, id, fd);
233
- const first = Math.floor(pos / PAGE);
234
- const last = Math.floor((pos + len - 1) / PAGE);
235
- for (let p = first; p <= last; p += 1) savePage(v, id, fd, p);
236
- Atomics.add(v.h, H_WRITES, 1);
237
- return fn(fd, view, off, len, pos);
238
- } finally {
239
- unlock(v, id);
240
- }
241
- }
242
- if (op === "truncate") {
243
- const [fd, size] = args;
244
- const id = fdFile.get(fd);
245
- if (id === undefined) return fn(...args);
246
- lock(v, id);
247
- try {
248
- track(v, id, fd);
249
- const cur = sizeOf(fd);
250
- for (let p = Math.floor(size / PAGE); p * PAGE < cur; p += 1) savePage(v, id, fd, p);
251
- return fn(...args);
252
- } finally {
253
- unlock(v, id);
254
- }
255
- }
256
- if (op === "sync") {
257
- const [fd] = args;
258
- const id = fdFile.get(fd);
259
- const r = fn(...args);
260
- if (id !== undefined) {
261
- lock(v, id);
262
- // Everything written before this sync returned is durable.
263
- v.files[id * F_WORDS + F_HEAD] = -1;
264
- v.files[id * F_WORDS + F_FLAGS] &= ~FLAG_TRACKED;
265
- unlock(v, id);
266
- Atomics.add(v.h, H_SYNCS, 1);
267
- }
268
- return r;
269
- }
270
- if (op === "syncdir") {
271
- const r = fn(...args);
272
- const dir = path.resolve(args[0]);
273
- for (let id = 0; id < v.maxFiles; id += 1) {
274
- if (Atomics.load(v.files, id * F_WORDS + F_STATE) !== F_LIVE) continue;
275
- if (!(v.files[id * F_WORDS + F_FLAGS] & FLAG_NEW_ENTRY)) continue;
276
- if (path.dirname(pathOf(v, id)) !== dir) continue;
277
- lock(v, id);
278
- v.files[id * F_WORDS + F_FLAGS] &= ~FLAG_NEW_ENTRY;
279
- unlock(v, id);
280
- }
281
- return r;
282
- }
283
- if (op === "unlink") {
284
- const full = args[0];
285
- const r = fn(...args);
286
- const id = fileId(v, full, false);
287
- if (id >= 0) {
288
- lock(v, id);
289
- v.files[id * F_WORDS + F_FLAGS] = 0;
290
- v.files[id * F_WORDS + F_HEAD] = -1;
291
- unlock(v, id);
292
- }
293
- return r;
294
- }
295
- return fn(...args);
296
- };
297
- }
298
-
299
- function overlayApi(buffer) {
300
- const v = views(buffer);
301
- return {
302
- /** Every later mutating call fails with EIO (the process is dying). */
303
- freeze() {
304
- Atomics.store(v.h, H_FROZEN, 1);
305
- },
306
- stats() {
307
- return {
308
- writes: Atomics.load(v.h, H_WRITES),
309
- syncs: Atomics.load(v.h, H_SYNCS),
310
- pagesUsed: Math.min(Atomics.load(v.h, H_NEXT_PAGE), v.maxPages),
311
- overflow: Atomics.load(v.h, H_OVERFLOW) !== 0,
312
- };
313
- },
314
- /**
315
- * Apply a crash after every guest thread has stopped, then reset the
316
- * overlay (unfrozen, nothing tracked). `random` returns [0, 1).
317
- * @returns {{ pagesUndone: number, pagesKept: number, filesTruncated: number, entriesDropped: number }}
318
- */
319
- crash(mode, random = Math.random) {
320
- if (Atomics.load(v.h, H_OVERFLOW)) throw new Error("fault overlay overflowed: the crash would be incomplete");
321
- const out = { pagesUndone: 0, pagesKept: 0, filesTruncated: 0, entriesDropped: 0 };
322
- for (let id = 0; id < v.maxFiles; id += 1) {
323
- const at = id * F_WORDS;
324
- if (v.files[at + F_STATE] !== F_LIVE) continue;
325
- const flags = v.files[at + F_FLAGS];
326
- const full = pathOf(v, id);
327
- if (mode !== "keepAll" && flags & FLAG_NEW_ENTRY) {
328
- try {
329
- fs.unlinkSync(full);
330
- out.entriesDropped += 1;
331
- } catch {
332
- // already gone
333
- }
334
- continue;
335
- }
336
- if (mode === "keepAll" || !(flags & FLAG_TRACKED)) continue;
337
- let fd;
338
- try {
339
- fd = fs.openSync(full, "r+");
340
- } catch {
341
- continue; // unlinked durably since
342
- }
343
- try {
344
- for (let e = v.files[at + F_HEAD]; e >= 0; e = v.pages[e * P_WORDS + 3]) {
345
- const undo = mode === "dropAll" || random() < 0.5;
346
- if (!undo) {
347
- out.pagesKept += 1;
348
- continue;
349
- }
350
- const p = v.pages[e * P_WORDS + 1];
351
- const n = v.pages[e * P_WORDS + 2];
352
- if (n > 0) fs.writeSync(fd, v.images.subarray(e * PAGE, e * PAGE + n), 0, n, p * PAGE);
353
- out.pagesUndone += 1;
354
- }
355
- const synced = v.syncedSize[id];
356
- const size = fs.fstatSync(fd).size;
357
- const keepSize = mode === "pageSubset" && random() < 0.5;
358
- if (!keepSize && size !== synced) {
359
- fs.ftruncateSync(fd, synced);
360
- out.filesTruncated += 1;
361
- }
362
- } finally {
363
- fs.closeSync(fd);
364
- }
365
- }
366
- // Reset.
367
- for (let id = 0; id < v.maxFiles; id += 1) {
368
- v.files[id * F_WORDS + F_FLAGS] = 0;
369
- v.files[id * F_WORDS + F_HEAD] = -1;
370
- }
371
- Atomics.store(v.h, H_NEXT_PAGE, 0);
372
- Atomics.store(v.h, H_FROZEN, 0);
373
- return out;
374
- },
375
- };
376
- }
377
-
378
- /** Re-attach to an overlay buffer (for example in another thread). */
379
- export function attachFaultOverlay(buffer) {
380
- return { buffer, ...overlayApi(buffer) };
381
- }
@@ -1,173 +0,0 @@
1
- // FlatSQL partition store: a guest thread's worker under the Node host
2
- // (ps-node-host.mjs). Three modes:
3
- //
4
- // command the guest's own thread: runs `_start` (the wasm test commands)
5
- // and reports its exit;
6
- // reactor the exec thread of one flatsql-ps-threads.wasm instance: runs
7
- // `_initialize`, then serves the instance's C ABI calls (init,
8
- // start, stats, stop) posted by the supervisor;
9
- // pool a pooled thread for wasi.thread-spawn: waits on its slot of the
10
- // pool's SharedArrayBuffer, instantiates the module over the shared
11
- // memory, runs wasi_thread_start(tid, arg), and returns to idle.
12
- //
13
- // Every mode imports the same host (ps-node-imports.mjs), including
14
- // wasi.thread-spawn, so any guest thread may spawn threads. The supervisor
15
- // on the main thread starts pool workers and is never blocked by the guest.
16
-
17
- import { parentPort, threadId, workerData } from "node:worker_threads";
18
- import { writeSync } from "node:fs";
19
-
20
- import {
21
- CONTROL_EXITED,
22
- CONTROL_EXIT_CODE,
23
- CONTROL_FAULT,
24
- CONTROL_FAULT_TID,
25
- POOL_GEN,
26
- POOL_HEADER,
27
- POOL_MAX_RUNNING,
28
- POOL_RUNNING,
29
- POOL_SLOT_WORDS,
30
- SLOT_ASSIGNED,
31
- SLOT_DEAD,
32
- SLOT_IDLE,
33
- SLOT_RUNNING,
34
- WasiExitError,
35
- createThreadImports,
36
- poolThreadSpawn,
37
- registerThread,
38
- } from "./ps-node-imports.mjs";
39
-
40
- const { mode, wasmModule, memory, cfg } = workerData;
41
- // A pool worker counts as a guest OS thread once it runs a guest thread.
42
- let control = mode === "pool" ? new Int32Array(cfg.control) : registerThread(cfg.control);
43
- const pool = new Int32Array(cfg.pool);
44
-
45
- const built = createThreadImports(cfg, () => memory);
46
- const imports = {
47
- ...built.imports,
48
- env: { ...built.imports.env, memory },
49
- wasi: { "thread-spawn": poolThreadSpawn(cfg.pool, cfg.spawnWaitMs) },
50
- };
51
-
52
- const isExit = (error) => error instanceof WasiExitError || error?.name === "WasiExitError";
53
-
54
- function exitProcess(code) {
55
- if (Atomics.compareExchange(control, CONTROL_EXITED, 0, 1) === 0) {
56
- Atomics.store(control, CONTROL_EXIT_CODE, code);
57
- }
58
- Atomics.notify(control, CONTROL_EXITED);
59
- }
60
-
61
- function fault(tid, error) {
62
- // Written here, synchronously: the thread that would join this one may be
63
- // blocked in the guest forever.
64
- try {
65
- writeSync(2, `[ps-node-host] guest thread ${tid} trapped: ${error?.stack ?? error}\n`);
66
- } catch {
67
- // stderr gone
68
- }
69
- Atomics.store(control, CONTROL_FAULT_TID, tid);
70
- Atomics.store(control, CONTROL_FAULT, 1);
71
- Atomics.notify(control, CONTROL_FAULT);
72
- }
73
-
74
- if (mode === "pool") {
75
- const slot = POOL_HEADER + workerData.slot * POOL_SLOT_WORDS;
76
- Atomics.store(pool, slot + 3, threadId);
77
- let registered = false;
78
- for (;;) {
79
- Atomics.store(pool, slot, SLOT_IDLE);
80
- Atomics.add(pool, POOL_GEN, 1);
81
- Atomics.notify(pool, POOL_GEN);
82
- for (;;) {
83
- const state = Atomics.load(pool, slot);
84
- if (state === SLOT_ASSIGNED) break;
85
- Atomics.wait(pool, slot, state, 1000);
86
- }
87
- const tid = Atomics.load(pool, slot + 1);
88
- const arg = Atomics.load(pool, slot + 2);
89
- if (!registered) {
90
- registerThread(cfg.control);
91
- registered = true;
92
- }
93
- Atomics.store(pool, slot, SLOT_RUNNING);
94
- const running = Atomics.add(pool, POOL_RUNNING, 1) + 1;
95
- for (;;) {
96
- const max = Atomics.load(pool, POOL_MAX_RUNNING);
97
- if (running <= max || Atomics.compareExchange(pool, POOL_MAX_RUNNING, max, running) === max) break;
98
- }
99
- try {
100
- const instance = new WebAssembly.Instance(wasmModule, imports);
101
- instance.exports.wasi_thread_start(tid, arg);
102
- } catch (error) {
103
- Atomics.sub(pool, POOL_RUNNING, 1);
104
- Atomics.store(pool, slot, SLOT_DEAD);
105
- if (isExit(error)) exitProcess(error.code);
106
- else fault(tid, error);
107
- break;
108
- }
109
- Atomics.sub(pool, POOL_RUNNING, 1);
110
- }
111
- built.close();
112
- } else if (mode === "command") {
113
- const instance = new WebAssembly.Instance(wasmModule, imports);
114
- let code = 0;
115
- let error = null;
116
- try {
117
- instance.exports._start();
118
- } catch (e) {
119
- if (isExit(e)) code = e.code;
120
- else {
121
- code = 134;
122
- error = String(e?.stack ?? e);
123
- }
124
- }
125
- exitProcess(code);
126
- parentPort.postMessage({ t: "done", code: Atomics.load(control, CONTROL_EXIT_CODE), error });
127
- } else {
128
- const instance = new WebAssembly.Instance(wasmModule, imports);
129
- instance.exports._initialize?.();
130
- const mem = () => new Uint8Array(memory.buffer);
131
- parentPort.on("message", (msg) => {
132
- const reply = (value, err) =>
133
- parentPort.postMessage({ t: "reply", id: msg.id, value, error: err ? String(err?.stack ?? err) : null });
134
- try {
135
- if (msg.op === "call") {
136
- const fn = instance.exports[msg.fn];
137
- if (typeof fn !== "function") throw new Error(`no export ${msg.fn}`);
138
- reply(fn(...(msg.args ?? [])));
139
- } else if (msg.op === "write") {
140
- const n = msg.bytes.length;
141
- const ptr = instance.exports.flatsql_ps_alloc(Math.max(n, 1));
142
- if (!ptr) throw new Error("flatsql_ps_alloc failed");
143
- mem().set(msg.bytes, ptr);
144
- reply(ptr);
145
- } else if (msg.op === "read") {
146
- reply(mem().slice(msg.ptr, msg.ptr + msg.len));
147
- } else if (msg.op === "growHeap") {
148
- // The heap to msg.limitBytes of memory before any guest thread starts
149
- // (ps-node-host.mjs heapLimitBytes): allocate in steps from 1 GiB
150
- // down to 1 MiB until each fails, then free them all. The guest
151
- // allocator keeps the freed blocks; wasm memory never shrinks.
152
- const { flatsql_ps_alloc: alloc, flatsql_ps_free: free } = instance.exports;
153
- const held = [];
154
- for (let step = 2 ** 30; step >= 2 ** 20 && held.length < 64; ) {
155
- const p = memory.buffer.byteLength + step + 2 ** 20 <= msg.limitBytes ? alloc(step) : 0;
156
- if (!p) {
157
- step /= 2;
158
- continue;
159
- }
160
- held.push(p);
161
- }
162
- for (const p of held) free(p);
163
- reply(memory.buffer.byteLength);
164
- } else {
165
- throw new Error(`unknown op ${msg.op}`);
166
- }
167
- } catch (err) {
168
- if (isExit(err)) exitProcess(err.code);
169
- reply(undefined, err);
170
- }
171
- });
172
- parentPort.postMessage({ t: "ready" });
173
- }
@@ -1,95 +0,0 @@
1
- // Types for wasm/ps-node-host.mjs (the partition store's Node wasi-threads host).
2
-
3
- export interface PsThreadReport {
4
- /** wasi.thread-spawn calls that started a guest thread. */
5
- guestThreadsSpawned: number;
6
- /** Most guest threads running at once, the guest's own thread included. */
7
- maxConcurrentGuestThreads: number;
8
- runningGuestThreads: number;
9
- spawnsDeclined: number;
10
- /** Node worker threads (OS threads) that ran guest code. */
11
- workerThreads: number;
12
- distinctWorkerThreadIds: number;
13
- memoryBytes: number;
14
- /** Bytes the memory grew after the first guest thread ran (V8: a trap hazard). */
15
- memoryGrownWhileThreadsRan: number;
16
- }
17
-
18
- export interface PsMount {
19
- guest: string;
20
- host: string;
21
- }
22
-
23
- export interface PsHostOptions {
24
- /** Host directory the guest sees as "/" (WASI and flatsql_io). Default: a private temporary directory, removed afterwards. */
25
- root?: string;
26
- keepRoot?: boolean;
27
- /** Further preopened directories. */
28
- mounts?: PsMount[];
29
- env?: Record<string, string>;
30
- /** Pool cap on guest threads (default 512). */
31
- maxThreads?: number;
32
- /** Pool workers started up front (default 16). */
33
- warmThreads?: number;
34
- /** How long a spawn waits for a free worker before failing with EAGAIN (default 60000). */
35
- spawnWaitMs?: number;
36
- /** Fault overlay (ps-node-fault.mjs createFaultOverlay). */
37
- fault?: { buffer: SharedArrayBuffer };
38
- stdoutFd?: number;
39
- stderrFd?: number;
40
- /** Seeded random_get. */
41
- deterministic?: { randomSeed?: bigint };
42
- initialPages?: number;
43
- maximumPages?: number;
44
- /**
45
- * createPsNodeInstance grows the guest heap to the memory's maximum before
46
- * any guest thread starts (V8 can trap a thread touching memory another
47
- * thread has just grown). false: no pre-grow. heapBytes caps the growth.
48
- */
49
- growHeap?: boolean;
50
- heapBytes?: number;
51
- maxHandles?: number;
52
- instanceId?: number;
53
- }
54
-
55
- export interface PsCommandOptions extends PsHostOptions {
56
- args?: string[];
57
- timeoutMs?: number;
58
- /** Kill the guest after this long: a crash (result.killed), not a failure. */
59
- killAfterMs?: number;
60
- beforeKill?: () => void;
61
- }
62
-
63
- export interface PsCommandResult {
64
- exitCode: number;
65
- killed: boolean;
66
- timedOut: boolean;
67
- fault: boolean;
68
- error: string | null;
69
- elapsedMs: number;
70
- threads: PsThreadReport;
71
- root: string;
72
- poolErrors: string[];
73
- }
74
-
75
- export function runPsCommand(wasm: string | Uint8Array, options?: PsCommandOptions): Promise<PsCommandResult>;
76
-
77
- export interface PsNodeInstance {
78
- memory: WebAssembly.Memory;
79
- root: string;
80
- /** Call an export of the instance on its exec thread. */
81
- call(fn: string, ...args: number[]): Promise<number>;
82
- /** Copy bytes into instance memory (flatsql_ps_alloc); returns the pointer. */
83
- write(bytes: Uint8Array | ArrayBuffer): Promise<number>;
84
- read(ptr: number, len: number): Promise<Uint8Array>;
85
- threads(): PsThreadReport;
86
- faulted(): boolean;
87
- close(): Promise<void>;
88
- }
89
-
90
- /** flatsql-ps-threads.wasm as one instance (writer or reader) with its own exec thread and thread pool. */
91
- export function createPsNodeInstance(wasm: string | Uint8Array, options?: PsHostOptions): Promise<PsNodeInstance>;
92
-
93
- export function importedMemoryLimits(
94
- bytes: Uint8Array | ArrayBuffer,
95
- ): { module: string; name: string; initial: number; maximum?: number; shared: boolean } | null;