@minnowdb/core 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (276) hide show
  1. package/README.md +4 -2
  2. package/dist/block-format/block.d.ts +32 -13
  3. package/dist/block-format/block.js +179 -55
  4. package/dist/block-format/checksum.d.ts +2 -1
  5. package/dist/block-format/checksum.js +5 -2
  6. package/dist/block-format/codecs.d.ts +7 -2
  7. package/dist/block-format/codecs.js +53 -12
  8. package/dist/block-format/column.d.ts +3 -2
  9. package/dist/block-format/column.js +96 -30
  10. package/dist/block-format/index.d.ts +2 -2
  11. package/dist/block-format/index.js +2 -2
  12. package/dist/block-format/physical.d.ts +8 -2
  13. package/dist/block-format/physical.js +14 -6
  14. package/dist/block-format/types.d.ts +7 -3
  15. package/dist/block-format/types.js +0 -1
  16. package/dist/block-format/unicode.d.ts +11 -0
  17. package/dist/block-format/unicode.js +46 -0
  18. package/dist/date-value.d.ts +15 -0
  19. package/dist/date-value.js +64 -0
  20. package/dist/engine/artifact-cache.d.ts +0 -1
  21. package/dist/engine/artifact-cache.js +0 -1
  22. package/dist/engine/batch.d.ts +5 -2
  23. package/dist/engine/batch.js +0 -1
  24. package/dist/engine/buffered-writer.d.ts +2 -1
  25. package/dist/engine/buffered-writer.js +34 -9
  26. package/dist/engine/cache-limits.d.ts +25 -0
  27. package/dist/engine/cache-limits.js +25 -0
  28. package/dist/engine/catalog.d.ts +5 -6
  29. package/dist/engine/catalog.js +5 -5
  30. package/dist/engine/client.d.ts +27 -7
  31. package/dist/engine/client.js +197 -31
  32. package/dist/engine/database.d.ts +161 -38
  33. package/dist/engine/database.js +4320 -1523
  34. package/dist/engine/defaults.d.ts +0 -1
  35. package/dist/engine/defaults.js +2 -4
  36. package/dist/engine/errors.d.ts +19 -1
  37. package/dist/engine/errors.js +32 -2
  38. package/dist/engine/fts.d.ts +0 -6
  39. package/dist/engine/fts.js +39 -14
  40. package/dist/engine/group-index.d.ts +0 -1
  41. package/dist/engine/group-index.js +6 -11
  42. package/dist/engine/index.d.ts +16 -11
  43. package/dist/engine/index.js +12 -10
  44. package/dist/engine/join-index.d.ts +0 -1
  45. package/dist/engine/join-index.js +2 -2
  46. package/dist/engine/keyed-live.d.ts +57 -0
  47. package/dist/engine/keyed-live.js +226 -0
  48. package/dist/engine/live-api.d.ts +4 -0
  49. package/dist/engine/live-api.js +4 -0
  50. package/dist/engine/live.d.ts +43 -26
  51. package/dist/engine/live.js +499 -129
  52. package/dist/engine/memory.d.ts +2 -1
  53. package/dist/engine/memory.js +3 -2
  54. package/dist/engine/optimizer.d.ts +0 -1
  55. package/dist/engine/optimizer.js +4 -4
  56. package/dist/engine/query-api.d.ts +3 -0
  57. package/dist/engine/query-api.js +3 -0
  58. package/dist/engine/query-cache.d.ts +0 -1
  59. package/dist/engine/query-cache.js +4 -4
  60. package/dist/engine/query.d.ts +4 -7
  61. package/dist/engine/query.js +348 -155
  62. package/dist/engine/result-wire.d.ts +0 -1
  63. package/dist/engine/result-wire.js +2 -2
  64. package/dist/engine/schema-wire.d.ts +1 -2
  65. package/dist/engine/schema-wire.js +0 -1
  66. package/dist/engine/schema.d.ts +3 -8
  67. package/dist/engine/schema.js +41 -24
  68. package/dist/engine/sort-keys.d.ts +0 -1
  69. package/dist/engine/sort-keys.js +2 -2
  70. package/dist/engine/sql-domains.d.ts +6 -1
  71. package/dist/engine/sql-domains.js +209 -45
  72. package/dist/engine/sql-driver.d.ts +2 -2
  73. package/dist/engine/sql-driver.js +0 -1
  74. package/dist/engine/sql-json.d.ts +0 -1
  75. package/dist/engine/sql-json.js +53 -7
  76. package/dist/engine/sql-semantics.d.ts +8 -5
  77. package/dist/engine/sql-semantics.js +427 -65
  78. package/dist/engine/typed-live.d.ts +67 -0
  79. package/dist/engine/typed-live.js +349 -0
  80. package/dist/engine/vector.d.ts +7 -1
  81. package/dist/engine/vector.js +202 -58
  82. package/dist/engine/worker-host.d.ts +8 -2
  83. package/dist/engine/worker-host.js +616 -152
  84. package/dist/engine/worker.d.ts +0 -1
  85. package/dist/engine/worker.js +3 -3
  86. package/dist/engine/write-block-planner.d.ts +18 -0
  87. package/dist/engine/write-block-planner.js +112 -0
  88. package/dist/index.d.ts +0 -1
  89. package/dist/index.js +0 -1
  90. package/dist/plan/index.d.ts +0 -1
  91. package/dist/plan/index.js +0 -1
  92. package/dist/storage/index.d.ts +1 -1
  93. package/dist/storage/index.js +1 -1
  94. package/dist/storage/indexeddb.d.ts +74 -65
  95. package/dist/storage/indexeddb.js +12301 -2076
  96. package/dist/storage/memory.d.ts +56 -66
  97. package/dist/storage/memory.js +1004 -139
  98. package/dist/storage/opfs/files.d.ts +18 -7
  99. package/dist/storage/opfs/files.js +115 -28
  100. package/dist/storage/opfs/index.d.ts +1 -1
  101. package/dist/storage/opfs/index.js +1 -1
  102. package/dist/storage/opfs/leader.d.ts +403 -91
  103. package/dist/storage/opfs/leader.js +4306 -602
  104. package/dist/storage/opfs/rpc.d.ts +15 -3
  105. package/dist/storage/opfs/rpc.js +223 -2
  106. package/dist/storage/opfs/snapshot-ledger.d.ts +40 -0
  107. package/dist/storage/opfs/snapshot-ledger.js +281 -0
  108. package/dist/storage/opfs/store.d.ts +26 -121
  109. package/dist/storage/opfs/store.js +436 -388
  110. package/dist/storage/persistence.d.ts +37 -0
  111. package/dist/storage/persistence.js +78 -0
  112. package/dist/storage/snapshot-stream.d.ts +24 -0
  113. package/dist/storage/snapshot-stream.js +897 -0
  114. package/dist/storage/snapshot.d.ts +9 -99
  115. package/dist/storage/snapshot.js +33 -309
  116. package/dist/storage/toolkit/extents.d.ts +41 -7
  117. package/dist/storage/toolkit/extents.js +402 -39
  118. package/dist/storage/toolkit/index.d.ts +3 -4
  119. package/dist/storage/toolkit/index.js +27 -3
  120. package/dist/storage/toolkit/record-core.d.ts +174 -61
  121. package/dist/storage/toolkit/record-core.js +5401 -849
  122. package/dist/storage/toolkit/sync-file.d.ts +10 -1
  123. package/dist/storage/toolkit/sync-file.js +50 -2
  124. package/dist/storage/toolkit/wal.d.ts +23 -5
  125. package/dist/storage/toolkit/wal.js +89 -27
  126. package/dist/storage/toolkit/wire.d.ts +4 -6
  127. package/dist/storage/toolkit/wire.js +80 -25
  128. package/dist/storage/types.d.ts +1271 -218
  129. package/dist/storage/types.js +1471 -138
  130. package/dist/testing/block-store-conformance.d.ts +0 -1
  131. package/dist/testing/block-store-conformance.js +877 -138
  132. package/dist/testing/index.d.ts +42 -31
  133. package/dist/testing/index.js +100 -68
  134. package/dist/testing/opfs-shim.d.ts +13 -3
  135. package/dist/testing/opfs-shim.js +40 -6
  136. package/dist/testing/simulator.d.ts +0 -1
  137. package/dist/testing/simulator.js +43 -12
  138. package/dist/testing/sqllogictest.d.ts +0 -1
  139. package/dist/testing/sqllogictest.js +0 -1
  140. package/dist/transactions/index.d.ts +81 -20
  141. package/dist/transactions/index.js +1050 -268
  142. package/dist/worker-protocol/index.d.ts +3 -2
  143. package/dist/worker-protocol/index.js +2 -5
  144. package/package.json +49 -1
  145. package/dist/block-format/block.d.ts.map +0 -1
  146. package/dist/block-format/block.js.map +0 -1
  147. package/dist/block-format/checksum.d.ts.map +0 -1
  148. package/dist/block-format/checksum.js.map +0 -1
  149. package/dist/block-format/codecs.d.ts.map +0 -1
  150. package/dist/block-format/codecs.js.map +0 -1
  151. package/dist/block-format/column.d.ts.map +0 -1
  152. package/dist/block-format/column.js.map +0 -1
  153. package/dist/block-format/index.d.ts.map +0 -1
  154. package/dist/block-format/index.js.map +0 -1
  155. package/dist/block-format/physical.d.ts.map +0 -1
  156. package/dist/block-format/physical.js.map +0 -1
  157. package/dist/block-format/types.d.ts.map +0 -1
  158. package/dist/block-format/types.js.map +0 -1
  159. package/dist/engine/artifact-cache.d.ts.map +0 -1
  160. package/dist/engine/artifact-cache.js.map +0 -1
  161. package/dist/engine/batch.d.ts.map +0 -1
  162. package/dist/engine/batch.js.map +0 -1
  163. package/dist/engine/buffered-writer.d.ts.map +0 -1
  164. package/dist/engine/buffered-writer.js.map +0 -1
  165. package/dist/engine/catalog.d.ts.map +0 -1
  166. package/dist/engine/catalog.js.map +0 -1
  167. package/dist/engine/client.d.ts.map +0 -1
  168. package/dist/engine/client.js.map +0 -1
  169. package/dist/engine/coordinator.d.ts +0 -17
  170. package/dist/engine/coordinator.d.ts.map +0 -1
  171. package/dist/engine/coordinator.js +0 -60
  172. package/dist/engine/coordinator.js.map +0 -1
  173. package/dist/engine/database.d.ts.map +0 -1
  174. package/dist/engine/database.js.map +0 -1
  175. package/dist/engine/defaults.d.ts.map +0 -1
  176. package/dist/engine/defaults.js.map +0 -1
  177. package/dist/engine/errors.d.ts.map +0 -1
  178. package/dist/engine/errors.js.map +0 -1
  179. package/dist/engine/fts.d.ts.map +0 -1
  180. package/dist/engine/fts.js.map +0 -1
  181. package/dist/engine/group-index.d.ts.map +0 -1
  182. package/dist/engine/group-index.js.map +0 -1
  183. package/dist/engine/index.d.ts.map +0 -1
  184. package/dist/engine/index.js.map +0 -1
  185. package/dist/engine/join-index.d.ts.map +0 -1
  186. package/dist/engine/join-index.js.map +0 -1
  187. package/dist/engine/live.d.ts.map +0 -1
  188. package/dist/engine/live.js.map +0 -1
  189. package/dist/engine/memory.d.ts.map +0 -1
  190. package/dist/engine/memory.js.map +0 -1
  191. package/dist/engine/optimizer.d.ts.map +0 -1
  192. package/dist/engine/optimizer.js.map +0 -1
  193. package/dist/engine/query-cache.d.ts.map +0 -1
  194. package/dist/engine/query-cache.js.map +0 -1
  195. package/dist/engine/query.d.ts.map +0 -1
  196. package/dist/engine/query.js.map +0 -1
  197. package/dist/engine/result-wire.d.ts.map +0 -1
  198. package/dist/engine/result-wire.js.map +0 -1
  199. package/dist/engine/schema-wire.d.ts.map +0 -1
  200. package/dist/engine/schema-wire.js.map +0 -1
  201. package/dist/engine/schema.d.ts.map +0 -1
  202. package/dist/engine/schema.js.map +0 -1
  203. package/dist/engine/sort-keys.d.ts.map +0 -1
  204. package/dist/engine/sort-keys.js.map +0 -1
  205. package/dist/engine/sql-domains.d.ts.map +0 -1
  206. package/dist/engine/sql-domains.js.map +0 -1
  207. package/dist/engine/sql-driver.d.ts.map +0 -1
  208. package/dist/engine/sql-driver.js.map +0 -1
  209. package/dist/engine/sql-json.d.ts.map +0 -1
  210. package/dist/engine/sql-json.js.map +0 -1
  211. package/dist/engine/sql-semantics.d.ts.map +0 -1
  212. package/dist/engine/sql-semantics.js.map +0 -1
  213. package/dist/engine/vector.d.ts.map +0 -1
  214. package/dist/engine/vector.js.map +0 -1
  215. package/dist/engine/worker-host.d.ts.map +0 -1
  216. package/dist/engine/worker-host.js.map +0 -1
  217. package/dist/engine/worker.d.ts.map +0 -1
  218. package/dist/engine/worker.js.map +0 -1
  219. package/dist/index.d.ts.map +0 -1
  220. package/dist/index.js.map +0 -1
  221. package/dist/plan/index.d.ts.map +0 -1
  222. package/dist/plan/index.js.map +0 -1
  223. package/dist/storage/fixture-shape.d.ts +0 -42
  224. package/dist/storage/fixture-shape.d.ts.map +0 -1
  225. package/dist/storage/fixture-shape.js +0 -146
  226. package/dist/storage/fixture-shape.js.map +0 -1
  227. package/dist/storage/index.d.ts.map +0 -1
  228. package/dist/storage/index.js.map +0 -1
  229. package/dist/storage/indexeddb.d.ts.map +0 -1
  230. package/dist/storage/indexeddb.js.map +0 -1
  231. package/dist/storage/memory.d.ts.map +0 -1
  232. package/dist/storage/memory.js.map +0 -1
  233. package/dist/storage/opfs/files.d.ts.map +0 -1
  234. package/dist/storage/opfs/files.js.map +0 -1
  235. package/dist/storage/opfs/index.d.ts.map +0 -1
  236. package/dist/storage/opfs/index.js.map +0 -1
  237. package/dist/storage/opfs/leader.d.ts.map +0 -1
  238. package/dist/storage/opfs/leader.js.map +0 -1
  239. package/dist/storage/opfs/rpc.d.ts.map +0 -1
  240. package/dist/storage/opfs/rpc.js.map +0 -1
  241. package/dist/storage/opfs/store.d.ts.map +0 -1
  242. package/dist/storage/opfs/store.js.map +0 -1
  243. package/dist/storage/snapshot.d.ts.map +0 -1
  244. package/dist/storage/snapshot.js.map +0 -1
  245. package/dist/storage/toolkit/extents.d.ts.map +0 -1
  246. package/dist/storage/toolkit/extents.js.map +0 -1
  247. package/dist/storage/toolkit/index.d.ts.map +0 -1
  248. package/dist/storage/toolkit/index.js.map +0 -1
  249. package/dist/storage/toolkit/record-core.d.ts.map +0 -1
  250. package/dist/storage/toolkit/record-core.js.map +0 -1
  251. package/dist/storage/toolkit/sync-file.d.ts.map +0 -1
  252. package/dist/storage/toolkit/sync-file.js.map +0 -1
  253. package/dist/storage/toolkit/wal.d.ts.map +0 -1
  254. package/dist/storage/toolkit/wal.js.map +0 -1
  255. package/dist/storage/toolkit/wire.d.ts.map +0 -1
  256. package/dist/storage/toolkit/wire.js.map +0 -1
  257. package/dist/storage/types.d.ts.map +0 -1
  258. package/dist/storage/types.js.map +0 -1
  259. package/dist/testing/block-store-conformance.d.ts.map +0 -1
  260. package/dist/testing/block-store-conformance.js.map +0 -1
  261. package/dist/testing/index.d.ts.map +0 -1
  262. package/dist/testing/index.js.map +0 -1
  263. package/dist/testing/opfs-shim.d.ts.map +0 -1
  264. package/dist/testing/opfs-shim.js.map +0 -1
  265. package/dist/testing/seeds.d.ts +0 -11
  266. package/dist/testing/seeds.d.ts.map +0 -1
  267. package/dist/testing/seeds.js +0 -50
  268. package/dist/testing/seeds.js.map +0 -1
  269. package/dist/testing/simulator.d.ts.map +0 -1
  270. package/dist/testing/simulator.js.map +0 -1
  271. package/dist/testing/sqllogictest.d.ts.map +0 -1
  272. package/dist/testing/sqllogictest.js.map +0 -1
  273. package/dist/transactions/index.d.ts.map +0 -1
  274. package/dist/transactions/index.js.map +0 -1
  275. package/dist/worker-protocol/index.d.ts.map +0 -1
  276. package/dist/worker-protocol/index.js.map +0 -1
@@ -26,4 +26,13 @@ export interface SyncFileHandle {
26
26
  /** Releases the handle (and any exclusive lock the platform ties to it). */
27
27
  close(): void;
28
28
  }
29
- //# sourceMappingURL=sync-file.d.ts.map
29
+ /**
30
+ * Writes the complete buffer, retrying the short transfers permitted by sync-access handles.
31
+ * The common path is one `write` call; retries pass subarray views and never copy payload bytes.
32
+ */
33
+ export declare function writeFully(handle: SyncFileHandle, buffer: Uint8Array, at: number, operation?: string): void;
34
+ /**
35
+ * Reads exactly `buffer.byteLength` bytes, retrying short transfers without copying data. A zero
36
+ * transfer before the requested range is complete is an unexpected EOF, not a successful read.
37
+ */
38
+ export declare function readFully(handle: SyncFileHandle, buffer: Uint8Array, at: number, operation?: string): void;
@@ -1,2 +1,50 @@
1
- export {};
2
- //# sourceMappingURL=sync-file.js.map
1
+ /**
2
+ * Writes the complete buffer, retrying the short transfers permitted by sync-access handles.
3
+ * The common path is one `write` call; retries pass subarray views and never copy payload bytes.
4
+ */
5
+ export function writeFully(handle, buffer, at, operation = "writing file bytes") {
6
+ assertTransferRange(at, buffer.byteLength, "write", operation);
7
+ let transferred = 0;
8
+ while (transferred < buffer.byteLength) {
9
+ const remaining = buffer.byteLength - transferred;
10
+ const chunk = transferred === 0 ? buffer : buffer.subarray(transferred);
11
+ const count = handle.write(chunk, { at: at + transferred });
12
+ assertTransferCount(count, remaining, "write", operation, at + transferred);
13
+ if (count === 0) {
14
+ throw new Error(`OPFS write made no progress while ${operation}: wrote ${String(transferred)} of ` +
15
+ `${String(buffer.byteLength)} bytes (stalled at file offset ${String(at + transferred)})`);
16
+ }
17
+ transferred += count;
18
+ }
19
+ }
20
+ /**
21
+ * Reads exactly `buffer.byteLength` bytes, retrying short transfers without copying data. A zero
22
+ * transfer before the requested range is complete is an unexpected EOF, not a successful read.
23
+ */
24
+ export function readFully(handle, buffer, at, operation = "reading file bytes") {
25
+ assertTransferRange(at, buffer.byteLength, "read", operation);
26
+ let transferred = 0;
27
+ while (transferred < buffer.byteLength) {
28
+ const remaining = buffer.byteLength - transferred;
29
+ const chunk = transferred === 0 ? buffer : buffer.subarray(transferred);
30
+ const count = handle.read(chunk, { at: at + transferred });
31
+ assertTransferCount(count, remaining, "read", operation, at + transferred);
32
+ if (count === 0) {
33
+ throw new Error(`Unexpected EOF while ${operation}: read ${String(transferred)} of ` +
34
+ `${String(buffer.byteLength)} bytes (EOF at file offset ${String(at + transferred)})`);
35
+ }
36
+ transferred += count;
37
+ }
38
+ }
39
+ function assertTransferRange(at, length, kind, operation) {
40
+ if (!Number.isSafeInteger(at) || at < 0 || length > Number.MAX_SAFE_INTEGER - at) {
41
+ throw new RangeError(`Invalid OPFS ${kind} range while ${operation}: ${String(at)}+${String(length)} ` +
42
+ `must be a non-negative safe-integer byte range`);
43
+ }
44
+ }
45
+ function assertTransferCount(count, remaining, kind, operation, at) {
46
+ if (!Number.isSafeInteger(count) || count < 0 || count > remaining) {
47
+ throw new Error(`Invalid OPFS ${kind} result while ${operation}: handle reported ${String(count)} bytes ` +
48
+ `for a ${String(remaining)}-byte request at file offset ${String(at)}`);
49
+ }
50
+ }
@@ -1,4 +1,15 @@
1
- import type { SyncFileHandle } from "./sync-file.js";
1
+ import { type SyncFileHandle } from "./sync-file.js";
2
+ /**
3
+ * A WAL frame is control data, never bulk block data. Bounding it prevents a corrupt length
4
+ * field (or an accidentally enormous catalog mutation) from turning recovery into an
5
+ * unbounded allocation. Snapshot/block payloads live in extents, so 64 MiB leaves generous
6
+ * headroom for legitimate schema and transaction records.
7
+ */
8
+ export declare const MAX_WAL_FRAME_BYTES: number;
9
+ export interface ReplayedWalFrame {
10
+ payload: unknown;
11
+ frameEnd: number;
12
+ }
2
13
  export declare class WalWriter {
3
14
  #private;
4
15
  constructor(handle: SyncFileHandle, offset: number);
@@ -11,12 +22,19 @@ export declare class WalWriter {
11
22
  close(): void;
12
23
  }
13
24
  /**
14
- * Reads every whole, checksum-valid frame from the handle's current content, in order,
15
- * stopping at the first torn or foreign bytes. Returns the payloads and the byte offset where
16
- * appending should resume (the end of the last valid frame — a torn tail is overwritten).
25
+ * Reads every whole, checksum-valid frame from the handle's current content, in order. A
26
+ * truncated final frame is ignored; complete foreign or corrupt bytes are rejected. Returns the
27
+ * payloads and the byte offset where appending should resume (the end of the last valid frame —
28
+ * a truncated tail is overwritten).
17
29
  */
18
30
  export declare function replayWalFrames(handle: SyncFileHandle): {
19
31
  payloads: unknown[];
32
+ /** End offset of each corresponding payload frame. */
33
+ frameEnds: number[];
20
34
  endOffset: number;
21
35
  };
22
- //# sourceMappingURL=wal.d.ts.map
36
+ /**
37
+ * Streams checksum-valid frames with O(max-frame) memory. A truncated final frame is ignored;
38
+ * complete foreign or checksum-invalid bytes are corruption and fail closed.
39
+ */
40
+ export declare function iterateWalFrames(handle: SyncFileHandle): Generator<ReplayedWalFrame>;
@@ -1,21 +1,34 @@
1
1
  import { crc32 } from "../../block-format/index.js";
2
2
  import { decodeRecordJson, encodeRecordJson } from "./wire.js";
3
+ import { readFully, writeFully } from "./sync-file.js";
3
4
  /**
4
5
  * The write-ahead log, framed over one synchronous file handle its single writer holds open.
5
6
  *
6
7
  * Frame layout, little-endian: `u32 magic | u32 payloadLength | u32 crc32(payload) | payload`.
7
- * An append is a single synchronous write at the tail — measured at microseconds on a held
8
- * handle and the only artifact a crash can leave is a torn final frame, which fails its
9
- * checksum and reads as "not written". Frames carry a global sequence number inside the
10
- * payload; the file is truncated to zero only after a checkpoint covering every frame has
11
- * been flushed, so replay is always newest-checkpoint-plus-tail.
8
+ * An append is one complete synchronous transfer at the tail — normally one microsecond-scale
9
+ * write on a held handle, with retries if the platform reports a short transfer. The only
10
+ * artifact a crash can leave is a truncated final frame, which reads as "not written". A
11
+ * complete frame with foreign marker or checksum-invalid bytes is corruption and fails closed.
12
+ * Frames carry a global sequence number inside the payload; the file is truncated to zero only
13
+ * after a checkpoint covering every frame has been flushed, so replay is always
14
+ * newest-checkpoint-plus-tail.
12
15
  */
13
16
  const FRAME_MAGIC = 0x4c574e4d; // "MNWL"
14
17
  const FRAME_HEADER_BYTES = 12;
18
+ /**
19
+ * A WAL frame is control data, never bulk block data. Bounding it prevents a corrupt length
20
+ * field (or an accidentally enormous catalog mutation) from turning recovery into an
21
+ * unbounded allocation. Snapshot/block payloads live in extents, so 64 MiB leaves generous
22
+ * headroom for legitimate schema and transaction records.
23
+ */
24
+ export const MAX_WAL_FRAME_BYTES = 64 * 1024 * 1024;
15
25
  export class WalWriter {
16
26
  #handle;
17
27
  #offset;
18
28
  constructor(handle, offset) {
29
+ if (!Number.isSafeInteger(offset) || offset < 0) {
30
+ throw new RangeError("WAL offset is outside the safe integer range");
31
+ }
19
32
  this.#handle = handle;
20
33
  this.#offset = offset;
21
34
  }
@@ -25,21 +38,47 @@ export class WalWriter {
25
38
  /** Appends one frame; with `flush`, durable before return. Synchronous — never yields. */
26
39
  append(payload, flush) {
27
40
  const bytes = encodeRecordJson(payload);
41
+ if (bytes.byteLength > MAX_WAL_FRAME_BYTES) {
42
+ throw new RangeError(`WAL frame payload exceeds the ${String(MAX_WAL_FRAME_BYTES)} byte limit: ` +
43
+ String(bytes.byteLength));
44
+ }
28
45
  const frame = new Uint8Array(FRAME_HEADER_BYTES + bytes.byteLength);
46
+ if (this.#offset > Number.MAX_SAFE_INTEGER - frame.byteLength) {
47
+ throw new RangeError("WAL offset exceeds the safe integer range");
48
+ }
29
49
  const view = new DataView(frame.buffer);
30
50
  view.setUint32(0, FRAME_MAGIC, true);
31
51
  view.setUint32(4, bytes.byteLength, true);
32
52
  view.setUint32(8, crc32(bytes), true);
33
53
  frame.set(bytes, FRAME_HEADER_BYTES);
34
- this.#handle.write(frame, { at: this.#offset });
35
- if (flush)
36
- this.#handle.flush();
37
- this.#offset += frame.byteLength;
54
+ const start = this.#offset;
55
+ try {
56
+ writeFully(this.#handle, frame, start, "appending a WAL frame");
57
+ if (flush)
58
+ this.#handle.flush();
59
+ this.#offset = start + frame.byteLength;
60
+ }
61
+ catch (error) {
62
+ // A complete write followed by a failed flush is not an acknowledged frame either.
63
+ // Remove it before unpublished extent bytes are rolled back. If truncation itself
64
+ // fails, recovery will fail closed when it verifies the frame's referenced payloads.
65
+ try {
66
+ this.#handle.truncate(start);
67
+ }
68
+ catch {
69
+ // Preserve the original quota/I/O error.
70
+ }
71
+ throw error;
72
+ }
38
73
  }
39
74
  /** Empties the log after a flushed checkpoint has covered every frame in it. */
40
75
  reset() {
41
76
  this.#handle.truncate(0);
77
+ // truncate() already changed the handle's logical file position. Keep the writer aligned
78
+ // with it even if the durability flush is refused; a later append must start at zero rather
79
+ // than leave a sparse, unreplayable gap after the checkpoint.
42
80
  this.#offset = 0;
81
+ this.#handle.flush();
43
82
  }
44
83
  flush() {
45
84
  this.#handle.flush();
@@ -49,31 +88,54 @@ export class WalWriter {
49
88
  }
50
89
  }
51
90
  /**
52
- * Reads every whole, checksum-valid frame from the handle's current content, in order,
53
- * stopping at the first torn or foreign bytes. Returns the payloads and the byte offset where
54
- * appending should resume (the end of the last valid frame — a torn tail is overwritten).
91
+ * Reads every whole, checksum-valid frame from the handle's current content, in order. A
92
+ * truncated final frame is ignored; complete foreign or corrupt bytes are rejected. Returns the
93
+ * payloads and the byte offset where appending should resume (the end of the last valid frame —
94
+ * a truncated tail is overwritten).
55
95
  */
56
96
  export function replayWalFrames(handle) {
57
- const size = handle.getSize();
58
- const bytes = new Uint8Array(size);
59
- handle.read(bytes, { at: 0 });
60
- const view = new DataView(bytes.buffer);
61
97
  const payloads = [];
98
+ const frameEnds = [];
99
+ let endOffset = 0;
100
+ for (const frame of iterateWalFrames(handle)) {
101
+ payloads.push(frame.payload);
102
+ frameEnds.push(frame.frameEnd);
103
+ endOffset = frame.frameEnd;
104
+ }
105
+ return { payloads, frameEnds, endOffset };
106
+ }
107
+ /**
108
+ * Streams checksum-valid frames with O(max-frame) memory. A truncated final frame is ignored;
109
+ * complete foreign or checksum-invalid bytes are corruption and fail closed.
110
+ */
111
+ export function* iterateWalFrames(handle) {
112
+ const size = handle.getSize();
113
+ if (!Number.isSafeInteger(size) || size < 0) {
114
+ throw new Error(`Invalid WAL byte length: ${String(size)}`);
115
+ }
116
+ const header = new Uint8Array(FRAME_HEADER_BYTES);
117
+ const headerView = new DataView(header.buffer);
62
118
  let offset = 0;
63
119
  while (offset + FRAME_HEADER_BYTES <= size) {
64
- if (view.getUint32(offset, true) !== FRAME_MAGIC)
65
- break;
66
- const length = view.getUint32(offset + 4, true);
67
- const checksum = view.getUint32(offset + 8, true);
120
+ readFully(handle, header, offset, "reading a WAL frame header for recovery");
121
+ if (headerView.getUint32(0, true) !== FRAME_MAGIC) {
122
+ throw new Error(`WAL frame marker mismatch at offset ${String(offset)}`);
123
+ }
124
+ const length = headerView.getUint32(4, true);
125
+ const checksum = headerView.getUint32(8, true);
126
+ if (length > MAX_WAL_FRAME_BYTES) {
127
+ throw new Error(`WAL frame at offset ${String(offset)} exceeds the ${String(MAX_WAL_FRAME_BYTES)} ` +
128
+ `byte limit: ${String(length)}`);
129
+ }
68
130
  const end = offset + FRAME_HEADER_BYTES + length;
69
- if (end > size)
70
- break;
71
- const payload = bytes.subarray(offset + FRAME_HEADER_BYTES, end);
72
- if (crc32(payload) !== checksum)
131
+ if (!Number.isSafeInteger(end) || end > size)
73
132
  break;
74
- payloads.push(decodeRecordJson(payload));
133
+ const payloadBytes = new Uint8Array(length);
134
+ readFully(handle, payloadBytes, offset + FRAME_HEADER_BYTES, "reading a WAL frame payload for recovery");
135
+ if (crc32(payloadBytes) !== checksum) {
136
+ throw new Error(`WAL frame checksum mismatch at offset ${String(offset)}`);
137
+ }
138
+ yield { payload: decodeRecordJson(payloadBytes), frameEnd: end };
75
139
  offset = end;
76
140
  }
77
- return { payloads, endOffset: offset };
78
141
  }
79
- //# sourceMappingURL=wal.js.map
@@ -9,7 +9,7 @@
9
9
  * can produce that object naturally — keys are ids and column names, values are JSON scalars —
10
10
  * and the reviver only converts an object whose sole key is `$n`.
11
11
  */
12
- export declare const LOG_FORMAT_VERSION = 2;
12
+ export declare const LOG_FORMAT_VERSION = 5;
13
13
  export declare function encodeRecordJson(value: unknown): Uint8Array;
14
14
  export declare function decodeRecordJson(bytes: Uint8Array): unknown;
15
15
  /** Immutable artifact payloads (full-text base chunks) inside extents. */
@@ -22,13 +22,12 @@ export interface PostingChunkEntry {
22
22
  tf: number[];
23
23
  }
24
24
  /**
25
- * Compact postings payload for OPFS. The old generic JSON encoding spelled every bigint row
26
- * locator as `{\"$n\":\"123\"}`; secondary indexes therefore took more space than the compressed
27
- * table they accelerated. Terms are UTF-8 and the already-sorted row locators are delta-varints.
25
+ * Compact postings payload for OPFS. Terms are UTF-8 and the already-sorted row locators are
26
+ * delta-varints, keeping secondary indexes smaller than the table they accelerate.
28
27
  * The envelope keeps the same version and checksum guarantees as every other immutable chunk.
29
28
  */
30
29
  export declare function encodePostingChunk(entries: readonly PostingChunkEntry[]): Uint8Array;
31
- /** `undefined` means this is an older generic chunk; callers then use the JSON decoder. */
30
+ /** `undefined` means the bytes are torn or are not a canonical postings envelope. */
32
31
  export declare function decodePostingChunk(bytes: Uint8Array): PostingChunkEntry[] | undefined;
33
32
  /**
34
33
  * The leader's checkpoint slots are written synchronously (no compression — that needs an
@@ -37,4 +36,3 @@ export declare function decodePostingChunk(bytes: Uint8Array): PostingChunkEntry
37
36
  */
38
37
  export declare function encodeSyncCheckpoint(state: unknown): Uint8Array;
39
38
  export declare function decodeSyncCheckpoint(bytes: Uint8Array): unknown;
40
- //# sourceMappingURL=wire.d.ts.map
@@ -1,4 +1,5 @@
1
1
  import { crc32 } from "../../block-format/index.js";
2
+ import { MAX_ROW_ID_EXCLUSIVE_END, StorageFormatVersionError } from "../types.js";
2
3
  /**
3
4
  * Payload encodings for a log-structured adapter's control data: checkpoint files, immutable
4
5
  * artifact chunks, and the JSON codec write-ahead-log frames carry. Every envelope is
@@ -10,9 +11,9 @@ import { crc32 } from "../../block-format/index.js";
10
11
  * can produce that object naturally — keys are ids and column names, values are JSON scalars —
11
12
  * and the reviver only converts an object whose sole key is `$n`.
12
13
  */
13
- export const LOG_FORMAT_VERSION = 2;
14
+ export const LOG_FORMAT_VERSION = 5;
14
15
  const textEncoder = new TextEncoder();
15
- const textDecoder = new TextDecoder();
16
+ const textDecoder = new TextDecoder("utf-8", { fatal: true });
16
17
  export function encodeRecordJson(value) {
17
18
  // The replacer costs ~3x on big payloads because it runs per node; most frames carry no
18
19
  // bigints at all, and stringify announces one by throwing.
@@ -21,8 +22,17 @@ export function encodeRecordJson(value) {
21
22
  json = JSON.stringify(value);
22
23
  }
23
24
  catch {
24
- json = JSON.stringify(value, (_key, entry) => typeof entry === "bigint" ? { $n: entry.toString() } : entry);
25
+ json = JSON.stringify(value, (_key, entry) => {
26
+ if (typeof entry !== "bigint")
27
+ return entry;
28
+ if (entry < 0n || entry > MAX_ROW_ID_EXCLUSIVE_END) {
29
+ throw new RangeError("Record bigint exceeds the unsigned 64-bit persisted range");
30
+ }
31
+ return { $n: entry.toString() };
32
+ });
25
33
  }
34
+ if (typeof json !== "string")
35
+ throw new TypeError("Record value is not JSON-serializable");
26
36
  return textEncoder.encode(json);
27
37
  }
28
38
  export function decodeRecordJson(bytes) {
@@ -33,7 +43,15 @@ export function decodeRecordJson(bytes) {
33
43
  !Array.isArray(entry) &&
34
44
  Object.keys(entry).length === 1 &&
35
45
  typeof entry.$n === "string") {
36
- return BigInt(entry.$n);
46
+ const decimal = entry.$n;
47
+ if (!/^(?:0|[1-9][0-9]{0,19})$/.test(decimal)) {
48
+ throw new TypeError("Record bigint is not a canonical bounded unsigned decimal");
49
+ }
50
+ const value = BigInt(decimal);
51
+ if (value > MAX_ROW_ID_EXCLUSIVE_END) {
52
+ throw new RangeError("Record bigint exceeds the unsigned 64-bit persisted range");
53
+ }
54
+ return value;
37
55
  }
38
56
  return entry;
39
57
  });
@@ -51,8 +69,8 @@ function encodeEnvelope(magic, payload) {
51
69
  }
52
70
  /**
53
71
  * `undefined` means the bytes are torn or foreign — treat as "not written". A recognized magic
54
- * with an unknown format version throws instead: that file was written by a newer build and
55
- * ignoring it would silently roll the database back.
72
+ * with an unknown format version throws instead: reading a different layout as the current one
73
+ * would silently corrupt or roll back the database.
56
74
  */
57
75
  function decodeEnvelope(magic, bytes) {
58
76
  if (bytes.byteLength < ENVELOPE_HEADER_BYTES)
@@ -64,11 +82,10 @@ function decodeEnvelope(magic, bytes) {
64
82
  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
65
83
  const version = view.getUint32(8, true);
66
84
  if (version !== LOG_FORMAT_VERSION) {
67
- throw new Error(`This database was written by a newer format (version ${String(version)}); ` +
68
- `this build reads version ${String(LOG_FORMAT_VERSION)}`);
85
+ throw new StorageFormatVersionError("opfs", `envelope/${magic}`, version, LOG_FORMAT_VERSION, version < LOG_FORMAT_VERSION ? "older" : "newer");
69
86
  }
70
87
  const payloadLength = view.getUint32(12, true);
71
- if (bytes.byteLength < ENVELOPE_HEADER_BYTES + payloadLength)
88
+ if (bytes.byteLength !== ENVELOPE_HEADER_BYTES + payloadLength)
72
89
  return undefined;
73
90
  const payload = bytes.subarray(ENVELOPE_HEADER_BYTES, ENVELOPE_HEADER_BYTES + payloadLength);
74
91
  if (crc32(payload) !== view.getUint32(16, true))
@@ -87,9 +104,8 @@ export function decodeChunk(bytes) {
87
104
  return payload === undefined ? undefined : decodeRecordJson(payload);
88
105
  }
89
106
  /**
90
- * Compact postings payload for OPFS. The old generic JSON encoding spelled every bigint row
91
- * locator as `{\"$n\":\"123\"}`; secondary indexes therefore took more space than the compressed
92
- * table they accelerated. Terms are UTF-8 and the already-sorted row locators are delta-varints.
107
+ * Compact postings payload for OPFS. Terms are UTF-8 and the already-sorted row locators are
108
+ * delta-varints, keeping secondary indexes smaller than the table they accelerate.
93
109
  * The envelope keeps the same version and checksum guarantees as every other immutable chunk.
94
110
  */
95
111
  export function encodePostingChunk(entries) {
@@ -100,46 +116,69 @@ export function encodePostingChunk(entries) {
100
116
  throw new TypeError("Posting row IDs and term frequencies must have the same length");
101
117
  }
102
118
  const term = textEncoder.encode(entry.term);
119
+ if (term.byteLength === 0)
120
+ throw new TypeError("Posting terms cannot be empty");
121
+ if (entry.rowIds.length === 0)
122
+ throw new TypeError("Postings cannot have no row IDs");
103
123
  writer.varuint(BigInt(term.byteLength));
104
124
  writer.bytes(term);
105
125
  writer.varuint(BigInt(entry.rowIds.length));
106
126
  let previous = 0n;
107
127
  for (const rowId of entry.rowIds) {
108
- if (rowId < previous)
109
- throw new TypeError("Posting row IDs must be sorted");
128
+ if (rowId <= previous || rowId > MAX_ROW_ID_EXCLUSIVE_END - 1n) {
129
+ throw new TypeError("Posting row IDs must be positive, uint64, and strictly sorted");
130
+ }
110
131
  writer.varuint(rowId - previous);
111
132
  previous = rowId;
112
133
  }
113
134
  for (const frequency of entry.tf) {
114
- if (!Number.isSafeInteger(frequency) || frequency < 0) {
115
- throw new TypeError("Posting term frequencies must be non-negative whole numbers");
135
+ if (!Number.isSafeInteger(frequency) || frequency <= 0) {
136
+ throw new TypeError("Posting term frequencies must be positive whole numbers");
116
137
  }
117
138
  writer.varuint(BigInt(frequency));
118
139
  }
119
140
  }
120
141
  return encodeEnvelope(POSTING_CHUNK_MAGIC, writer.finish());
121
142
  }
122
- /** `undefined` means this is an older generic chunk; callers then use the JSON decoder. */
143
+ /** `undefined` means the bytes are torn or are not a canonical postings envelope. */
123
144
  export function decodePostingChunk(bytes) {
124
145
  const payload = decodeEnvelope(POSTING_CHUNK_MAGIC, bytes);
125
146
  if (payload === undefined)
126
147
  return undefined;
127
148
  const reader = new BinaryReader(payload);
128
149
  const count = reader.safeInteger("posting count");
150
+ if (count > Math.floor(reader.remaining / 5)) {
151
+ throw new Error("Posting count exceeds the remaining payload");
152
+ }
129
153
  const entries = new Array(count);
130
154
  for (let entryIndex = 0; entryIndex < count; entryIndex += 1) {
131
155
  const termLength = reader.safeInteger("posting term length");
156
+ if (termLength < 1 || termLength > reader.remaining) {
157
+ throw new Error("Posting term length exceeds the remaining payload");
158
+ }
132
159
  const term = textDecoder.decode(reader.bytes(termLength));
133
160
  const rowCount = reader.safeInteger("posting row count");
161
+ if (rowCount < 1 || rowCount > Math.floor(reader.remaining / 2)) {
162
+ throw new Error("Posting row count exceeds the remaining payload");
163
+ }
134
164
  const rowIds = new Array(rowCount);
135
165
  let previous = 0n;
136
166
  for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
137
167
  previous += reader.varuint();
168
+ if (previous < 1n || previous > MAX_ROW_ID_EXCLUSIVE_END - 1n) {
169
+ throw new RangeError("Posting row ID exceeds the uint64 range");
170
+ }
171
+ if (rowIndex > 0 && previous <= (rowIds[rowIndex - 1] ?? 0n)) {
172
+ throw new TypeError("Posting row IDs are not strictly sorted");
173
+ }
138
174
  rowIds[rowIndex] = previous;
139
175
  }
140
176
  const tf = new Array(rowCount);
141
177
  for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
142
- tf[rowIndex] = reader.safeInteger("posting term frequency");
178
+ const frequency = reader.safeInteger("posting term frequency");
179
+ if (frequency <= 0)
180
+ throw new TypeError("Posting term frequency must be positive");
181
+ tf[rowIndex] = frequency;
143
182
  }
144
183
  entries[entryIndex] = { term, rowIds, tf };
145
184
  }
@@ -156,8 +195,9 @@ class BinaryWriter {
156
195
  this.#length += bytes.byteLength;
157
196
  }
158
197
  varuint(value) {
159
- if (value < 0n)
160
- throw new RangeError("A varuint cannot be negative");
198
+ if (value < 0n || value >= MAX_ROW_ID_EXCLUSIVE_END) {
199
+ throw new RangeError("A varuint must fit canonical uint64");
200
+ }
161
201
  do {
162
202
  this.#reserve(1);
163
203
  const byte = Number(value & 0x7fn);
@@ -170,12 +210,18 @@ class BinaryWriter {
170
210
  return this.#buffer.slice(0, this.#length);
171
211
  }
172
212
  #reserve(extra) {
213
+ if (!Number.isSafeInteger(extra) || extra < 0)
214
+ throw new RangeError("Invalid binary growth");
173
215
  const needed = this.#length + extra;
216
+ if (!Number.isSafeInteger(needed))
217
+ throw new RangeError("Binary payload is too large");
174
218
  if (needed <= this.#buffer.byteLength)
175
219
  return;
176
220
  let capacity = this.#buffer.byteLength;
177
- while (capacity < needed)
178
- capacity *= 2;
221
+ while (capacity < needed) {
222
+ const doubled = capacity * 2;
223
+ capacity = Number.isSafeInteger(doubled) ? doubled : needed;
224
+ }
179
225
  const grown = new Uint8Array(capacity);
180
226
  grown.set(this.#buffer);
181
227
  this.#buffer = grown;
@@ -190,8 +236,11 @@ class BinaryReader {
190
236
  get done() {
191
237
  return this.#offset === this.source.byteLength;
192
238
  }
239
+ get remaining() {
240
+ return this.source.byteLength - this.#offset;
241
+ }
193
242
  bytes(length) {
194
- if (this.#offset + length > this.source.byteLength)
243
+ if (!Number.isSafeInteger(length) || length < 0 || length > this.remaining)
195
244
  throw new Error("Posting chunk is truncated");
196
245
  const bytes = this.source.subarray(this.#offset, this.#offset + length);
197
246
  this.#offset += length;
@@ -205,9 +254,16 @@ class BinaryReader {
205
254
  if (byte === undefined)
206
255
  throw new Error("Posting chunk is truncated");
207
256
  this.#offset += 1;
257
+ if (byteIndex === 9 && (byte & 0xfe) !== 0) {
258
+ throw new Error("Posting chunk varuint exceeds uint64");
259
+ }
208
260
  value |= BigInt(byte & 0x7f) << shift;
209
- if ((byte & 0x80) === 0)
261
+ if ((byte & 0x80) === 0) {
262
+ if (byteIndex > 0 && (byte & 0x7f) === 0) {
263
+ throw new Error("Posting chunk varuint is overlong");
264
+ }
210
265
  return value;
266
+ }
211
267
  shift += 7n;
212
268
  }
213
269
  throw new Error("Posting chunk varuint is too wide");
@@ -231,4 +287,3 @@ export function decodeSyncCheckpoint(bytes) {
231
287
  const payload = decodeEnvelope(SYNC_CHECKPOINT_MAGIC, bytes);
232
288
  return payload === undefined ? undefined : decodeRecordJson(payload);
233
289
  }
234
- //# sourceMappingURL=wire.js.map