@minnowdb/core 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/README.md +23 -32
  2. package/dist/block-format/block.d.ts +32 -13
  3. package/dist/block-format/block.js +179 -55
  4. package/dist/block-format/checksum.d.ts +2 -1
  5. package/dist/block-format/checksum.js +5 -2
  6. package/dist/block-format/codecs.d.ts +7 -2
  7. package/dist/block-format/codecs.js +53 -12
  8. package/dist/block-format/column.d.ts +3 -2
  9. package/dist/block-format/column.js +96 -30
  10. package/dist/block-format/index.d.ts +2 -2
  11. package/dist/block-format/index.js +2 -2
  12. package/dist/block-format/physical.d.ts +8 -2
  13. package/dist/block-format/physical.js +14 -6
  14. package/dist/block-format/types.d.ts +7 -3
  15. package/dist/block-format/types.js +0 -1
  16. package/dist/block-format/unicode.d.ts +11 -0
  17. package/dist/block-format/unicode.js +46 -0
  18. package/dist/date-value.d.ts +15 -0
  19. package/dist/date-value.js +64 -0
  20. package/dist/engine/artifact-cache.d.ts +2 -1
  21. package/dist/engine/artifact-cache.js +5 -1
  22. package/dist/engine/batch.d.ts +11 -3
  23. package/dist/engine/batch.js +17 -7
  24. package/dist/engine/buffered-writer.d.ts +2 -1
  25. package/dist/engine/buffered-writer.js +34 -9
  26. package/dist/engine/cache-limits.d.ts +25 -0
  27. package/dist/engine/cache-limits.js +25 -0
  28. package/dist/engine/catalog.d.ts +24 -5
  29. package/dist/engine/catalog.js +34 -7
  30. package/dist/engine/client.d.ts +29 -7
  31. package/dist/engine/client.js +216 -40
  32. package/dist/engine/database.d.ts +224 -39
  33. package/dist/engine/database.js +7475 -1539
  34. package/dist/engine/defaults.d.ts +4 -8
  35. package/dist/engine/defaults.js +46 -22
  36. package/dist/engine/errors.d.ts +19 -1
  37. package/dist/engine/errors.js +32 -2
  38. package/dist/engine/fts.d.ts +0 -6
  39. package/dist/engine/fts.js +41 -14
  40. package/dist/engine/group-index.d.ts +0 -1
  41. package/dist/engine/group-index.js +6 -11
  42. package/dist/engine/index.d.ts +16 -10
  43. package/dist/engine/index.js +12 -9
  44. package/dist/engine/join-index.d.ts +0 -1
  45. package/dist/engine/join-index.js +2 -2
  46. package/dist/engine/keyed-live.d.ts +57 -0
  47. package/dist/engine/keyed-live.js +226 -0
  48. package/dist/engine/live-api.d.ts +4 -0
  49. package/dist/engine/live-api.js +4 -0
  50. package/dist/engine/live.d.ts +43 -26
  51. package/dist/engine/live.js +504 -126
  52. package/dist/engine/memory.d.ts +2 -1
  53. package/dist/engine/memory.js +3 -2
  54. package/dist/engine/optimizer.d.ts +0 -1
  55. package/dist/engine/optimizer.js +262 -27
  56. package/dist/engine/query-api.d.ts +3 -0
  57. package/dist/engine/query-api.js +3 -0
  58. package/dist/engine/query-cache.d.ts +1 -2
  59. package/dist/engine/query-cache.js +7 -5
  60. package/dist/engine/query.d.ts +134 -29
  61. package/dist/engine/query.js +1668 -372
  62. package/dist/engine/result-wire.d.ts +0 -1
  63. package/dist/engine/result-wire.js +2 -2
  64. package/dist/engine/schema-wire.d.ts +6 -3
  65. package/dist/engine/schema-wire.js +40 -34
  66. package/dist/engine/schema.d.ts +156 -80
  67. package/dist/engine/schema.js +570 -108
  68. package/dist/engine/sort-keys.d.ts +4 -4
  69. package/dist/engine/sort-keys.js +44 -29
  70. package/dist/engine/sql-domains.d.ts +31 -0
  71. package/dist/engine/sql-domains.js +585 -0
  72. package/dist/engine/sql-driver.d.ts +19 -0
  73. package/dist/engine/sql-driver.js +1 -0
  74. package/dist/engine/sql-json.d.ts +7 -9
  75. package/dist/engine/sql-json.js +75 -15
  76. package/dist/engine/sql-semantics.d.ts +8 -3
  77. package/dist/engine/sql-semantics.js +458 -30
  78. package/dist/engine/typed-live.d.ts +67 -0
  79. package/dist/engine/typed-live.js +349 -0
  80. package/dist/engine/vector.d.ts +12 -2
  81. package/dist/engine/vector.js +635 -119
  82. package/dist/engine/worker-host.d.ts +9 -2
  83. package/dist/engine/worker-host.js +620 -152
  84. package/dist/engine/worker.d.ts +0 -1
  85. package/dist/engine/worker.js +3 -3
  86. package/dist/engine/write-block-planner.d.ts +18 -0
  87. package/dist/engine/write-block-planner.js +112 -0
  88. package/dist/index.d.ts +0 -1
  89. package/dist/index.js +0 -1
  90. package/dist/plan/index.d.ts +3 -4
  91. package/dist/plan/index.js +3 -4
  92. package/dist/storage/index.d.ts +1 -1
  93. package/dist/storage/index.js +1 -1
  94. package/dist/storage/indexeddb.d.ts +99 -51
  95. package/dist/storage/indexeddb.js +13448 -2660
  96. package/dist/storage/memory.d.ts +64 -57
  97. package/dist/storage/memory.js +1042 -128
  98. package/dist/storage/opfs/files.d.ts +18 -7
  99. package/dist/storage/opfs/files.js +115 -28
  100. package/dist/storage/opfs/index.d.ts +1 -1
  101. package/dist/storage/opfs/index.js +1 -1
  102. package/dist/storage/opfs/leader.d.ts +409 -75
  103. package/dist/storage/opfs/leader.js +4621 -650
  104. package/dist/storage/opfs/rpc.d.ts +15 -3
  105. package/dist/storage/opfs/rpc.js +224 -2
  106. package/dist/storage/opfs/snapshot-ledger.d.ts +40 -0
  107. package/dist/storage/opfs/snapshot-ledger.js +281 -0
  108. package/dist/storage/opfs/store.d.ts +33 -99
  109. package/dist/storage/opfs/store.js +453 -364
  110. package/dist/storage/persistence.d.ts +37 -0
  111. package/dist/storage/persistence.js +78 -0
  112. package/dist/storage/snapshot-stream.d.ts +24 -0
  113. package/dist/storage/snapshot-stream.js +897 -0
  114. package/dist/storage/snapshot.d.ts +9 -85
  115. package/dist/storage/snapshot.js +33 -265
  116. package/dist/storage/toolkit/extents.d.ts +41 -7
  117. package/dist/storage/toolkit/extents.js +402 -39
  118. package/dist/storage/toolkit/index.d.ts +4 -5
  119. package/dist/storage/toolkit/index.js +28 -4
  120. package/dist/storage/toolkit/record-core.d.ts +182 -55
  121. package/dist/storage/toolkit/record-core.js +6158 -1331
  122. package/dist/storage/toolkit/sync-file.d.ts +10 -1
  123. package/dist/storage/toolkit/sync-file.js +50 -2
  124. package/dist/storage/toolkit/wal.d.ts +23 -5
  125. package/dist/storage/toolkit/wal.js +89 -27
  126. package/dist/storage/toolkit/wire.d.ts +15 -2
  127. package/dist/storage/toolkit/wire.js +199 -10
  128. package/dist/storage/types.d.ts +1439 -204
  129. package/dist/storage/types.js +1745 -141
  130. package/dist/testing/block-store-conformance.d.ts +0 -1
  131. package/dist/testing/block-store-conformance.js +1009 -119
  132. package/dist/testing/index.d.ts +46 -26
  133. package/dist/testing/index.js +112 -57
  134. package/dist/testing/opfs-shim.d.ts +13 -3
  135. package/dist/testing/opfs-shim.js +40 -6
  136. package/dist/testing/simulator.d.ts +97 -0
  137. package/dist/testing/simulator.js +591 -0
  138. package/dist/testing/sqllogictest.d.ts +89 -0
  139. package/dist/testing/sqllogictest.js +434 -0
  140. package/dist/transactions/index.d.ts +92 -16
  141. package/dist/transactions/index.js +1074 -236
  142. package/dist/worker-protocol/index.d.ts +3 -2
  143. package/dist/worker-protocol/index.js +2 -5
  144. package/package.json +53 -3
  145. package/postgres-feature-profile.json +223 -0
  146. package/sql-feature-matrix.json +172 -252
  147. package/dist/block-format/block.d.ts.map +0 -1
  148. package/dist/block-format/block.js.map +0 -1
  149. package/dist/block-format/checksum.d.ts.map +0 -1
  150. package/dist/block-format/checksum.js.map +0 -1
  151. package/dist/block-format/codecs.d.ts.map +0 -1
  152. package/dist/block-format/codecs.js.map +0 -1
  153. package/dist/block-format/column.d.ts.map +0 -1
  154. package/dist/block-format/column.js.map +0 -1
  155. package/dist/block-format/index.d.ts.map +0 -1
  156. package/dist/block-format/index.js.map +0 -1
  157. package/dist/block-format/physical.d.ts.map +0 -1
  158. package/dist/block-format/physical.js.map +0 -1
  159. package/dist/block-format/types.d.ts.map +0 -1
  160. package/dist/block-format/types.js.map +0 -1
  161. package/dist/engine/artifact-cache.d.ts.map +0 -1
  162. package/dist/engine/artifact-cache.js.map +0 -1
  163. package/dist/engine/batch.d.ts.map +0 -1
  164. package/dist/engine/batch.js.map +0 -1
  165. package/dist/engine/buffered-writer.d.ts.map +0 -1
  166. package/dist/engine/buffered-writer.js.map +0 -1
  167. package/dist/engine/catalog.d.ts.map +0 -1
  168. package/dist/engine/catalog.js.map +0 -1
  169. package/dist/engine/client.d.ts.map +0 -1
  170. package/dist/engine/client.js.map +0 -1
  171. package/dist/engine/coordinator.d.ts +0 -17
  172. package/dist/engine/coordinator.d.ts.map +0 -1
  173. package/dist/engine/coordinator.js +0 -60
  174. package/dist/engine/coordinator.js.map +0 -1
  175. package/dist/engine/database.d.ts.map +0 -1
  176. package/dist/engine/database.js.map +0 -1
  177. package/dist/engine/defaults.d.ts.map +0 -1
  178. package/dist/engine/defaults.js.map +0 -1
  179. package/dist/engine/errors.d.ts.map +0 -1
  180. package/dist/engine/errors.js.map +0 -1
  181. package/dist/engine/fts.d.ts.map +0 -1
  182. package/dist/engine/fts.js.map +0 -1
  183. package/dist/engine/group-index.d.ts.map +0 -1
  184. package/dist/engine/group-index.js.map +0 -1
  185. package/dist/engine/index.d.ts.map +0 -1
  186. package/dist/engine/index.js.map +0 -1
  187. package/dist/engine/join-index.d.ts.map +0 -1
  188. package/dist/engine/join-index.js.map +0 -1
  189. package/dist/engine/live.d.ts.map +0 -1
  190. package/dist/engine/live.js.map +0 -1
  191. package/dist/engine/memory.d.ts.map +0 -1
  192. package/dist/engine/memory.js.map +0 -1
  193. package/dist/engine/optimizer.d.ts.map +0 -1
  194. package/dist/engine/optimizer.js.map +0 -1
  195. package/dist/engine/query-cache.d.ts.map +0 -1
  196. package/dist/engine/query-cache.js.map +0 -1
  197. package/dist/engine/query.d.ts.map +0 -1
  198. package/dist/engine/query.js.map +0 -1
  199. package/dist/engine/result-wire.d.ts.map +0 -1
  200. package/dist/engine/result-wire.js.map +0 -1
  201. package/dist/engine/schema-wire.d.ts.map +0 -1
  202. package/dist/engine/schema-wire.js.map +0 -1
  203. package/dist/engine/schema.d.ts.map +0 -1
  204. package/dist/engine/schema.js.map +0 -1
  205. package/dist/engine/sort-keys.d.ts.map +0 -1
  206. package/dist/engine/sort-keys.js.map +0 -1
  207. package/dist/engine/sql-json.d.ts.map +0 -1
  208. package/dist/engine/sql-json.js.map +0 -1
  209. package/dist/engine/sql-semantics.d.ts.map +0 -1
  210. package/dist/engine/sql-semantics.js.map +0 -1
  211. package/dist/engine/vector.d.ts.map +0 -1
  212. package/dist/engine/vector.js.map +0 -1
  213. package/dist/engine/worker-host.d.ts.map +0 -1
  214. package/dist/engine/worker-host.js.map +0 -1
  215. package/dist/engine/worker.d.ts.map +0 -1
  216. package/dist/engine/worker.js.map +0 -1
  217. package/dist/index.d.ts.map +0 -1
  218. package/dist/index.js.map +0 -1
  219. package/dist/plan/index.d.ts.map +0 -1
  220. package/dist/plan/index.js.map +0 -1
  221. package/dist/storage/fixture-shape.d.ts +0 -42
  222. package/dist/storage/fixture-shape.d.ts.map +0 -1
  223. package/dist/storage/fixture-shape.js +0 -146
  224. package/dist/storage/fixture-shape.js.map +0 -1
  225. package/dist/storage/index.d.ts.map +0 -1
  226. package/dist/storage/index.js.map +0 -1
  227. package/dist/storage/indexeddb.d.ts.map +0 -1
  228. package/dist/storage/indexeddb.js.map +0 -1
  229. package/dist/storage/memory.d.ts.map +0 -1
  230. package/dist/storage/memory.js.map +0 -1
  231. package/dist/storage/opfs/files.d.ts.map +0 -1
  232. package/dist/storage/opfs/files.js.map +0 -1
  233. package/dist/storage/opfs/index.d.ts.map +0 -1
  234. package/dist/storage/opfs/index.js.map +0 -1
  235. package/dist/storage/opfs/leader.d.ts.map +0 -1
  236. package/dist/storage/opfs/leader.js.map +0 -1
  237. package/dist/storage/opfs/rpc.d.ts.map +0 -1
  238. package/dist/storage/opfs/rpc.js.map +0 -1
  239. package/dist/storage/opfs/store.d.ts.map +0 -1
  240. package/dist/storage/opfs/store.js.map +0 -1
  241. package/dist/storage/snapshot.d.ts.map +0 -1
  242. package/dist/storage/snapshot.js.map +0 -1
  243. package/dist/storage/toolkit/extents.d.ts.map +0 -1
  244. package/dist/storage/toolkit/extents.js.map +0 -1
  245. package/dist/storage/toolkit/index.d.ts.map +0 -1
  246. package/dist/storage/toolkit/index.js.map +0 -1
  247. package/dist/storage/toolkit/record-core.d.ts.map +0 -1
  248. package/dist/storage/toolkit/record-core.js.map +0 -1
  249. package/dist/storage/toolkit/sync-file.d.ts.map +0 -1
  250. package/dist/storage/toolkit/sync-file.js.map +0 -1
  251. package/dist/storage/toolkit/wal.d.ts.map +0 -1
  252. package/dist/storage/toolkit/wal.js.map +0 -1
  253. package/dist/storage/toolkit/wire.d.ts.map +0 -1
  254. package/dist/storage/toolkit/wire.js.map +0 -1
  255. package/dist/storage/types.d.ts.map +0 -1
  256. package/dist/storage/types.js.map +0 -1
  257. package/dist/testing/block-store-conformance.d.ts.map +0 -1
  258. package/dist/testing/block-store-conformance.js.map +0 -1
  259. package/dist/testing/index.d.ts.map +0 -1
  260. package/dist/testing/index.js.map +0 -1
  261. package/dist/testing/opfs-shim.d.ts.map +0 -1
  262. package/dist/testing/opfs-shim.js.map +0 -1
  263. package/dist/testing/seeds.d.ts +0 -11
  264. package/dist/testing/seeds.d.ts.map +0 -1
  265. package/dist/testing/seeds.js +0 -50
  266. package/dist/testing/seeds.js.map +0 -1
  267. package/dist/transactions/index.d.ts.map +0 -1
  268. package/dist/transactions/index.js.map +0 -1
  269. package/dist/worker-protocol/index.d.ts.map +0 -1
  270. package/dist/worker-protocol/index.js.map +0 -1
@@ -26,4 +26,13 @@ export interface SyncFileHandle {
26
26
  /** Releases the handle (and any exclusive lock the platform ties to it). */
27
27
  close(): void;
28
28
  }
29
- //# sourceMappingURL=sync-file.d.ts.map
29
+ /**
30
+ * Writes the complete buffer, retrying the short transfers permitted by sync-access handles.
31
+ * The common path is one `write` call; retries pass subarray views and never copy payload bytes.
32
+ */
33
+ export declare function writeFully(handle: SyncFileHandle, buffer: Uint8Array, at: number, operation?: string): void;
34
+ /**
35
+ * Reads exactly `buffer.byteLength` bytes, retrying short transfers without copying data. A zero
36
+ * transfer before the requested range is complete is an unexpected EOF, not a successful read.
37
+ */
38
+ export declare function readFully(handle: SyncFileHandle, buffer: Uint8Array, at: number, operation?: string): void;
@@ -1,2 +1,50 @@
1
- export {};
2
- //# sourceMappingURL=sync-file.js.map
1
+ /**
2
+ * Writes the complete buffer, retrying the short transfers permitted by sync-access handles.
3
+ * The common path is one `write` call; retries pass subarray views and never copy payload bytes.
4
+ */
5
+ export function writeFully(handle, buffer, at, operation = "writing file bytes") {
6
+ assertTransferRange(at, buffer.byteLength, "write", operation);
7
+ let transferred = 0;
8
+ while (transferred < buffer.byteLength) {
9
+ const remaining = buffer.byteLength - transferred;
10
+ const chunk = transferred === 0 ? buffer : buffer.subarray(transferred);
11
+ const count = handle.write(chunk, { at: at + transferred });
12
+ assertTransferCount(count, remaining, "write", operation, at + transferred);
13
+ if (count === 0) {
14
+ throw new Error(`OPFS write made no progress while ${operation}: wrote ${String(transferred)} of ` +
15
+ `${String(buffer.byteLength)} bytes (stalled at file offset ${String(at + transferred)})`);
16
+ }
17
+ transferred += count;
18
+ }
19
+ }
20
+ /**
21
+ * Reads exactly `buffer.byteLength` bytes, retrying short transfers without copying data. A zero
22
+ * transfer before the requested range is complete is an unexpected EOF, not a successful read.
23
+ */
24
+ export function readFully(handle, buffer, at, operation = "reading file bytes") {
25
+ assertTransferRange(at, buffer.byteLength, "read", operation);
26
+ let transferred = 0;
27
+ while (transferred < buffer.byteLength) {
28
+ const remaining = buffer.byteLength - transferred;
29
+ const chunk = transferred === 0 ? buffer : buffer.subarray(transferred);
30
+ const count = handle.read(chunk, { at: at + transferred });
31
+ assertTransferCount(count, remaining, "read", operation, at + transferred);
32
+ if (count === 0) {
33
+ throw new Error(`Unexpected EOF while ${operation}: read ${String(transferred)} of ` +
34
+ `${String(buffer.byteLength)} bytes (EOF at file offset ${String(at + transferred)})`);
35
+ }
36
+ transferred += count;
37
+ }
38
+ }
39
+ function assertTransferRange(at, length, kind, operation) {
40
+ if (!Number.isSafeInteger(at) || at < 0 || length > Number.MAX_SAFE_INTEGER - at) {
41
+ throw new RangeError(`Invalid OPFS ${kind} range while ${operation}: ${String(at)}+${String(length)} ` +
42
+ `must be a non-negative safe-integer byte range`);
43
+ }
44
+ }
45
+ function assertTransferCount(count, remaining, kind, operation, at) {
46
+ if (!Number.isSafeInteger(count) || count < 0 || count > remaining) {
47
+ throw new Error(`Invalid OPFS ${kind} result while ${operation}: handle reported ${String(count)} bytes ` +
48
+ `for a ${String(remaining)}-byte request at file offset ${String(at)}`);
49
+ }
50
+ }
@@ -1,4 +1,15 @@
1
- import type { SyncFileHandle } from "./sync-file.js";
1
+ import { type SyncFileHandle } from "./sync-file.js";
2
+ /**
3
+ * A WAL frame is control data, never bulk block data. Bounding it prevents a corrupt length
4
+ * field (or an accidentally enormous catalog mutation) from turning recovery into an
5
+ * unbounded allocation. Snapshot/block payloads live in extents, so 64 MiB leaves generous
6
+ * headroom for legitimate schema and transaction records.
7
+ */
8
+ export declare const MAX_WAL_FRAME_BYTES: number;
9
+ export interface ReplayedWalFrame {
10
+ payload: unknown;
11
+ frameEnd: number;
12
+ }
2
13
  export declare class WalWriter {
3
14
  #private;
4
15
  constructor(handle: SyncFileHandle, offset: number);
@@ -11,12 +22,19 @@ export declare class WalWriter {
11
22
  close(): void;
12
23
  }
13
24
  /**
14
- * Reads every whole, checksum-valid frame from the handle's current content, in order,
15
- * stopping at the first torn or foreign bytes. Returns the payloads and the byte offset where
16
- * appending should resume (the end of the last valid frame — a torn tail is overwritten).
25
+ * Reads every whole, checksum-valid frame from the handle's current content, in order. A
26
+ * truncated final frame is ignored; complete foreign or corrupt bytes are rejected. Returns the
27
+ * payloads and the byte offset where appending should resume (the end of the last valid frame —
28
+ * a truncated tail is overwritten).
17
29
  */
18
30
  export declare function replayWalFrames(handle: SyncFileHandle): {
19
31
  payloads: unknown[];
32
+ /** End offset of each corresponding payload frame. */
33
+ frameEnds: number[];
20
34
  endOffset: number;
21
35
  };
22
- //# sourceMappingURL=wal.d.ts.map
36
+ /**
37
+ * Streams checksum-valid frames with O(max-frame) memory. A truncated final frame is ignored;
38
+ * complete foreign or checksum-invalid bytes are corruption and fail closed.
39
+ */
40
+ export declare function iterateWalFrames(handle: SyncFileHandle): Generator<ReplayedWalFrame>;
@@ -1,21 +1,34 @@
1
1
  import { crc32 } from "../../block-format/index.js";
2
2
  import { decodeRecordJson, encodeRecordJson } from "./wire.js";
3
+ import { readFully, writeFully } from "./sync-file.js";
3
4
  /**
4
5
  * The write-ahead log, framed over one synchronous file handle its single writer holds open.
5
6
  *
6
7
  * Frame layout, little-endian: `u32 magic | u32 payloadLength | u32 crc32(payload) | payload`.
7
- * An append is a single synchronous write at the tail — measured at microseconds on a held
8
- * handle and the only artifact a crash can leave is a torn final frame, which fails its
9
- * checksum and reads as "not written". Frames carry a global sequence number inside the
10
- * payload; the file is truncated to zero only after a checkpoint covering every frame has
11
- * been flushed, so replay is always newest-checkpoint-plus-tail.
8
+ * An append is one complete synchronous transfer at the tail — normally one microsecond-scale
9
+ * write on a held handle, with retries if the platform reports a short transfer. The only
10
+ * artifact a crash can leave is a truncated final frame, which reads as "not written". A
11
+ * complete frame with foreign marker or checksum-invalid bytes is corruption and fails closed.
12
+ * Frames carry a global sequence number inside the payload; the file is truncated to zero only
13
+ * after a checkpoint covering every frame has been flushed, so replay is always
14
+ * newest-checkpoint-plus-tail.
12
15
  */
13
16
  const FRAME_MAGIC = 0x4c574e4d; // "MNWL"
14
17
  const FRAME_HEADER_BYTES = 12;
18
+ /**
19
+ * A WAL frame is control data, never bulk block data. Bounding it prevents a corrupt length
20
+ * field (or an accidentally enormous catalog mutation) from turning recovery into an
21
+ * unbounded allocation. Snapshot/block payloads live in extents, so 64 MiB leaves generous
22
+ * headroom for legitimate schema and transaction records.
23
+ */
24
+ export const MAX_WAL_FRAME_BYTES = 64 * 1024 * 1024;
15
25
  export class WalWriter {
16
26
  #handle;
17
27
  #offset;
18
28
  constructor(handle, offset) {
29
+ if (!Number.isSafeInteger(offset) || offset < 0) {
30
+ throw new RangeError("WAL offset is outside the safe integer range");
31
+ }
19
32
  this.#handle = handle;
20
33
  this.#offset = offset;
21
34
  }
@@ -25,21 +38,47 @@ export class WalWriter {
25
38
  /** Appends one frame; with `flush`, durable before return. Synchronous — never yields. */
26
39
  append(payload, flush) {
27
40
  const bytes = encodeRecordJson(payload);
41
+ if (bytes.byteLength > MAX_WAL_FRAME_BYTES) {
42
+ throw new RangeError(`WAL frame payload exceeds the ${String(MAX_WAL_FRAME_BYTES)} byte limit: ` +
43
+ String(bytes.byteLength));
44
+ }
28
45
  const frame = new Uint8Array(FRAME_HEADER_BYTES + bytes.byteLength);
46
+ if (this.#offset > Number.MAX_SAFE_INTEGER - frame.byteLength) {
47
+ throw new RangeError("WAL offset exceeds the safe integer range");
48
+ }
29
49
  const view = new DataView(frame.buffer);
30
50
  view.setUint32(0, FRAME_MAGIC, true);
31
51
  view.setUint32(4, bytes.byteLength, true);
32
52
  view.setUint32(8, crc32(bytes), true);
33
53
  frame.set(bytes, FRAME_HEADER_BYTES);
34
- this.#handle.write(frame, { at: this.#offset });
35
- if (flush)
36
- this.#handle.flush();
37
- this.#offset += frame.byteLength;
54
+ const start = this.#offset;
55
+ try {
56
+ writeFully(this.#handle, frame, start, "appending a WAL frame");
57
+ if (flush)
58
+ this.#handle.flush();
59
+ this.#offset = start + frame.byteLength;
60
+ }
61
+ catch (error) {
62
+ // A complete write followed by a failed flush is not an acknowledged frame either.
63
+ // Remove it before unpublished extent bytes are rolled back. If truncation itself
64
+ // fails, recovery will fail closed when it verifies the frame's referenced payloads.
65
+ try {
66
+ this.#handle.truncate(start);
67
+ }
68
+ catch {
69
+ // Preserve the original quota/I/O error.
70
+ }
71
+ throw error;
72
+ }
38
73
  }
39
74
  /** Empties the log after a flushed checkpoint has covered every frame in it. */
40
75
  reset() {
41
76
  this.#handle.truncate(0);
77
+ // truncate() already changed the handle's logical file position. Keep the writer aligned
78
+ // with it even if the durability flush is refused; a later append must start at zero rather
79
+ // than leave a sparse, unreplayable gap after the checkpoint.
42
80
  this.#offset = 0;
81
+ this.#handle.flush();
43
82
  }
44
83
  flush() {
45
84
  this.#handle.flush();
@@ -49,31 +88,54 @@ export class WalWriter {
49
88
  }
50
89
  }
51
90
  /**
52
- * Reads every whole, checksum-valid frame from the handle's current content, in order,
53
- * stopping at the first torn or foreign bytes. Returns the payloads and the byte offset where
54
- * appending should resume (the end of the last valid frame — a torn tail is overwritten).
91
+ * Reads every whole, checksum-valid frame from the handle's current content, in order. A
92
+ * truncated final frame is ignored; complete foreign or corrupt bytes are rejected. Returns the
93
+ * payloads and the byte offset where appending should resume (the end of the last valid frame —
94
+ * a truncated tail is overwritten).
55
95
  */
56
96
  export function replayWalFrames(handle) {
57
- const size = handle.getSize();
58
- const bytes = new Uint8Array(size);
59
- handle.read(bytes, { at: 0 });
60
- const view = new DataView(bytes.buffer);
61
97
  const payloads = [];
98
+ const frameEnds = [];
99
+ let endOffset = 0;
100
+ for (const frame of iterateWalFrames(handle)) {
101
+ payloads.push(frame.payload);
102
+ frameEnds.push(frame.frameEnd);
103
+ endOffset = frame.frameEnd;
104
+ }
105
+ return { payloads, frameEnds, endOffset };
106
+ }
107
+ /**
108
+ * Streams checksum-valid frames with O(max-frame) memory. A truncated final frame is ignored;
109
+ * complete foreign or checksum-invalid bytes are corruption and fail closed.
110
+ */
111
+ export function* iterateWalFrames(handle) {
112
+ const size = handle.getSize();
113
+ if (!Number.isSafeInteger(size) || size < 0) {
114
+ throw new Error(`Invalid WAL byte length: ${String(size)}`);
115
+ }
116
+ const header = new Uint8Array(FRAME_HEADER_BYTES);
117
+ const headerView = new DataView(header.buffer);
62
118
  let offset = 0;
63
119
  while (offset + FRAME_HEADER_BYTES <= size) {
64
- if (view.getUint32(offset, true) !== FRAME_MAGIC)
65
- break;
66
- const length = view.getUint32(offset + 4, true);
67
- const checksum = view.getUint32(offset + 8, true);
120
+ readFully(handle, header, offset, "reading a WAL frame header for recovery");
121
+ if (headerView.getUint32(0, true) !== FRAME_MAGIC) {
122
+ throw new Error(`WAL frame marker mismatch at offset ${String(offset)}`);
123
+ }
124
+ const length = headerView.getUint32(4, true);
125
+ const checksum = headerView.getUint32(8, true);
126
+ if (length > MAX_WAL_FRAME_BYTES) {
127
+ throw new Error(`WAL frame at offset ${String(offset)} exceeds the ${String(MAX_WAL_FRAME_BYTES)} ` +
128
+ `byte limit: ${String(length)}`);
129
+ }
68
130
  const end = offset + FRAME_HEADER_BYTES + length;
69
- if (end > size)
70
- break;
71
- const payload = bytes.subarray(offset + FRAME_HEADER_BYTES, end);
72
- if (crc32(payload) !== checksum)
131
+ if (!Number.isSafeInteger(end) || end > size)
73
132
  break;
74
- payloads.push(decodeRecordJson(payload));
133
+ const payloadBytes = new Uint8Array(length);
134
+ readFully(handle, payloadBytes, offset + FRAME_HEADER_BYTES, "reading a WAL frame payload for recovery");
135
+ if (crc32(payloadBytes) !== checksum) {
136
+ throw new Error(`WAL frame checksum mismatch at offset ${String(offset)}`);
137
+ }
138
+ yield { payload: decodeRecordJson(payloadBytes), frameEnd: end };
75
139
  offset = end;
76
140
  }
77
- return { payloads, endOffset: offset };
78
141
  }
79
- //# sourceMappingURL=wal.js.map
@@ -9,12 +9,26 @@
9
9
  * can produce that object naturally — keys are ids and column names, values are JSON scalars —
10
10
  * and the reviver only converts an object whose sole key is `$n`.
11
11
  */
12
- export declare const LOG_FORMAT_VERSION = 2;
12
+ export declare const LOG_FORMAT_VERSION = 5;
13
13
  export declare function encodeRecordJson(value: unknown): Uint8Array;
14
14
  export declare function decodeRecordJson(bytes: Uint8Array): unknown;
15
15
  /** Immutable artifact payloads (full-text base chunks) inside extents. */
16
16
  export declare function encodeChunk(value: unknown): Uint8Array;
17
17
  export declare function decodeChunk(bytes: Uint8Array): unknown;
18
+ /** The structural part of FtsPosting kept here to avoid coupling the low-level codec to storage. */
19
+ export interface PostingChunkEntry {
20
+ term: string;
21
+ rowIds: bigint[];
22
+ tf: number[];
23
+ }
24
+ /**
25
+ * Compact postings payload for OPFS. Terms are UTF-8 and the already-sorted row locators are
26
+ * delta-varints, keeping secondary indexes smaller than the table they accelerate.
27
+ * The envelope keeps the same version and checksum guarantees as every other immutable chunk.
28
+ */
29
+ export declare function encodePostingChunk(entries: readonly PostingChunkEntry[]): Uint8Array;
30
+ /** `undefined` means the bytes are torn or are not a canonical postings envelope. */
31
+ export declare function decodePostingChunk(bytes: Uint8Array): PostingChunkEntry[] | undefined;
18
32
  /**
19
33
  * The leader's checkpoint slots are written synchronously (no compression — that needs an
20
34
  * await, and the checkpoint must land in the same synchronous run that resets the WAL, or
@@ -22,4 +36,3 @@ export declare function decodeChunk(bytes: Uint8Array): unknown;
22
36
  */
23
37
  export declare function encodeSyncCheckpoint(state: unknown): Uint8Array;
24
38
  export declare function decodeSyncCheckpoint(bytes: Uint8Array): unknown;
25
- //# sourceMappingURL=wire.d.ts.map
@@ -1,4 +1,5 @@
1
1
  import { crc32 } from "../../block-format/index.js";
2
+ import { MAX_ROW_ID_EXCLUSIVE_END, StorageFormatVersionError } from "../types.js";
2
3
  /**
3
4
  * Payload encodings for a log-structured adapter's control data: checkpoint files, immutable
4
5
  * artifact chunks, and the JSON codec write-ahead-log frames carry. Every envelope is
@@ -10,9 +11,9 @@ import { crc32 } from "../../block-format/index.js";
10
11
  * can produce that object naturally — keys are ids and column names, values are JSON scalars —
11
12
  * and the reviver only converts an object whose sole key is `$n`.
12
13
  */
13
- export const LOG_FORMAT_VERSION = 2;
14
+ export const LOG_FORMAT_VERSION = 5;
14
15
  const textEncoder = new TextEncoder();
15
- const textDecoder = new TextDecoder();
16
+ const textDecoder = new TextDecoder("utf-8", { fatal: true });
16
17
  export function encodeRecordJson(value) {
17
18
  // The replacer costs ~3x on big payloads because it runs per node; most frames carry no
18
19
  // bigints at all, and stringify announces one by throwing.
@@ -21,8 +22,17 @@ export function encodeRecordJson(value) {
21
22
  json = JSON.stringify(value);
22
23
  }
23
24
  catch {
24
- json = JSON.stringify(value, (_key, entry) => typeof entry === "bigint" ? { $n: entry.toString() } : entry);
25
+ json = JSON.stringify(value, (_key, entry) => {
26
+ if (typeof entry !== "bigint")
27
+ return entry;
28
+ if (entry < 0n || entry > MAX_ROW_ID_EXCLUSIVE_END) {
29
+ throw new RangeError("Record bigint exceeds the unsigned 64-bit persisted range");
30
+ }
31
+ return { $n: entry.toString() };
32
+ });
25
33
  }
34
+ if (typeof json !== "string")
35
+ throw new TypeError("Record value is not JSON-serializable");
26
36
  return textEncoder.encode(json);
27
37
  }
28
38
  export function decodeRecordJson(bytes) {
@@ -33,7 +43,15 @@ export function decodeRecordJson(bytes) {
33
43
  !Array.isArray(entry) &&
34
44
  Object.keys(entry).length === 1 &&
35
45
  typeof entry.$n === "string") {
36
- return BigInt(entry.$n);
46
+ const decimal = entry.$n;
47
+ if (!/^(?:0|[1-9][0-9]{0,19})$/.test(decimal)) {
48
+ throw new TypeError("Record bigint is not a canonical bounded unsigned decimal");
49
+ }
50
+ const value = BigInt(decimal);
51
+ if (value > MAX_ROW_ID_EXCLUSIVE_END) {
52
+ throw new RangeError("Record bigint exceeds the unsigned 64-bit persisted range");
53
+ }
54
+ return value;
37
55
  }
38
56
  return entry;
39
57
  });
@@ -51,8 +69,8 @@ function encodeEnvelope(magic, payload) {
51
69
  }
52
70
  /**
53
71
  * `undefined` means the bytes are torn or foreign — treat as "not written". A recognized magic
54
- * with an unknown format version throws instead: that file was written by a newer build and
55
- * ignoring it would silently roll the database back.
72
+ * with an unknown format version throws instead: reading a different layout as the current one
73
+ * would silently corrupt or roll back the database.
56
74
  */
57
75
  function decodeEnvelope(magic, bytes) {
58
76
  if (bytes.byteLength < ENVELOPE_HEADER_BYTES)
@@ -64,11 +82,10 @@ function decodeEnvelope(magic, bytes) {
64
82
  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
65
83
  const version = view.getUint32(8, true);
66
84
  if (version !== LOG_FORMAT_VERSION) {
67
- throw new Error(`This database was written by a newer format (version ${String(version)}); ` +
68
- `this build reads version ${String(LOG_FORMAT_VERSION)}`);
85
+ throw new StorageFormatVersionError("opfs", `envelope/${magic}`, version, LOG_FORMAT_VERSION, version < LOG_FORMAT_VERSION ? "older" : "newer");
69
86
  }
70
87
  const payloadLength = view.getUint32(12, true);
71
- if (bytes.byteLength < ENVELOPE_HEADER_BYTES + payloadLength)
88
+ if (bytes.byteLength !== ENVELOPE_HEADER_BYTES + payloadLength)
72
89
  return undefined;
73
90
  const payload = bytes.subarray(ENVELOPE_HEADER_BYTES, ENVELOPE_HEADER_BYTES + payloadLength);
74
91
  if (crc32(payload) !== view.getUint32(16, true))
@@ -76,6 +93,7 @@ function decodeEnvelope(magic, bytes) {
76
93
  return payload;
77
94
  }
78
95
  const CHUNK_MAGIC = "MNWCHNK1";
96
+ const POSTING_CHUNK_MAGIC = "MNWPOST1";
79
97
  const SYNC_CHECKPOINT_MAGIC = "MNWCKPS1";
80
98
  /** Immutable artifact payloads (full-text base chunks) inside extents. */
81
99
  export function encodeChunk(value) {
@@ -85,6 +103,178 @@ export function decodeChunk(bytes) {
85
103
  const payload = decodeEnvelope(CHUNK_MAGIC, bytes);
86
104
  return payload === undefined ? undefined : decodeRecordJson(payload);
87
105
  }
106
+ /**
107
+ * Compact postings payload for OPFS. Terms are UTF-8 and the already-sorted row locators are
108
+ * delta-varints, keeping secondary indexes smaller than the table they accelerate.
109
+ * The envelope keeps the same version and checksum guarantees as every other immutable chunk.
110
+ */
111
+ export function encodePostingChunk(entries) {
112
+ const writer = new BinaryWriter();
113
+ writer.varuint(BigInt(entries.length));
114
+ for (const entry of entries) {
115
+ if (entry.rowIds.length !== entry.tf.length) {
116
+ throw new TypeError("Posting row IDs and term frequencies must have the same length");
117
+ }
118
+ const term = textEncoder.encode(entry.term);
119
+ if (term.byteLength === 0)
120
+ throw new TypeError("Posting terms cannot be empty");
121
+ if (entry.rowIds.length === 0)
122
+ throw new TypeError("Postings cannot have no row IDs");
123
+ writer.varuint(BigInt(term.byteLength));
124
+ writer.bytes(term);
125
+ writer.varuint(BigInt(entry.rowIds.length));
126
+ let previous = 0n;
127
+ for (const rowId of entry.rowIds) {
128
+ if (rowId <= previous || rowId > MAX_ROW_ID_EXCLUSIVE_END - 1n) {
129
+ throw new TypeError("Posting row IDs must be positive, uint64, and strictly sorted");
130
+ }
131
+ writer.varuint(rowId - previous);
132
+ previous = rowId;
133
+ }
134
+ for (const frequency of entry.tf) {
135
+ if (!Number.isSafeInteger(frequency) || frequency <= 0) {
136
+ throw new TypeError("Posting term frequencies must be positive whole numbers");
137
+ }
138
+ writer.varuint(BigInt(frequency));
139
+ }
140
+ }
141
+ return encodeEnvelope(POSTING_CHUNK_MAGIC, writer.finish());
142
+ }
143
+ /** `undefined` means the bytes are torn or are not a canonical postings envelope. */
144
+ export function decodePostingChunk(bytes) {
145
+ const payload = decodeEnvelope(POSTING_CHUNK_MAGIC, bytes);
146
+ if (payload === undefined)
147
+ return undefined;
148
+ const reader = new BinaryReader(payload);
149
+ const count = reader.safeInteger("posting count");
150
+ if (count > Math.floor(reader.remaining / 5)) {
151
+ throw new Error("Posting count exceeds the remaining payload");
152
+ }
153
+ const entries = new Array(count);
154
+ for (let entryIndex = 0; entryIndex < count; entryIndex += 1) {
155
+ const termLength = reader.safeInteger("posting term length");
156
+ if (termLength < 1 || termLength > reader.remaining) {
157
+ throw new Error("Posting term length exceeds the remaining payload");
158
+ }
159
+ const term = textDecoder.decode(reader.bytes(termLength));
160
+ const rowCount = reader.safeInteger("posting row count");
161
+ if (rowCount < 1 || rowCount > Math.floor(reader.remaining / 2)) {
162
+ throw new Error("Posting row count exceeds the remaining payload");
163
+ }
164
+ const rowIds = new Array(rowCount);
165
+ let previous = 0n;
166
+ for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
167
+ previous += reader.varuint();
168
+ if (previous < 1n || previous > MAX_ROW_ID_EXCLUSIVE_END - 1n) {
169
+ throw new RangeError("Posting row ID exceeds the uint64 range");
170
+ }
171
+ if (rowIndex > 0 && previous <= (rowIds[rowIndex - 1] ?? 0n)) {
172
+ throw new TypeError("Posting row IDs are not strictly sorted");
173
+ }
174
+ rowIds[rowIndex] = previous;
175
+ }
176
+ const tf = new Array(rowCount);
177
+ for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
178
+ const frequency = reader.safeInteger("posting term frequency");
179
+ if (frequency <= 0)
180
+ throw new TypeError("Posting term frequency must be positive");
181
+ tf[rowIndex] = frequency;
182
+ }
183
+ entries[entryIndex] = { term, rowIds, tf };
184
+ }
185
+ if (!reader.done)
186
+ throw new Error("Posting chunk has trailing bytes");
187
+ return entries;
188
+ }
189
+ class BinaryWriter {
190
+ #buffer = new Uint8Array(1_024);
191
+ #length = 0;
192
+ bytes(bytes) {
193
+ this.#reserve(bytes.byteLength);
194
+ this.#buffer.set(bytes, this.#length);
195
+ this.#length += bytes.byteLength;
196
+ }
197
+ varuint(value) {
198
+ if (value < 0n || value >= MAX_ROW_ID_EXCLUSIVE_END) {
199
+ throw new RangeError("A varuint must fit canonical uint64");
200
+ }
201
+ do {
202
+ this.#reserve(1);
203
+ const byte = Number(value & 0x7fn);
204
+ value >>= 7n;
205
+ this.#buffer[this.#length] = value === 0n ? byte : byte | 0x80;
206
+ this.#length += 1;
207
+ } while (value !== 0n);
208
+ }
209
+ finish() {
210
+ return this.#buffer.slice(0, this.#length);
211
+ }
212
+ #reserve(extra) {
213
+ if (!Number.isSafeInteger(extra) || extra < 0)
214
+ throw new RangeError("Invalid binary growth");
215
+ const needed = this.#length + extra;
216
+ if (!Number.isSafeInteger(needed))
217
+ throw new RangeError("Binary payload is too large");
218
+ if (needed <= this.#buffer.byteLength)
219
+ return;
220
+ let capacity = this.#buffer.byteLength;
221
+ while (capacity < needed) {
222
+ const doubled = capacity * 2;
223
+ capacity = Number.isSafeInteger(doubled) ? doubled : needed;
224
+ }
225
+ const grown = new Uint8Array(capacity);
226
+ grown.set(this.#buffer);
227
+ this.#buffer = grown;
228
+ }
229
+ }
230
+ class BinaryReader {
231
+ source;
232
+ #offset = 0;
233
+ constructor(source) {
234
+ this.source = source;
235
+ }
236
+ get done() {
237
+ return this.#offset === this.source.byteLength;
238
+ }
239
+ get remaining() {
240
+ return this.source.byteLength - this.#offset;
241
+ }
242
+ bytes(length) {
243
+ if (!Number.isSafeInteger(length) || length < 0 || length > this.remaining)
244
+ throw new Error("Posting chunk is truncated");
245
+ const bytes = this.source.subarray(this.#offset, this.#offset + length);
246
+ this.#offset += length;
247
+ return bytes;
248
+ }
249
+ varuint() {
250
+ let value = 0n;
251
+ let shift = 0n;
252
+ for (let byteIndex = 0; byteIndex < 10; byteIndex += 1) {
253
+ const byte = this.source[this.#offset];
254
+ if (byte === undefined)
255
+ throw new Error("Posting chunk is truncated");
256
+ this.#offset += 1;
257
+ if (byteIndex === 9 && (byte & 0xfe) !== 0) {
258
+ throw new Error("Posting chunk varuint exceeds uint64");
259
+ }
260
+ value |= BigInt(byte & 0x7f) << shift;
261
+ if ((byte & 0x80) === 0) {
262
+ if (byteIndex > 0 && (byte & 0x7f) === 0) {
263
+ throw new Error("Posting chunk varuint is overlong");
264
+ }
265
+ return value;
266
+ }
267
+ shift += 7n;
268
+ }
269
+ throw new Error("Posting chunk varuint is too wide");
270
+ }
271
+ safeInteger(label) {
272
+ const value = this.varuint();
273
+ if (value > BigInt(Number.MAX_SAFE_INTEGER))
274
+ throw new RangeError(`${label} is too large`);
275
+ return Number(value);
276
+ }
277
+ }
88
278
  /**
89
279
  * The leader's checkpoint slots are written synchronously (no compression — that needs an
90
280
  * await, and the checkpoint must land in the same synchronous run that resets the WAL, or
@@ -97,4 +287,3 @@ export function decodeSyncCheckpoint(bytes) {
97
287
  const payload = decodeEnvelope(SYNC_CHECKPOINT_MAGIC, bytes);
98
288
  return payload === undefined ? undefined : decodeRecordJson(payload);
99
289
  }
100
- //# sourceMappingURL=wire.js.map