@minnowdb/core 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +27 -1
- package/dist/block-format/block.d.ts +46 -0
- package/dist/block-format/block.d.ts.map +1 -0
- package/dist/block-format/block.js +244 -0
- package/dist/block-format/block.js.map +1 -0
- package/dist/block-format/checksum.d.ts +7 -0
- package/dist/block-format/checksum.d.ts.map +1 -0
- package/dist/block-format/checksum.js +56 -0
- package/dist/block-format/checksum.js.map +1 -0
- package/dist/block-format/codecs.d.ts +24 -0
- package/dist/block-format/codecs.d.ts.map +1 -0
- package/dist/block-format/codecs.js +102 -0
- package/dist/block-format/codecs.js.map +1 -0
- package/dist/block-format/column.d.ts +8 -0
- package/dist/block-format/column.d.ts.map +1 -0
- package/dist/block-format/column.js +166 -0
- package/dist/block-format/column.js.map +1 -0
- package/dist/block-format/index.d.ts +7 -0
- package/dist/block-format/index.d.ts.map +1 -0
- package/dist/block-format/index.js +7 -0
- package/dist/block-format/index.js.map +1 -0
- package/dist/block-format/physical.d.ts +20 -0
- package/dist/block-format/physical.d.ts.map +1 -0
- package/dist/block-format/physical.js +439 -0
- package/dist/block-format/physical.js.map +1 -0
- package/dist/block-format/types.d.ts +93 -0
- package/dist/block-format/types.d.ts.map +1 -0
- package/dist/block-format/types.js +2 -0
- package/dist/block-format/types.js.map +1 -0
- package/dist/engine/artifact-cache.d.ts +22 -0
- package/dist/engine/artifact-cache.d.ts.map +1 -0
- package/dist/engine/artifact-cache.js +68 -0
- package/dist/engine/artifact-cache.js.map +1 -0
- package/dist/engine/batch.d.ts +28 -0
- package/dist/engine/batch.d.ts.map +1 -0
- package/dist/engine/batch.js +38 -0
- package/dist/engine/batch.js.map +1 -0
- package/dist/engine/buffered-writer.d.ts +43 -0
- package/dist/engine/buffered-writer.d.ts.map +1 -0
- package/dist/engine/buffered-writer.js +158 -0
- package/dist/engine/buffered-writer.js.map +1 -0
- package/dist/engine/catalog.d.ts +82 -0
- package/dist/engine/catalog.d.ts.map +1 -0
- package/dist/engine/catalog.js +47 -0
- package/dist/engine/catalog.js.map +1 -0
- package/dist/engine/client.d.ts +211 -0
- package/dist/engine/client.d.ts.map +1 -0
- package/dist/engine/client.js +538 -0
- package/dist/engine/client.js.map +1 -0
- package/dist/engine/coordinator.d.ts +17 -0
- package/dist/engine/coordinator.d.ts.map +1 -0
- package/dist/engine/coordinator.js +60 -0
- package/dist/engine/coordinator.js.map +1 -0
- package/dist/engine/database.d.ts +661 -0
- package/dist/engine/database.d.ts.map +1 -0
- package/dist/engine/database.js +10313 -0
- package/dist/engine/database.js.map +1 -0
- package/dist/engine/defaults.d.ts +33 -0
- package/dist/engine/defaults.d.ts.map +1 -0
- package/dist/engine/defaults.js +87 -0
- package/dist/engine/defaults.js.map +1 -0
- package/dist/engine/dsl/db.d.ts +137 -0
- package/dist/engine/dsl/db.d.ts.map +1 -0
- package/dist/engine/dsl/db.js +161 -0
- package/dist/engine/dsl/db.js.map +1 -0
- package/dist/engine/dsl/expression.d.ts +172 -0
- package/dist/engine/dsl/expression.d.ts.map +1 -0
- package/dist/engine/dsl/expression.js +397 -0
- package/dist/engine/dsl/expression.js.map +1 -0
- package/dist/engine/dsl/index.d.ts +8 -0
- package/dist/engine/dsl/index.d.ts.map +1 -0
- package/dist/engine/dsl/index.js +7 -0
- package/dist/engine/dsl/index.js.map +1 -0
- package/dist/engine/dsl/live-query.d.ts +45 -0
- package/dist/engine/dsl/live-query.d.ts.map +1 -0
- package/dist/engine/dsl/live-query.js +116 -0
- package/dist/engine/dsl/live-query.js.map +1 -0
- package/dist/engine/dsl/mutations.d.ts +123 -0
- package/dist/engine/dsl/mutations.d.ts.map +1 -0
- package/dist/engine/dsl/mutations.js +241 -0
- package/dist/engine/dsl/mutations.js.map +1 -0
- package/dist/engine/dsl/select-query-builder.d.ts +148 -0
- package/dist/engine/dsl/select-query-builder.d.ts.map +1 -0
- package/dist/engine/dsl/select-query-builder.js +377 -0
- package/dist/engine/dsl/select-query-builder.js.map +1 -0
- package/dist/engine/dsl/sql-tag.d.ts +46 -0
- package/dist/engine/dsl/sql-tag.d.ts.map +1 -0
- package/dist/engine/dsl/sql-tag.js +85 -0
- package/dist/engine/dsl/sql-tag.js.map +1 -0
- package/dist/engine/dsl/types.d.ts +133 -0
- package/dist/engine/dsl/types.d.ts.map +1 -0
- package/dist/engine/dsl/types.js +9 -0
- package/dist/engine/dsl/types.js.map +1 -0
- package/dist/engine/errors.d.ts +55 -0
- package/dist/engine/errors.d.ts.map +1 -0
- package/dist/engine/errors.js +81 -0
- package/dist/engine/errors.js.map +1 -0
- package/dist/engine/fts.d.ts +93 -0
- package/dist/engine/fts.d.ts.map +1 -0
- package/dist/engine/fts.js +273 -0
- package/dist/engine/fts.js.map +1 -0
- package/dist/engine/group-index.d.ts +32 -0
- package/dist/engine/group-index.d.ts.map +1 -0
- package/dist/engine/group-index.js +366 -0
- package/dist/engine/group-index.js.map +1 -0
- package/dist/engine/index.d.ts +13 -0
- package/dist/engine/index.d.ts.map +1 -0
- package/dist/engine/index.js +12 -0
- package/dist/engine/index.js.map +1 -0
- package/dist/engine/join-index.d.ts +12 -0
- package/dist/engine/join-index.d.ts.map +1 -0
- package/dist/engine/join-index.js +217 -0
- package/dist/engine/join-index.js.map +1 -0
- package/dist/engine/live.d.ts +86 -0
- package/dist/engine/live.d.ts.map +1 -0
- package/dist/engine/live.js +289 -0
- package/dist/engine/live.js.map +1 -0
- package/dist/engine/memory.d.ts +37 -0
- package/dist/engine/memory.d.ts.map +1 -0
- package/dist/engine/memory.js +119 -0
- package/dist/engine/memory.js.map +1 -0
- package/dist/engine/optimizer.d.ts +20 -0
- package/dist/engine/optimizer.d.ts.map +1 -0
- package/dist/engine/optimizer.js +1170 -0
- package/dist/engine/optimizer.js.map +1 -0
- package/dist/engine/query-cache.d.ts +10 -0
- package/dist/engine/query-cache.d.ts.map +1 -0
- package/dist/engine/query-cache.js +58 -0
- package/dist/engine/query-cache.js.map +1 -0
- package/dist/engine/query.d.ts +844 -0
- package/dist/engine/query.d.ts.map +1 -0
- package/dist/engine/query.js +7450 -0
- package/dist/engine/query.js.map +1 -0
- package/dist/engine/schema-wire.d.ts +92 -0
- package/dist/engine/schema-wire.d.ts.map +1 -0
- package/dist/engine/schema-wire.js +115 -0
- package/dist/engine/schema-wire.js.map +1 -0
- package/dist/engine/schema.d.ts +394 -0
- package/dist/engine/schema.d.ts.map +1 -0
- package/dist/engine/schema.js +709 -0
- package/dist/engine/schema.js.map +1 -0
- package/dist/engine/sort-keys.d.ts +36 -0
- package/dist/engine/sort-keys.d.ts.map +1 -0
- package/dist/engine/sort-keys.js +60 -0
- package/dist/engine/sort-keys.js.map +1 -0
- package/dist/engine/sql-json.d.ts +30 -0
- package/dist/engine/sql-json.d.ts.map +1 -0
- package/dist/engine/sql-json.js +120 -0
- package/dist/engine/sql-json.js.map +1 -0
- package/dist/engine/sql-semantics.d.ts +31 -0
- package/dist/engine/sql-semantics.d.ts.map +1 -0
- package/dist/engine/sql-semantics.js +121 -0
- package/dist/engine/sql-semantics.js.map +1 -0
- package/dist/engine/vector.d.ts +87 -0
- package/dist/engine/vector.d.ts.map +1 -0
- package/dist/engine/vector.js +4227 -0
- package/dist/engine/vector.js.map +1 -0
- package/dist/engine/worker-host.d.ts +48 -0
- package/dist/engine/worker-host.d.ts.map +1 -0
- package/dist/engine/worker-host.js +501 -0
- package/dist/engine/worker-host.js.map +1 -0
- package/dist/engine/worker.d.ts +2 -0
- package/dist/engine/worker.d.ts.map +1 -0
- package/dist/engine/worker.js +14 -0
- package/dist/engine/worker.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -0
- package/dist/plan/index.d.ts +17 -0
- package/dist/plan/index.d.ts.map +1 -0
- package/dist/plan/index.js +17 -0
- package/dist/plan/index.js.map +1 -0
- package/dist/storage/fixture-shape.d.ts +42 -0
- package/dist/storage/fixture-shape.d.ts.map +1 -0
- package/dist/storage/fixture-shape.js +146 -0
- package/dist/storage/fixture-shape.js.map +1 -0
- package/dist/storage/index.d.ts +5 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +5 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/indexeddb.d.ts +139 -0
- package/dist/storage/indexeddb.d.ts.map +1 -0
- package/dist/storage/indexeddb.js +3161 -0
- package/dist/storage/indexeddb.js.map +1 -0
- package/dist/storage/memory.d.ts +121 -0
- package/dist/storage/memory.d.ts.map +1 -0
- package/dist/storage/memory.js +1455 -0
- package/dist/storage/memory.js.map +1 -0
- package/dist/storage/snapshot.d.ts +108 -0
- package/dist/storage/snapshot.d.ts.map +1 -0
- package/dist/storage/snapshot.js +276 -0
- package/dist/storage/snapshot.js.map +1 -0
- package/dist/storage/types.d.ts +895 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +1549 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/testing/index.d.ts +76 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +233 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/seeds.d.ts +11 -0
- package/dist/testing/seeds.d.ts.map +1 -0
- package/dist/testing/seeds.js +50 -0
- package/dist/testing/seeds.js.map +1 -0
- package/dist/transactions/index.d.ts +148 -0
- package/dist/transactions/index.d.ts.map +1 -0
- package/dist/transactions/index.js +662 -0
- package/dist/transactions/index.js.map +1 -0
- package/dist/worker-protocol/index.d.ts +84 -0
- package/dist/worker-protocol/index.d.ts.map +1 -0
- package/dist/worker-protocol/index.js +117 -0
- package/dist/worker-protocol/index.js.map +1 -0
- package/package.json +66 -16
- package/sql-feature-matrix.json +1275 -0
- package/index.d.ts +0 -1
- package/index.js +0 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Eric Wilhite
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,29 @@
|
|
|
1
1
|
# @minnowdb/core
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Columnar SQL engine for the browser: real SQL over immutable snapshots, durable on IndexedDB,
|
|
4
|
+
with no WebAssembly to download. Runs in a worker (`MinnowDatabase`) or on the main thread through
|
|
5
|
+
an RPC proxy (`MinnowDatabaseClient`); the API is identical over both.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install @minnowdb/core
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
- **One schema, everything typed** — `table()`, `column`, and `schema()` drive migrations,
|
|
12
|
+
autocomplete, row types, insert validation, and live subscriptions.
|
|
13
|
+
- **SQL is the contract** — every statement compiles through the same plan assembly, and the
|
|
14
|
+
supported surface ships as a checked-in feature matrix
|
|
15
|
+
(`@minnowdb/core/sql-feature-matrix.json`). The optional `@minnowdb/client` adds a typed builder
|
|
16
|
+
over the same published primitives.
|
|
17
|
+
- **Full-text search** — `MATCH` / BM25 on any column, no index DDL.
|
|
18
|
+
- **Live queries** — `.live()` subscriptions that update across tabs.
|
|
19
|
+
- **Durable browser storage** — compressed columnar blocks on IndexedDB, atomic multi-tab
|
|
20
|
+
commits, snapshot reads, compaction and GC.
|
|
21
|
+
- **Subpath exports** — `/storage`, `/client`, `/worker`, `/transactions`, `/block-format`,
|
|
22
|
+
`/testing`.
|
|
23
|
+
|
|
24
|
+
Quick start, guides, benchmark results, contributor workflows, and the complete API reference live
|
|
25
|
+
in the [Minnow docs](https://minnowdb.com/docs/) — the single source of truth.
|
|
26
|
+
|
|
27
|
+
## License
|
|
28
|
+
|
|
29
|
+
MIT
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { BlockDescription, BlockMetadata, ColumnInput, Compression, DecodedBlock, DecodedPhysicalBlock, PhysicalColumnPayload } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Version 1 layout (all integers little-endian):
|
|
4
|
+
*
|
|
5
|
+
* 0 magic "BRDB"
|
|
6
|
+
* 4 envelope checksum: crc32 over bytes [8, headerLength + metadataLength)
|
|
7
|
+
* 8 format version (u16)
|
|
8
|
+
* 10 header length (u16)
|
|
9
|
+
* 12 logical type id (u8), 13 physical encoding id (u8), 14 codec id (u8), 15 flags (u8)
|
|
10
|
+
* 16 row count (u32)
|
|
11
|
+
* 20 null count (u32)
|
|
12
|
+
* 24 metadata length (u32)
|
|
13
|
+
* 28 encoded length (u32)
|
|
14
|
+
* 32 stored length (u32)
|
|
15
|
+
* 36 payload checksum: crc32 over the uncompressed encoded payload
|
|
16
|
+
*
|
|
17
|
+
* The envelope checksum authenticates every header field and the metadata JSON, so header-only
|
|
18
|
+
* reads (zone-map pruning, block inventories) can trust derived statistics without paying the
|
|
19
|
+
* decompress-and-revalidate cost of the payload.
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* The block envelope version. Every persisted block carries it, and `readBlock` refuses any
|
|
23
|
+
* value it does not recognize -- so this number is a compatibility contract with every database
|
|
24
|
+
* already sitting in a user's browser, not an internal detail.
|
|
25
|
+
*
|
|
26
|
+
* Changing it means older blocks stop being readable unless a read path for them is kept. Before
|
|
27
|
+
* changing it, run `npm run fixture:format` on the *current* build to freeze what this version
|
|
28
|
+
* writes; format-compatibility.test.ts then holds the new build to still reading it.
|
|
29
|
+
*/
|
|
30
|
+
export declare const BLOCK_FORMAT_VERSION = 1;
|
|
31
|
+
/**
|
|
32
|
+
* Returns a conservative bound for the complete persisted block value: the fixed header, the
|
|
33
|
+
* exact UTF-8 JSON metadata, and the selected codec's maximum stored payload.
|
|
34
|
+
*/
|
|
35
|
+
export declare function maximumPhysicalBlockByteLength(encodedByteLength: number, metadata: BlockMetadata, compression: Compression): number;
|
|
36
|
+
export declare function encodeBlock(input: ColumnInput, compression?: Compression): Promise<Uint8Array>;
|
|
37
|
+
/** Encodes physical column bytes after validating their complete version-zero layout. */
|
|
38
|
+
export declare function encodePhysicalBlock(input: PhysicalColumnPayload, compression?: Compression): Promise<Uint8Array>;
|
|
39
|
+
export declare function decodeBlock(bytes: Uint8Array): Promise<DecodedBlock>;
|
|
40
|
+
/**
|
|
41
|
+
* Decompresses and checksum-verifies a block without materializing row values.
|
|
42
|
+
* The returned physical byte array is owned by the result.
|
|
43
|
+
*/
|
|
44
|
+
export declare function decodePhysicalBlock(bytes: Uint8Array): Promise<DecodedPhysicalBlock>;
|
|
45
|
+
export declare function inspectBlock(bytes: Uint8Array): BlockDescription;
|
|
46
|
+
//# sourceMappingURL=block.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"block.d.ts","sourceRoot":"","sources":["../../src/block-format/block.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAChB,aAAa,EACb,WAAW,EACX,WAAW,EACX,YAAY,EACZ,oBAAoB,EAEpB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAGpB;;;;;;;;;;;;;;;;;;GAkBG;AACH;;;;;;;;GAQG;AACH,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAkBtC;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,aAAa,EACvB,WAAW,EAAE,WAAW,GACvB,MAAM,CAsBR;AAED,wBAAsB,WAAW,CAC/B,KAAK,EAAE,WAAW,EAClB,WAAW,GAAE,WAAmB,GAC/B,OAAO,CAAC,UAAU,CAAC,CAarB;AAED,yFAAyF;AACzF,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,qBAAqB,EAC5B,WAAW,GAAE,WAAmB,GAC/B,OAAO,CAAC,UAAU,CAAC,CAErB;AAqCD,wBAAsB,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC,CAU1E;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAsB1F;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,gBAAgB,CA4DhE"}
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { crc32 } from "./checksum.js";
|
|
2
|
+
import { decodeColumn, encodeColumn } from "./column.js";
|
|
3
|
+
import { getCodec, getCompressionMemoryBound } from "./codecs.js";
|
|
4
|
+
import { MAX_PHYSICAL_COLUMN_BYTE_LENGTH, validatePhysicalColumn } from "./physical.js";
|
|
5
|
+
const MAGIC = Uint8Array.of(0x42, 0x52, 0x44, 0x42);
|
|
6
|
+
/**
|
|
7
|
+
* Version 1 layout (all integers little-endian):
|
|
8
|
+
*
|
|
9
|
+
* 0 magic "BRDB"
|
|
10
|
+
* 4 envelope checksum: crc32 over bytes [8, headerLength + metadataLength)
|
|
11
|
+
* 8 format version (u16)
|
|
12
|
+
* 10 header length (u16)
|
|
13
|
+
* 12 logical type id (u8), 13 physical encoding id (u8), 14 codec id (u8), 15 flags (u8)
|
|
14
|
+
* 16 row count (u32)
|
|
15
|
+
* 20 null count (u32)
|
|
16
|
+
* 24 metadata length (u32)
|
|
17
|
+
* 28 encoded length (u32)
|
|
18
|
+
* 32 stored length (u32)
|
|
19
|
+
* 36 payload checksum: crc32 over the uncompressed encoded payload
|
|
20
|
+
*
|
|
21
|
+
* The envelope checksum authenticates every header field and the metadata JSON, so header-only
|
|
22
|
+
* reads (zone-map pruning, block inventories) can trust derived statistics without paying the
|
|
23
|
+
* decompress-and-revalidate cost of the payload.
|
|
24
|
+
*/
|
|
25
|
+
/**
|
|
26
|
+
* The block envelope version. Every persisted block carries it, and `readBlock` refuses any
|
|
27
|
+
* value it does not recognize -- so this number is a compatibility contract with every database
|
|
28
|
+
* already sitting in a user's browser, not an internal detail.
|
|
29
|
+
*
|
|
30
|
+
* Changing it means older blocks stop being readable unless a read path for them is kept. Before
|
|
31
|
+
* changing it, run `npm run fixture:format` on the *current* build to freeze what this version
|
|
32
|
+
* writes; format-compatibility.test.ts then holds the new build to still reading it.
|
|
33
|
+
*/
|
|
34
|
+
export const BLOCK_FORMAT_VERSION = 1;
|
|
35
|
+
const VERSION = BLOCK_FORMAT_VERSION;
|
|
36
|
+
const HEADER_LENGTH = 40;
|
|
37
|
+
const ENVELOPE_CHECKSUM_START = 8;
|
|
38
|
+
const MAX_BLOCK_LENGTH = MAX_PHYSICAL_COLUMN_BYTE_LENGTH;
|
|
39
|
+
const textEncoder = new TextEncoder();
|
|
40
|
+
const textDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
41
|
+
const typeIds = { boolean: 1, number: 2, string: 3, datetime: 4 };
|
|
42
|
+
const typesById = new Map(Object.entries(typeIds).map(([type, id]) => [id, type]));
|
|
43
|
+
// Codec 1 was run-length encoding, removed because it was both slower and larger than raw on
|
|
44
|
+
// every column shape measured. The ids of the survivors do not move: renumbering would make an
|
|
45
|
+
// old block decode as the wrong codec instead of failing.
|
|
46
|
+
const codecIds = { raw: 0, gzip: 2 };
|
|
47
|
+
const codecsById = new Map(Object.entries(codecIds).map(([codec, id]) => [id, codec]));
|
|
48
|
+
/**
|
|
49
|
+
* Returns a conservative bound for the complete persisted block value: the fixed header, the
|
|
50
|
+
* exact UTF-8 JSON metadata, and the selected codec's maximum stored payload.
|
|
51
|
+
*/
|
|
52
|
+
export function maximumPhysicalBlockByteLength(encodedByteLength, metadata, compression) {
|
|
53
|
+
assertLength(encodedByteLength, "encoded payload");
|
|
54
|
+
let serializedMetadata;
|
|
55
|
+
try {
|
|
56
|
+
serializedMetadata = JSON.stringify(metadata);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
throw new TypeError("Block metadata must be JSON serializable");
|
|
60
|
+
}
|
|
61
|
+
if (typeof serializedMetadata !== "string") {
|
|
62
|
+
throw new TypeError("Block metadata must be JSON serializable");
|
|
63
|
+
}
|
|
64
|
+
const metadataByteLength = textEncoder.encode(serializedMetadata).byteLength;
|
|
65
|
+
assertLength(metadataByteLength, "metadata");
|
|
66
|
+
const maximumStoredPayloadBytes = getCompressionMemoryBound(compression, encodedByteLength).maximumOutputBytes;
|
|
67
|
+
const total = HEADER_LENGTH + metadataByteLength + maximumStoredPayloadBytes;
|
|
68
|
+
if (!Number.isSafeInteger(total)) {
|
|
69
|
+
throw new RangeError("Stored block byte bound exceeds the safe integer range");
|
|
70
|
+
}
|
|
71
|
+
return total;
|
|
72
|
+
}
|
|
73
|
+
export async function encodeBlock(input, compression = "raw") {
|
|
74
|
+
if (input.values.length > 0xffffffff)
|
|
75
|
+
throw new RangeError("Too many rows in one block");
|
|
76
|
+
const encoded = encodeColumn(input);
|
|
77
|
+
return encodeValidatedPhysicalBlock({
|
|
78
|
+
type: input.type,
|
|
79
|
+
rowCount: input.values.length,
|
|
80
|
+
nullCount: encoded.nullCount,
|
|
81
|
+
bytes: encoded.bytes,
|
|
82
|
+
metadata: encoded.metadata,
|
|
83
|
+
}, compression);
|
|
84
|
+
}
|
|
85
|
+
/** Encodes physical column bytes after validating their complete version-zero layout. */
|
|
86
|
+
export async function encodePhysicalBlock(input, compression = "raw") {
|
|
87
|
+
return encodeValidatedPhysicalBlock(validatePhysicalColumn(input), compression);
|
|
88
|
+
}
|
|
89
|
+
async function encodeValidatedPhysicalBlock(input, compression) {
|
|
90
|
+
const metadata = textEncoder.encode(JSON.stringify(input.metadata));
|
|
91
|
+
const stored = await getCodec(compression).compress(input.bytes);
|
|
92
|
+
assertLength(metadata.byteLength, "metadata");
|
|
93
|
+
assertLength(input.bytes.byteLength, "encoded payload");
|
|
94
|
+
assertLength(stored.byteLength, "stored payload");
|
|
95
|
+
const output = new Uint8Array(HEADER_LENGTH + metadata.byteLength + stored.byteLength);
|
|
96
|
+
output.set(MAGIC);
|
|
97
|
+
const view = new DataView(output.buffer);
|
|
98
|
+
view.setUint16(8, VERSION, true);
|
|
99
|
+
view.setUint16(10, HEADER_LENGTH, true);
|
|
100
|
+
view.setUint8(12, typeIds[input.type]);
|
|
101
|
+
view.setUint8(13, typeIds[input.type]); // the physical encoding mirrors the logical type
|
|
102
|
+
view.setUint8(14, codecIds[compression]);
|
|
103
|
+
view.setUint8(15, 0);
|
|
104
|
+
view.setUint32(16, input.rowCount, true);
|
|
105
|
+
view.setUint32(20, input.nullCount, true);
|
|
106
|
+
view.setUint32(24, metadata.byteLength, true);
|
|
107
|
+
view.setUint32(28, input.bytes.byteLength, true);
|
|
108
|
+
view.setUint32(32, stored.byteLength, true);
|
|
109
|
+
view.setUint32(36, crc32(input.bytes), true);
|
|
110
|
+
output.set(metadata, HEADER_LENGTH);
|
|
111
|
+
output.set(stored, HEADER_LENGTH + metadata.byteLength);
|
|
112
|
+
view.setUint32(4, crc32(output.subarray(ENVELOPE_CHECKSUM_START, HEADER_LENGTH + metadata.byteLength)), true);
|
|
113
|
+
return output;
|
|
114
|
+
}
|
|
115
|
+
export async function decodeBlock(bytes) {
|
|
116
|
+
const decoded = await decodePhysicalBlock(bytes);
|
|
117
|
+
return {
|
|
118
|
+
description: decoded.description,
|
|
119
|
+
column: decodeColumn(decoded.description.type, decoded.column.bytes, decoded.description.rowCount),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Decompresses and checksum-verifies a block without materializing row values.
|
|
124
|
+
* The returned physical byte array is owned by the result.
|
|
125
|
+
*/
|
|
126
|
+
export async function decodePhysicalBlock(bytes) {
|
|
127
|
+
const description = inspectBlock(bytes);
|
|
128
|
+
const metadataLength = bytes.byteLength - HEADER_LENGTH - description.storedLength;
|
|
129
|
+
const stored = bytes.subarray(HEADER_LENGTH + metadataLength);
|
|
130
|
+
const encoded = await getCodec(description.compression).decompress(stored, description.encodedLength);
|
|
131
|
+
if (crc32(encoded) !== description.checksum)
|
|
132
|
+
throw new Error("Block checksum mismatch");
|
|
133
|
+
const column = validatePhysicalColumn({
|
|
134
|
+
type: description.type,
|
|
135
|
+
rowCount: description.rowCount,
|
|
136
|
+
bytes: encoded,
|
|
137
|
+
});
|
|
138
|
+
if (column.nullCount !== description.nullCount)
|
|
139
|
+
throw new Error("Block null count mismatch");
|
|
140
|
+
if (!metadataEquals(column.metadata, description.metadata)) {
|
|
141
|
+
throw new Error("Block metadata does not match physical payload");
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
description,
|
|
145
|
+
column,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
export function inspectBlock(bytes) {
|
|
149
|
+
if (bytes.byteLength < HEADER_LENGTH)
|
|
150
|
+
throw new Error("Truncated block header");
|
|
151
|
+
if (!MAGIC.every((byte, index) => bytes[index] === byte))
|
|
152
|
+
throw new Error("Invalid block magic");
|
|
153
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
154
|
+
const formatVersion = view.getUint16(8, true);
|
|
155
|
+
if (formatVersion !== VERSION) {
|
|
156
|
+
throw new Error(`Unsupported block version ${String(formatVersion)}`);
|
|
157
|
+
}
|
|
158
|
+
if (view.getUint16(10, true) !== HEADER_LENGTH)
|
|
159
|
+
throw new Error("Invalid block header length");
|
|
160
|
+
const metadataLength = view.getUint32(24, true);
|
|
161
|
+
const encodedLength = view.getUint32(28, true);
|
|
162
|
+
const storedLength = view.getUint32(32, true);
|
|
163
|
+
assertLength(metadataLength, "metadata");
|
|
164
|
+
assertLength(encodedLength, "encoded payload");
|
|
165
|
+
assertLength(storedLength, "stored payload");
|
|
166
|
+
if (HEADER_LENGTH + metadataLength + storedLength !== bytes.byteLength) {
|
|
167
|
+
throw new Error("Block length mismatch");
|
|
168
|
+
}
|
|
169
|
+
// Verified before any header field is trusted: the envelope checksum covers every field after
|
|
170
|
+
// itself plus the metadata JSON, making header-only reads safe against silent corruption.
|
|
171
|
+
const envelope = crc32(bytes.subarray(ENVELOPE_CHECKSUM_START, HEADER_LENGTH + metadataLength));
|
|
172
|
+
if (envelope !== view.getUint32(4, true)) {
|
|
173
|
+
throw new Error("Block envelope checksum mismatch");
|
|
174
|
+
}
|
|
175
|
+
const type = typesById.get(view.getUint8(12));
|
|
176
|
+
if (type === undefined)
|
|
177
|
+
throw new Error("Unknown logical type");
|
|
178
|
+
if (view.getUint8(13) !== typeIds[type])
|
|
179
|
+
throw new Error("Unsupported physical encoding");
|
|
180
|
+
const codecId = view.getUint8(14);
|
|
181
|
+
const compression = codecsById.get(codecId);
|
|
182
|
+
if (compression === undefined) {
|
|
183
|
+
throw new Error(codecId === 1
|
|
184
|
+
? "Block uses the removed RLE codec; rewrite it as raw or gzip with an older build"
|
|
185
|
+
: "Unknown compression codec");
|
|
186
|
+
}
|
|
187
|
+
if (view.getUint8(15) !== 0)
|
|
188
|
+
throw new Error("Unsupported mandatory block flags");
|
|
189
|
+
const rowCount = view.getUint32(16, true);
|
|
190
|
+
const nullCount = view.getUint32(20, true);
|
|
191
|
+
if (nullCount > rowCount)
|
|
192
|
+
throw new Error("Null count exceeds row count");
|
|
193
|
+
let parsedMetadata;
|
|
194
|
+
try {
|
|
195
|
+
parsedMetadata = JSON.parse(textDecoder.decode(bytes.subarray(HEADER_LENGTH, HEADER_LENGTH + metadataLength)));
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
throw new Error("Invalid block metadata");
|
|
199
|
+
}
|
|
200
|
+
const metadata = parseMetadata(parsedMetadata);
|
|
201
|
+
return {
|
|
202
|
+
formatVersion,
|
|
203
|
+
type,
|
|
204
|
+
compression,
|
|
205
|
+
rowCount,
|
|
206
|
+
nullCount,
|
|
207
|
+
encodedLength,
|
|
208
|
+
storedLength,
|
|
209
|
+
checksum: view.getUint32(36, true),
|
|
210
|
+
metadata,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
function assertLength(length, name) {
|
|
214
|
+
if (!Number.isSafeInteger(length) || length < 0 || length > MAX_BLOCK_LENGTH) {
|
|
215
|
+
throw new RangeError(`Invalid ${name} length`);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
function parseMetadata(value) {
|
|
219
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
220
|
+
throw new Error("Invalid block metadata shape");
|
|
221
|
+
}
|
|
222
|
+
const zoneMap = Reflect.get(value, "zoneMap");
|
|
223
|
+
if (zoneMap !== undefined) {
|
|
224
|
+
if (typeof zoneMap !== "object" ||
|
|
225
|
+
zoneMap === null ||
|
|
226
|
+
!Number.isFinite(Reflect.get(zoneMap, "min")) ||
|
|
227
|
+
!Number.isFinite(Reflect.get(zoneMap, "max"))) {
|
|
228
|
+
throw new Error("Invalid zone map");
|
|
229
|
+
}
|
|
230
|
+
const min = Reflect.get(zoneMap, "min");
|
|
231
|
+
const max = Reflect.get(zoneMap, "max");
|
|
232
|
+
if (min > max)
|
|
233
|
+
throw new Error("Invalid zone map");
|
|
234
|
+
return { zoneMap: { min, max } };
|
|
235
|
+
}
|
|
236
|
+
return {};
|
|
237
|
+
}
|
|
238
|
+
function metadataEquals(left, right) {
|
|
239
|
+
if (left.zoneMap === undefined || right.zoneMap === undefined) {
|
|
240
|
+
return left.zoneMap === undefined && right.zoneMap === undefined;
|
|
241
|
+
}
|
|
242
|
+
return left.zoneMap.min === right.zoneMap.min && left.zoneMap.max === right.zoneMap.max;
|
|
243
|
+
}
|
|
244
|
+
//# sourceMappingURL=block.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"block.js","sourceRoot":"","sources":["../../src/block-format/block.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,+BAA+B,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAaxF,MAAM,KAAK,GAAG,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACpD;;;;;;;;;;;;;;;;;;GAkBG;AACH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AACtC,MAAM,OAAO,GAAG,oBAAoB,CAAC;AACrC,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,MAAM,gBAAgB,GAAG,+BAA+B,CAAC;AACzD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AACtC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE9D,MAAM,OAAO,GAAgC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;AAC/F,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAmB,CAAC,CAAC,CAAC,CAAC;AAClG,6FAA6F;AAC7F,+FAA+F;AAC/F,0DAA0D;AAC1D,MAAM,QAAQ,GAAgC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAClE,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAoB,CAAC,CAAC,CAC1E,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,iBAAyB,EACzB,QAAuB,EACvB,WAAwB;IAExB,YAAY,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;IACnD,IAAI,kBAA2B,CAAC;IAChC,IAAI,CAAC;QACH,kBAAkB,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,0CAA0C,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,OAAO,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QAC3C,MAAM,IAAI,SAAS,CAAC,0CAA0C,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,kBAAkB,GAAG,WAAW,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,UAAU,CAAC;IAC7E,YAAY,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;IAC7C,MAAM,yBAAyB,GAAG,yBAAyB,CACzD,WAAW,EACX,iBAAiB,CAClB,CAAC,kBAAkB,CAAC;IACrB,MAAM,KAAK,GAAG,aAAa,GAAG,kBAAkB,GAAG,yBAAyB,CAAC;IAC7E,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,UAAU,CAAC,wDAAwD,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAkB,EAClB,cAA2B,KAAK;IAEhC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,UAAU;QAAE,MAAM,IAAI,UAAU,CAAC,4BAA4B,CAAC,CAAC;IACzF,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACpC,OAAO,4BAA4B,CACjC;QACE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;QAC7B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,EACD,WAAW,CACZ,CAAC;AACJ,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAA4B,EAC5B,cAA2B,KAAK;IAEhC,OAAO,4BAA4B,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC;AAClF,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,KAA8B,EAC9B,WAAwB;IAExB,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjE,YAAY,CAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC9C,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;IACxD,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;IAElD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACvF,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClB,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACjC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACvC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,iDAAiD;IACzF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IACzC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACzC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAC9C,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACjD,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACxD,IAAI,CAAC,SAAS,CACZ,CAAC,EACD,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,EAAE,aAAa,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,EACpF,IAAI,CACL,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAiB;IACjD,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACjD,OAAO;QACL,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,MAAM,EAAE,YAAY,CAClB,OAAO,CAAC,WAAW,CAAC,IAAI,EACxB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,OAAO,CAAC,WAAW,CAAC,QAAQ,CAC7B;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAiB;IACzD,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,GAAG,aAAa,GAAG,WAAW,CAAC,YAAY,CAAC;IACnF,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,aAAa,GAAG,cAAc,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,UAAU,CAChE,MAAM,EACN,WAAW,CAAC,aAAa,CAC1B,CAAC;IACF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,WAAW,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,sBAAsB,CAAC;QACpC,IAAI,EAAE,WAAW,CAAC,IAAI;QACtB,QAAQ,EAAE,WAAW,CAAC,QAAQ;QAC9B,KAAK,EAAE,OAAO;KACf,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,SAAS,KAAK,WAAW,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC7F,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,OAAO;QACL,WAAW;QACX,MAAM;KACP,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB;IAC5C,IAAI,KAAK,CAAC,UAAU,GAAG,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAChF,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACjG,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC9C,IAAI,aAAa,KAAK,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC/F,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAChD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC9C,YAAY,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;IACzC,YAAY,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAC/C,YAAY,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;IAC7C,IAAI,aAAa,GAAG,cAAc,GAAG,YAAY,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAC3C,CAAC;IACD,8FAA8F;IAC9F,0FAA0F;IAC1F,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,uBAAuB,EAAE,aAAa,GAAG,cAAc,CAAC,CAAC,CAAC;IAChG,IAAI,QAAQ,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9C,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAC1F,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,OAAO,KAAK,CAAC;YACX,CAAC,CAAC,iFAAiF;YACnF,CAAC,CAAC,2BAA2B,CAChC,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IAClF,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC3C,IAAI,SAAS,GAAG,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC1E,IAAI,cAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,cAAc,GAAG,IAAI,CAAC,KAAK,CACzB,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAAC,CAAC,CACvE,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD,MAAM,QAAQ,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;IAC/C,OAAO;QACL,aAAa;QACb,IAAI;QACJ,WAAW;QACX,QAAQ;QACR,SAAS;QACT,aAAa;QACb,YAAY;QACZ,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC;QAClC,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAAc,EAAE,IAAY;IAChD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,gBAAgB,EAAE,CAAC;QAC7E,MAAM,IAAI,UAAU,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC;IACjD,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAY,CAAC;IACzD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,IACE,OAAO,OAAO,KAAK,QAAQ;YAC3B,OAAO,KAAK,IAAI;YAChB,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC7C,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,EAC7C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAW,CAAC;QAClD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAW,CAAC;QAClD,IAAI,GAAG,GAAG,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACnD,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,KAAoB;IAC/D,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC;IACnE,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;AAC1F,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CRC-32 (IEEE) via slicing-by-8. Every decoded block is checksummed end to end, so this is a
|
|
3
|
+
* per-query hot path: eight table folds per 8-byte word beat the byte-at-a-time loop several
|
|
4
|
+
* times over, and the tail falls back to single-byte steps.
|
|
5
|
+
*/
|
|
6
|
+
export declare function crc32(bytes: Uint8Array): number;
|
|
7
|
+
//# sourceMappingURL=checksum.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checksum.d.ts","sourceRoot":"","sources":["../../src/block-format/checksum.ts"],"names":[],"mappings":"AAkBA;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmC/C"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
const TABLES = new Uint32Array(256 * 8);
|
|
2
|
+
for (let index = 0; index < 256; index += 1) {
|
|
3
|
+
let value = index;
|
|
4
|
+
for (let bit = 0; bit < 8; bit += 1) {
|
|
5
|
+
value = (value & 1) === 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
|
6
|
+
}
|
|
7
|
+
TABLES[index] = value >>> 0;
|
|
8
|
+
}
|
|
9
|
+
// Slicing-by-8 extension tables: TABLES[k*256+b] is the CRC of byte b followed by k zero bytes,
|
|
10
|
+
// letting the hot loop fold eight bytes per iteration instead of one.
|
|
11
|
+
for (let slice = 1; slice < 8; slice += 1) {
|
|
12
|
+
for (let index = 0; index < 256; index += 1) {
|
|
13
|
+
const previous = TABLES[(slice - 1) * 256 + index] ?? 0;
|
|
14
|
+
TABLES[slice * 256 + index] = ((previous >>> 8) ^ (TABLES[previous & 0xff] ?? 0)) >>> 0;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* CRC-32 (IEEE) via slicing-by-8. Every decoded block is checksummed end to end, so this is a
|
|
19
|
+
* per-query hot path: eight table folds per 8-byte word beat the byte-at-a-time loop several
|
|
20
|
+
* times over, and the tail falls back to single-byte steps.
|
|
21
|
+
*/
|
|
22
|
+
export function crc32(bytes) {
|
|
23
|
+
let checksum = 0xffffffff;
|
|
24
|
+
const length = bytes.byteLength;
|
|
25
|
+
const wordEnd = length - (length % 8);
|
|
26
|
+
let index = 0;
|
|
27
|
+
while (index < wordEnd) {
|
|
28
|
+
const low = (checksum ^
|
|
29
|
+
((bytes[index] ?? 0) |
|
|
30
|
+
((bytes[index + 1] ?? 0) << 8) |
|
|
31
|
+
((bytes[index + 2] ?? 0) << 16) |
|
|
32
|
+
((bytes[index + 3] ?? 0) << 24))) >>>
|
|
33
|
+
0;
|
|
34
|
+
const high = ((bytes[index + 4] ?? 0) |
|
|
35
|
+
((bytes[index + 5] ?? 0) << 8) |
|
|
36
|
+
((bytes[index + 6] ?? 0) << 16) |
|
|
37
|
+
((bytes[index + 7] ?? 0) << 24)) >>>
|
|
38
|
+
0;
|
|
39
|
+
checksum =
|
|
40
|
+
((TABLES[1792 + (low & 0xff)] ?? 0) ^
|
|
41
|
+
(TABLES[1536 + ((low >>> 8) & 0xff)] ?? 0) ^
|
|
42
|
+
(TABLES[1280 + ((low >>> 16) & 0xff)] ?? 0) ^
|
|
43
|
+
(TABLES[1024 + (low >>> 24)] ?? 0) ^
|
|
44
|
+
(TABLES[768 + (high & 0xff)] ?? 0) ^
|
|
45
|
+
(TABLES[512 + ((high >>> 8) & 0xff)] ?? 0) ^
|
|
46
|
+
(TABLES[256 + ((high >>> 16) & 0xff)] ?? 0) ^
|
|
47
|
+
(TABLES[high >>> 24] ?? 0)) >>>
|
|
48
|
+
0;
|
|
49
|
+
index += 8;
|
|
50
|
+
}
|
|
51
|
+
for (; index < length; index += 1) {
|
|
52
|
+
checksum = (checksum >>> 8) ^ (TABLES[(checksum ^ (bytes[index] ?? 0)) & 0xff] ?? 0);
|
|
53
|
+
}
|
|
54
|
+
return (checksum ^ 0xffffffff) >>> 0;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=checksum.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checksum.js","sourceRoot":"","sources":["../../src/block-format/checksum.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;AAExC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,GAAG,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;IAC5C,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACpC,KAAK,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC;AAC9B,CAAC;AACD,gGAAgG;AAChG,sEAAsE;AACtE,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,GAAG,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,CAAC,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1F,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,KAAiB;IACrC,IAAI,QAAQ,GAAG,UAAU,CAAC;IAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,OAAO,EAAE,CAAC;QACvB,MAAM,GAAG,GACP,CAAC,QAAQ;YACP,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClB,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACrC,CAAC,CAAC;QACJ,MAAM,IAAI,GACR,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAClC,CAAC,CAAC;QACJ,QAAQ;YACN,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjC,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC1C,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC3C,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC1C,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC3C,CAAC,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7B,CAAC,CAAC;QACJ,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAClC,QAAQ,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Compression } from "./types.js";
|
|
2
|
+
export interface CompressionCodec {
|
|
3
|
+
readonly id: Compression;
|
|
4
|
+
compress(bytes: Uint8Array): Promise<Uint8Array>;
|
|
5
|
+
decompress(bytes: Uint8Array, expectedLength: number): Promise<Uint8Array>;
|
|
6
|
+
}
|
|
7
|
+
export interface CompressionMemoryBound {
|
|
8
|
+
/** Largest returned buffer owned by the caller. */
|
|
9
|
+
readonly maximumOutputBytes: number;
|
|
10
|
+
/** Additional JavaScript-owned byte storage retained while producing the result. */
|
|
11
|
+
readonly scratchBytes: number;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The raw codec returns its input view unchanged in both directions: callers treat compressed
|
|
15
|
+
* and decompressed payloads as read-only (encode copies the payload into the block envelope,
|
|
16
|
+
* decode consumers only read or copy out), so a full-payload defensive copy per block would be
|
|
17
|
+
* pure overhead on the default compression path. The returned view may share the caller's
|
|
18
|
+
* buffer at a non-zero byte offset.
|
|
19
|
+
*/
|
|
20
|
+
export declare const rawCodec: CompressionCodec;
|
|
21
|
+
export declare function getCompressionMemoryBound(compression: Compression, inputLength: number): CompressionMemoryBound;
|
|
22
|
+
export declare const gzipCodec: CompressionCodec;
|
|
23
|
+
export declare function getCodec(id: Compression): CompressionCodec;
|
|
24
|
+
//# sourceMappingURL=codecs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codecs.d.ts","sourceRoot":"","sources":["../../src/block-format/codecs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACjD,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CAC5E;AAED,MAAM,WAAW,sBAAsB;IACrC,mDAAmD;IACnD,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,oFAAoF;IACpF,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,EAAE,gBAStB,CAAC;AAqBF,wBAAgB,yBAAyB,CACvC,WAAW,EAAE,WAAW,EACxB,WAAW,EAAE,MAAM,GAClB,sBAAsB,CAcxB;AA+CD,eAAO,MAAM,SAAS,EAAE,gBAgBvB,CAAC;AAEF,wBAAgB,QAAQ,CAAC,EAAE,EAAE,WAAW,GAAG,gBAAgB,CAO1D"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The raw codec returns its input view unchanged in both directions: callers treat compressed
|
|
3
|
+
* and decompressed payloads as read-only (encode copies the payload into the block envelope,
|
|
4
|
+
* decode consumers only read or copy out), so a full-payload defensive copy per block would be
|
|
5
|
+
* pure overhead on the default compression path. The returned view may share the caller's
|
|
6
|
+
* buffer at a non-zero byte offset.
|
|
7
|
+
*/
|
|
8
|
+
export const rawCodec = {
|
|
9
|
+
id: "raw",
|
|
10
|
+
compress(bytes) {
|
|
11
|
+
return Promise.resolve(bytes);
|
|
12
|
+
},
|
|
13
|
+
decompress(bytes, expectedLength) {
|
|
14
|
+
if (bytes.byteLength !== expectedLength)
|
|
15
|
+
throw new Error("Raw payload length mismatch");
|
|
16
|
+
return Promise.resolve(bytes);
|
|
17
|
+
},
|
|
18
|
+
};
|
|
19
|
+
function assertInputLength(inputLength, codec) {
|
|
20
|
+
if (!Number.isSafeInteger(inputLength) || inputLength < 0) {
|
|
21
|
+
throw new RangeError(`Invalid ${codec} input length`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function checkedLinearBound(inputLength, multiplier, overhead, codec) {
|
|
25
|
+
assertInputLength(inputLength, codec);
|
|
26
|
+
if (inputLength > Math.floor((Number.MAX_SAFE_INTEGER - overhead) / multiplier)) {
|
|
27
|
+
throw new RangeError(`Invalid ${codec} input length`);
|
|
28
|
+
}
|
|
29
|
+
return inputLength * multiplier + overhead;
|
|
30
|
+
}
|
|
31
|
+
export function getCompressionMemoryBound(compression, inputLength) {
|
|
32
|
+
switch (compression) {
|
|
33
|
+
case "raw":
|
|
34
|
+
assertInputLength(inputLength, compression);
|
|
35
|
+
return { maximumOutputBytes: inputLength, scratchBytes: 0 };
|
|
36
|
+
case "gzip": {
|
|
37
|
+
// Native gzip encoders stay well below two bytes per input byte. The extra envelope keeps
|
|
38
|
+
// small and empty streams bounded too. `transform` retains an input copy and its output
|
|
39
|
+
// chunks while joining them, so include both as scratch storage at the copy boundary.
|
|
40
|
+
checkedLinearBound(inputLength, 5, 128, compression);
|
|
41
|
+
const maximumOutputBytes = checkedLinearBound(inputLength, 2, 64, compression);
|
|
42
|
+
return { maximumOutputBytes, scratchBytes: inputLength + maximumOutputBytes };
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
async function transform(bytes, stream, maximumOutputLength = Number.POSITIVE_INFINITY, boundErrorMessage = "Decompressed payload exceeds declared length") {
|
|
47
|
+
const write = (async () => {
|
|
48
|
+
const writer = stream.writable.getWriter();
|
|
49
|
+
await writer.write(new Uint8Array(bytes));
|
|
50
|
+
await writer.close();
|
|
51
|
+
})();
|
|
52
|
+
const [, result] = await Promise.all([
|
|
53
|
+
write,
|
|
54
|
+
readBounded(stream.readable, maximumOutputLength, boundErrorMessage),
|
|
55
|
+
]);
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
58
|
+
async function readBounded(readable, maximumLength, boundErrorMessage) {
|
|
59
|
+
const reader = readable.getReader();
|
|
60
|
+
const chunks = [];
|
|
61
|
+
let length = 0;
|
|
62
|
+
for (;;) {
|
|
63
|
+
const { done, value } = await reader.read();
|
|
64
|
+
if (done)
|
|
65
|
+
break;
|
|
66
|
+
length += value.byteLength;
|
|
67
|
+
if (length > maximumLength) {
|
|
68
|
+
await reader.cancel(boundErrorMessage);
|
|
69
|
+
throw new Error(boundErrorMessage);
|
|
70
|
+
}
|
|
71
|
+
chunks.push(value);
|
|
72
|
+
}
|
|
73
|
+
const output = new Uint8Array(length);
|
|
74
|
+
let offset = 0;
|
|
75
|
+
for (const chunk of chunks) {
|
|
76
|
+
output.set(chunk, offset);
|
|
77
|
+
offset += chunk.byteLength;
|
|
78
|
+
}
|
|
79
|
+
return output;
|
|
80
|
+
}
|
|
81
|
+
export const gzipCodec = {
|
|
82
|
+
id: "gzip",
|
|
83
|
+
async compress(bytes) {
|
|
84
|
+
const { maximumOutputBytes } = getCompressionMemoryBound("gzip", bytes.byteLength);
|
|
85
|
+
return transform(bytes, new CompressionStream("gzip"), maximumOutputBytes, "Gzip payload exceeds compression bound");
|
|
86
|
+
},
|
|
87
|
+
async decompress(bytes, expectedLength) {
|
|
88
|
+
const output = await transform(bytes, new DecompressionStream("gzip"), expectedLength);
|
|
89
|
+
if (output.byteLength !== expectedLength)
|
|
90
|
+
throw new Error("Gzip payload length mismatch");
|
|
91
|
+
return output;
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
export function getCodec(id) {
|
|
95
|
+
switch (id) {
|
|
96
|
+
case "raw":
|
|
97
|
+
return rawCodec;
|
|
98
|
+
case "gzip":
|
|
99
|
+
return gzipCodec;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=codecs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codecs.js","sourceRoot":"","sources":["../../src/block-format/codecs.ts"],"names":[],"mappings":"AAeA;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAqB;IACxC,EAAE,EAAE,KAAK;IACT,QAAQ,CAAC,KAAK;QACZ,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IACD,UAAU,CAAC,KAAK,EAAE,cAAc;QAC9B,IAAI,KAAK,CAAC,UAAU,KAAK,cAAc;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACxF,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CACF,CAAC;AAEF,SAAS,iBAAiB,CAAC,WAAmB,EAAE,KAAkB;IAChE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,WAAW,KAAK,eAAe,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,WAAmB,EACnB,UAAkB,EAClB,QAAgB,EAChB,KAAkB;IAElB,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IACtC,IAAI,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,gBAAgB,GAAG,QAAQ,CAAC,GAAG,UAAU,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,UAAU,CAAC,WAAW,KAAK,eAAe,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,WAAwB,EACxB,WAAmB;IAEnB,QAAQ,WAAW,EAAE,CAAC;QACpB,KAAK,KAAK;YACR,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YAC5C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QAC9D,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,0FAA0F;YAC1F,wFAAwF;YACxF,sFAAsF;YACtF,kBAAkB,CAAC,WAAW,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;YACrD,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC;YAC/E,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,WAAW,GAAG,kBAAkB,EAAE,CAAC;QAChF,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,KAAiB,EACjB,MAA+C,EAC/C,mBAAmB,GAAG,MAAM,CAAC,iBAAiB,EAC9C,iBAAiB,GAAG,8CAA8C;IAElE,MAAM,KAAK,GAAG,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC,CAAC,EAAE,CAAC;IACL,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACnC,KAAK;QACL,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,mBAAmB,EAAE,iBAAiB,CAAC;KACrE,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,QAAoC,EACpC,aAAqB,EACrB,iBAAyB;IAEzB,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,SAAS,CAAC;QACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI;YAAE,MAAM;QAChB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,IAAI,MAAM,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAqB;IACzC,EAAE,EAAE,MAAM;IACV,KAAK,CAAC,QAAQ,CAAC,KAAK;QAClB,MAAM,EAAE,kBAAkB,EAAE,GAAG,yBAAyB,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QACnF,OAAO,SAAS,CACd,KAAK,EACL,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAC7B,kBAAkB,EAClB,wCAAwC,CACzC,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,cAAc;QACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC,CAAC;QACvF,IAAI,MAAM,CAAC,UAAU,KAAK,cAAc;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC1F,OAAO,MAAM,CAAC;IAChB,CAAC;CACF,CAAC;AAEF,MAAM,UAAU,QAAQ,CAAC,EAAe;IACtC,QAAQ,EAAE,EAAE,CAAC;QACX,KAAK,KAAK;YACR,OAAO,QAAQ,CAAC;QAClB,KAAK,MAAM;YACT,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { BlockMetadata, ColumnInput, DecodedColumn, LogicalType } from "./types.js";
|
|
2
|
+
export declare function encodeColumn(input: ColumnInput): {
|
|
3
|
+
bytes: Uint8Array;
|
|
4
|
+
metadata: BlockMetadata;
|
|
5
|
+
nullCount: number;
|
|
6
|
+
};
|
|
7
|
+
export declare function decodeColumn<T extends LogicalType>(type: T, bytes: Uint8Array, rowCount: number): DecodedColumn<T>;
|
|
8
|
+
//# sourceMappingURL=column.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"column.d.ts","sourceRoot":"","sources":["../../src/block-format/column.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAoBzF,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG;IAChD,KAAK,EAAE,UAAU,CAAC;IAClB,QAAQ,EAAE,aAAa,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB,CAoFA;AAED,wBAAgB,YAAY,CAAC,CAAC,SAAS,WAAW,EAChD,IAAI,EAAE,CAAC,EACP,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,MAAM,GACf,aAAa,CAAC,CAAC,CAAC,CAyClB"}
|