@brass-build/client 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +124 -45
- package/CHANGELOG.md +80 -4
- package/README.md +666 -179
- package/dist/api-fetch.d.ts +8 -0
- package/dist/api-fetch.d.ts.map +1 -1
- package/dist/api-fetch.js +35 -8
- package/dist/api-fetch.js.map +1 -1
- package/dist/batched-get.d.ts +11 -0
- package/dist/batched-get.d.ts.map +1 -0
- package/dist/batched-get.js +92 -0
- package/dist/batched-get.js.map +1 -0
- package/dist/call-service.d.ts +4 -0
- package/dist/call-service.d.ts.map +1 -1
- package/dist/call-service.js +66 -3
- package/dist/call-service.js.map +1 -1
- package/dist/client.d.ts +48 -2
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +918 -115
- package/dist/client.js.map +1 -1
- package/dist/document-checkpoints.d.ts +25 -2
- package/dist/document-checkpoints.d.ts.map +1 -1
- package/dist/document-checkpoints.js +258 -61
- package/dist/document-checkpoints.js.map +1 -1
- package/dist/document-commands.d.ts.map +1 -1
- package/dist/document-commands.js +5 -3
- package/dist/document-commands.js.map +1 -1
- package/dist/document-contents.d.ts +10 -0
- package/dist/document-contents.d.ts.map +1 -1
- package/dist/document-contents.js +23 -1
- package/dist/document-contents.js.map +1 -1
- package/dist/document-set-subscription.d.ts +56 -0
- package/dist/document-set-subscription.d.ts.map +1 -0
- package/dist/document-set-subscription.js +240 -0
- package/dist/document-set-subscription.js.map +1 -0
- package/dist/document-signals.d.ts +6 -0
- package/dist/document-signals.d.ts.map +1 -1
- package/dist/document-signals.js +31 -1
- package/dist/document-signals.js.map +1 -1
- package/dist/document-streams.d.ts +36 -2
- package/dist/document-streams.d.ts.map +1 -1
- package/dist/document-streams.js +69 -18
- package/dist/document-streams.js.map +1 -1
- package/dist/document-thumbnail-capture.d.ts +12 -2
- package/dist/document-thumbnail-capture.d.ts.map +1 -1
- package/dist/document-thumbnail-capture.js +103 -27
- package/dist/document-thumbnail-capture.js.map +1 -1
- package/dist/documents-namespace.d.ts +92 -10
- package/dist/documents-namespace.d.ts.map +1 -1
- package/dist/documents-namespace.js +285 -39
- package/dist/documents-namespace.js.map +1 -1
- package/dist/documents.d.ts +173 -8
- package/dist/documents.d.ts.map +1 -1
- package/dist/documents.js +1778 -310
- package/dist/documents.js.map +1 -1
- package/dist/errors.d.ts +3 -3
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/fetch-retry.d.ts +1 -0
- package/dist/fetch-retry.d.ts.map +1 -1
- package/dist/fetch-retry.js +2 -0
- package/dist/fetch-retry.js.map +1 -1
- package/dist/file-choice.d.ts +2 -0
- package/dist/file-choice.d.ts.map +1 -0
- package/dist/file-choice.js +27 -0
- package/dist/file-choice.js.map +1 -0
- package/dist/guest-link.d.ts +18 -1
- package/dist/guest-link.d.ts.map +1 -1
- package/dist/guest-link.js +329 -1
- package/dist/guest-link.js.map +1 -1
- package/dist/handle-types.d.ts +67 -5
- package/dist/handle-types.d.ts.map +1 -1
- package/dist/handles.d.ts +15 -19
- package/dist/handles.d.ts.map +1 -1
- package/dist/handles.js +231 -76
- package/dist/handles.js.map +1 -1
- package/dist/hosted-ui.d.ts +61 -0
- package/dist/hosted-ui.d.ts.map +1 -1
- package/dist/hosted-ui.js +94 -42
- package/dist/hosted-ui.js.map +1 -1
- package/dist/import-contract.d.ts +74 -0
- package/dist/import-contract.d.ts.map +1 -0
- package/dist/import-contract.js +163 -0
- package/dist/import-contract.js.map +1 -0
- package/dist/import-guest.d.ts +9 -0
- package/dist/import-guest.d.ts.map +1 -0
- package/dist/import-guest.js +155 -0
- package/dist/import-guest.js.map +1 -0
- package/dist/import-report.d.ts +17 -0
- package/dist/import-report.d.ts.map +1 -0
- package/dist/import-report.js +7 -0
- package/dist/import-report.js.map +1 -0
- package/dist/importer-wait.d.ts +2 -0
- package/dist/importer-wait.d.ts.map +1 -0
- package/dist/importer-wait.js +5 -0
- package/dist/importer-wait.js.map +1 -0
- package/dist/importer.d.ts +18 -4
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +322 -105
- package/dist/importer.js.map +1 -1
- package/dist/in-development.d.ts +2 -0
- package/dist/in-development.d.ts.map +1 -0
- package/dist/in-development.js +10 -0
- package/dist/in-development.js.map +1 -0
- package/dist/index.d.ts +16 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +24 -8
- package/dist/index.js.map +1 -1
- package/dist/init-keys.d.ts +1 -1
- package/dist/init-keys.d.ts.map +1 -1
- package/dist/init-keys.js +6 -1
- package/dist/init-keys.js.map +1 -1
- package/dist/input-documents.d.ts +67 -0
- package/dist/input-documents.d.ts.map +1 -0
- package/dist/input-documents.js +693 -0
- package/dist/input-documents.js.map +1 -0
- package/dist/inputs-status.d.ts +80 -0
- package/dist/inputs-status.d.ts.map +1 -0
- package/dist/inputs-status.js +460 -0
- package/dist/inputs-status.js.map +1 -0
- package/dist/inputs.d.ts +2 -0
- package/dist/inputs.d.ts.map +1 -0
- package/dist/inputs.js +793 -0
- package/dist/inputs.js.map +1 -0
- package/dist/internal.d.ts +6 -1
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +4 -1
- package/dist/internal.js.map +1 -1
- package/dist/join-content.d.ts +33 -0
- package/dist/join-content.d.ts.map +1 -0
- package/dist/join-content.js +114 -0
- package/dist/join-content.js.map +1 -0
- package/dist/join-stamp.d.ts +3 -0
- package/dist/join-stamp.d.ts.map +1 -0
- package/dist/join-stamp.js +12 -0
- package/dist/join-stamp.js.map +1 -0
- package/dist/manifest-drift.d.ts +9 -0
- package/dist/manifest-drift.d.ts.map +1 -0
- package/dist/manifest-drift.js +46 -0
- package/dist/manifest-drift.js.map +1 -0
- package/dist/open-document.d.ts +109 -2
- package/dist/open-document.d.ts.map +1 -1
- package/dist/open-document.js +1822 -811
- package/dist/open-document.js.map +1 -1
- package/dist/page-restore.d.ts +12 -0
- package/dist/page-restore.d.ts.map +1 -0
- package/dist/page-restore.js +23 -0
- package/dist/page-restore.js.map +1 -0
- package/dist/placement.d.ts +13 -0
- package/dist/placement.d.ts.map +1 -0
- package/dist/placement.js +34 -0
- package/dist/placement.js.map +1 -0
- package/dist/presence.d.ts +1 -0
- package/dist/presence.d.ts.map +1 -1
- package/dist/presence.js +16 -4
- package/dist/presence.js.map +1 -1
- package/dist/read-only-document.d.ts +87 -0
- package/dist/read-only-document.d.ts.map +1 -0
- package/dist/read-only-document.js +4 -0
- package/dist/read-only-document.js.map +1 -0
- package/dist/record-paging.d.ts +63 -6
- package/dist/record-paging.d.ts.map +1 -1
- package/dist/record-paging.js +122 -29
- package/dist/record-paging.js.map +1 -1
- package/dist/references.d.ts +40 -0
- package/dist/references.d.ts.map +1 -0
- package/dist/references.js +135 -0
- package/dist/references.js.map +1 -0
- package/dist/request-lane.d.ts +5 -0
- package/dist/request-lane.d.ts.map +1 -0
- package/dist/request-lane.js +48 -0
- package/dist/request-lane.js.map +1 -0
- package/dist/sealed-arrow.d.ts +170 -0
- package/dist/sealed-arrow.d.ts.map +1 -0
- package/dist/sealed-arrow.js +769 -0
- package/dist/sealed-arrow.js.map +1 -0
- package/dist/sealed-cache.d.ts +21 -0
- package/dist/sealed-cache.d.ts.map +1 -0
- package/dist/sealed-cache.js +247 -0
- package/dist/sealed-cache.js.map +1 -0
- package/dist/sealed-encode.d.ts +87 -0
- package/dist/sealed-encode.d.ts.map +1 -0
- package/dist/sealed-encode.js +876 -0
- package/dist/sealed-encode.js.map +1 -0
- package/dist/sealed-handles.d.ts +45 -0
- package/dist/sealed-handles.d.ts.map +1 -0
- package/dist/sealed-handles.js +408 -0
- package/dist/sealed-handles.js.map +1 -0
- package/dist/sealed-store.d.ts +242 -0
- package/dist/sealed-store.d.ts.map +1 -0
- package/dist/sealed-store.js +1155 -0
- package/dist/sealed-store.js.map +1 -0
- package/dist/serializer.d.ts +6 -15
- package/dist/serializer.d.ts.map +1 -1
- package/dist/session-toolbar.d.ts +20 -5
- package/dist/session-toolbar.d.ts.map +1 -1
- package/dist/session-toolbar.js +419 -67
- package/dist/session-toolbar.js.map +1 -1
- package/dist/session.d.ts +38 -1
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +562 -79
- package/dist/session.js.map +1 -1
- package/dist/sha256.d.ts +3 -0
- package/dist/sha256.d.ts.map +1 -0
- package/dist/sha256.js +74 -0
- package/dist/sha256.js.map +1 -0
- package/dist/stable-handles.d.ts +19 -0
- package/dist/stable-handles.d.ts.map +1 -0
- package/dist/stable-handles.js +193 -0
- package/dist/stable-handles.js.map +1 -0
- package/dist/staged-seal.d.ts +2 -0
- package/dist/staged-seal.d.ts.map +1 -0
- package/dist/staged-seal.js +76 -0
- package/dist/staged-seal.js.map +1 -0
- package/dist/subscriber-delivery.d.ts +8 -0
- package/dist/subscriber-delivery.d.ts.map +1 -0
- package/dist/subscriber-delivery.js +31 -0
- package/dist/subscriber-delivery.js.map +1 -0
- package/dist/tail-op-import.d.ts +10 -0
- package/dist/tail-op-import.d.ts.map +1 -0
- package/dist/tail-op-import.js +33 -0
- package/dist/tail-op-import.js.map +1 -0
- package/dist/topic-lifecycle.d.ts +6 -0
- package/dist/topic-lifecycle.d.ts.map +1 -1
- package/dist/topic-lifecycle.js +32 -2
- package/dist/topic-lifecycle.js.map +1 -1
- package/dist/trial-replace-dialog.d.ts +10 -0
- package/dist/trial-replace-dialog.d.ts.map +1 -0
- package/dist/trial-replace-dialog.js +153 -0
- package/dist/trial-replace-dialog.js.map +1 -0
- package/dist/types.d.ts +8 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/undo.d.ts +60 -5
- package/dist/undo.d.ts.map +1 -1
- package/dist/undo.js +344 -75
- package/dist/undo.js.map +1 -1
- package/dist/version-bar.d.ts +1 -0
- package/dist/version-bar.d.ts.map +1 -1
- package/dist/version-bar.js +2 -1
- package/dist/version-bar.js.map +1 -1
- package/dist/version.d.ts +2 -2
- package/dist/version.js +1 -1
- package/dist/ws-client.d.ts +21 -0
- package/dist/ws-client.d.ts.map +1 -1
- package/dist/ws-client.js +192 -8
- package/dist/ws-client.js.map +1 -1
- package/dist/ws-frame-limit.d.ts +1 -0
- package/dist/ws-frame-limit.d.ts.map +1 -1
- package/dist/ws-frame-limit.js.map +1 -1
- package/docs/app-manifest.md +121 -17
- package/docs/custom-chrome.md +24 -16
- package/docs/hosted-ui-advanced.md +38 -6
- package/docs/imported-documents.md +104 -51
- package/docs/presence.md +16 -4
- package/docs/schemas.md +8 -5
- package/docs/services.md +6 -6
- package/docs/streams.md +139 -150
- package/docs/text-fields.md +26 -19
- package/package.json +5 -2
- package/src/api-fetch.ts +53 -3
- package/src/auth-flow.ts +1 -1
- package/src/batched-get.ts +117 -0
- package/src/call-service.ts +78 -3
- package/src/client.ts +1022 -112
- package/src/document-checkpoints.ts +307 -65
- package/src/document-commands.ts +5 -5
- package/src/document-contents.ts +27 -1
- package/src/document-set-subscription.ts +274 -0
- package/src/document-signals.ts +47 -5
- package/src/document-streams.ts +85 -16
- package/src/document-thumbnail-capture.ts +118 -32
- package/src/documents-namespace.ts +539 -84
- package/src/documents.ts +2017 -348
- package/src/errors.ts +17 -6
- package/src/fetch-retry.ts +5 -0
- package/src/file-choice.ts +26 -0
- package/src/guest-link.ts +362 -1
- package/src/handle-types.ts +73 -47
- package/src/handles.ts +253 -83
- package/src/hosted-ui.ts +211 -50
- package/src/import-contract.ts +227 -0
- package/src/import-guest.ts +193 -0
- package/src/import-report.ts +24 -0
- package/src/importer-wait.ts +4 -0
- package/src/importer.ts +386 -131
- package/src/in-development.ts +10 -0
- package/src/index.ts +76 -16
- package/src/init-keys.ts +6 -1
- package/src/input-documents.ts +819 -0
- package/src/inputs-status.ts +598 -0
- package/src/inputs.ts +1090 -0
- package/src/internal.ts +7 -0
- package/src/join-content.ts +135 -0
- package/src/join-stamp.ts +12 -0
- package/src/manifest-drift.ts +54 -0
- package/src/open-document.ts +2346 -930
- package/src/page-restore.ts +38 -0
- package/src/placement.ts +54 -0
- package/src/presence.ts +21 -5
- package/src/read-only-document.ts +133 -0
- package/src/record-paging.ts +147 -25
- package/src/references.ts +207 -0
- package/src/request-lane.ts +59 -0
- package/src/sealed-arrow.ts +843 -0
- package/src/sealed-cache.ts +267 -0
- package/src/sealed-encode.ts +1015 -0
- package/src/sealed-handles.ts +486 -0
- package/src/sealed-store.ts +1330 -0
- package/src/serializer.ts +8 -20
- package/src/session-toolbar.ts +463 -72
- package/src/session.ts +740 -99
- package/src/sha256.ts +65 -0
- package/src/stable-handles.ts +235 -0
- package/src/staged-seal.ts +90 -0
- package/src/subscriber-delivery.ts +37 -0
- package/src/tail-op-import.ts +30 -0
- package/src/topic-lifecycle.ts +38 -2
- package/src/trial-replace-dialog.ts +163 -0
- package/src/types.ts +6 -6
- package/src/undo.ts +386 -73
- package/src/version-bar.ts +5 -1
- package/src/version.ts +1 -1
- package/src/ws-client.ts +208 -16
- package/src/ws-frame-limit.ts +1 -0
|
@@ -0,0 +1,1330 @@
|
|
|
1
|
+
// The FROZEN form: a stream's content as plain bytes, with no CRDT engine on
|
|
2
|
+
// either side of it.
|
|
3
|
+
//
|
|
4
|
+
// Content is JSON in the shape the stream's schema describes: each top-level
|
|
5
|
+
// entry is a root, and a root is a map (an object whose entries are its
|
|
6
|
+
// records), a list (an array), or a text (a string). An ordered map is two
|
|
7
|
+
// roots, the map and the list of its keys. That is exactly what an importer
|
|
8
|
+
// returns and what `doc.map`, `doc.list`, `doc.orderedMap` and `doc.text` read
|
|
9
|
+
// back, so a stream nobody has edited is fully determined by it. A CRDT
|
|
10
|
+
// snapshot of the same content pays for per-container state, a version vector
|
|
11
|
+
// and an op history describing no edits, and every reader pays an engine to
|
|
12
|
+
// get the content back; this holds the content and nothing else.
|
|
13
|
+
//
|
|
14
|
+
// It lives beside `record-paging.ts` because the published `@brass-build/client`
|
|
15
|
+
// ships unbundled and can only import published packages and its own relative
|
|
16
|
+
// files, so the frame and the one decoder every reader uses are defined once,
|
|
17
|
+
// here. Deliberately loro-free.
|
|
18
|
+
//
|
|
19
|
+
// The stored frame (version 4, below) holds every map as typed Arrow tables
|
|
20
|
+
// and the stream as one ZSTD frame (`sealed-arrow.ts` reads the tables,
|
|
21
|
+
// `sealed-encode.ts` writes them). Version 5 is the same tables gzipped, the
|
|
22
|
+
// staging form a producer without ZSTD uploads. Content can also be held in
|
|
23
|
+
// memory with the same roots (`SealedContent.fromJSON`), so a reader never
|
|
24
|
+
// asks which it holds. A join serves the stored frame only to a caller whose
|
|
25
|
+
// `accept` names it (`SEALED_ACCEPT`).
|
|
26
|
+
//
|
|
27
|
+
// A list whose items are exactly a map's keys (an ordered map's order list) is
|
|
28
|
+
// not stored: the map is stored in the list's order and the list is read back
|
|
29
|
+
// as the map's keys in stored order.
|
|
30
|
+
|
|
31
|
+
import {
|
|
32
|
+
HEADER_KEY,
|
|
33
|
+
META_KEY,
|
|
34
|
+
ORDER_KEY,
|
|
35
|
+
RECORDS_KEY,
|
|
36
|
+
SCHEMA_VERSION_KEY,
|
|
37
|
+
recordValueOf,
|
|
38
|
+
type PackedRecord,
|
|
39
|
+
} from './record-paging.js';
|
|
40
|
+
import { Gunzip } from 'fflate';
|
|
41
|
+
import type { ArrowData, ArrowReader, ReadContext, ReadyTable, TableMeta } from './sealed-arrow.js';
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The peer every conversion of frozen content into a CRDT authors as.
|
|
45
|
+
*
|
|
46
|
+
* Pinned, so one frozen stream produces one document: two sides that convert
|
|
47
|
+
* the same content independently write the same ops, and their later edits
|
|
48
|
+
* merge rather than doubling every record.
|
|
49
|
+
*
|
|
50
|
+
* A live editor never authors as this peer. It keeps the one its own document
|
|
51
|
+
* was opened with, so every op it writes after a conversion is its own.
|
|
52
|
+
*/
|
|
53
|
+
export const CONVERSION_PEER_ID = '1' as const;
|
|
54
|
+
|
|
55
|
+
/** `BRST`. Four bytes so a reader handed the wrong object fails on its first
|
|
56
|
+
* word rather than on a length field it read out of a compressed run. */
|
|
57
|
+
const MAGIC = 0x42_52_53_54;
|
|
58
|
+
/** Named roots, every map as typed Arrow tables, the whole stream one ZSTD
|
|
59
|
+
* frame. What the platform stores; only its encoder writes it. */
|
|
60
|
+
export const VERSION_ARROW = 4;
|
|
61
|
+
/** Frame 4's body under gzip rather than ZSTD: what a producer without a ZSTD
|
|
62
|
+
* compressor (a browser) uploads. Staging only; the seal stores it as frame
|
|
63
|
+
* 4, so no reader is ever handed it. */
|
|
64
|
+
export const VERSION_ARROW_STAGED = 5;
|
|
65
|
+
/** The newest frame this build reads. */
|
|
66
|
+
export const NEWEST_VERSION = VERSION_ARROW;
|
|
67
|
+
|
|
68
|
+
/** The join `accept` token naming the stored frames this build decodes. */
|
|
69
|
+
export const SEALED_ACCEPT = `sealed@${VERSION_ARROW}`;
|
|
70
|
+
|
|
71
|
+
// Magic (u32), version (u8), reserved (3 bytes, zero), then 8 reserved bytes
|
|
72
|
+
// before the compressed body.
|
|
73
|
+
const PROLOGUE_BYTES = 16;
|
|
74
|
+
|
|
75
|
+
/** One root of frozen content, as JSON: an object is a map, an array a list,
|
|
76
|
+
* a string a text. */
|
|
77
|
+
export type SealedRootValue =
|
|
78
|
+
| Record<string, unknown>
|
|
79
|
+
| readonly unknown[]
|
|
80
|
+
| string;
|
|
81
|
+
|
|
82
|
+
/** A stream's content as JSON, one entry per root. */
|
|
83
|
+
export type SealedContentJson = Record<string, SealedRootValue>;
|
|
84
|
+
|
|
85
|
+
export type SealedRootKind = 'map' | 'list' | 'text';
|
|
86
|
+
|
|
87
|
+
/** The kind a JSON value would be stored as, or null for a value no root can
|
|
88
|
+
* hold. */
|
|
89
|
+
export function sealedRootKindOf(value: unknown): SealedRootKind | null {
|
|
90
|
+
if (typeof value === 'string') return 'text';
|
|
91
|
+
if (Array.isArray(value)) return 'list';
|
|
92
|
+
if (value !== null && typeof value === 'object') return 'map';
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** One header entry of a record store: the format's own preamble (an IFC
|
|
97
|
+
* `FILE_DESCRIPTION`, a DXF header variable), tagged the way a record is. */
|
|
98
|
+
export interface SealedHeaderEntry {
|
|
99
|
+
tag: string;
|
|
100
|
+
body: unknown;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** What produced a stream's content: an importer's id and version. A reader
|
|
104
|
+
* answers from the content whoever produced it; this is for deciding which
|
|
105
|
+
* stored streams an importer fix should rebuild. */
|
|
106
|
+
export interface SealedStoreBuiltBy {
|
|
107
|
+
id: string;
|
|
108
|
+
version: string;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** A record store as triples, which is what a caller holding parsed records
|
|
112
|
+
* hands the encoder without first building a map object from them. */
|
|
113
|
+
export interface SealedStoreSource {
|
|
114
|
+
schemaVersion: string;
|
|
115
|
+
header: readonly SealedHeaderEntry[];
|
|
116
|
+
/** Absent for content nothing parsed (an app's own). */
|
|
117
|
+
builtBy?: SealedStoreBuiltBy;
|
|
118
|
+
/** `[key, tag, body]` triples in the order the store holds them. */
|
|
119
|
+
records: PackedRecord<unknown>[];
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** A record-store entry split into its tag and body: the tag of a value in
|
|
123
|
+
* the record shape (`{ type, params }` and nothing else), or null for any
|
|
124
|
+
* other value, which is the body whole. `recordValueOf` rebuilds the value. */
|
|
125
|
+
export interface SealedRecord {
|
|
126
|
+
tag: string | null;
|
|
127
|
+
body: unknown;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* One map root of frozen content, stored or held in memory.
|
|
132
|
+
*
|
|
133
|
+
* Opening one decodes nothing. A keyed read builds the record it asks for;
|
|
134
|
+
* `forEach` walks the records in stored order without keeping them. Values
|
|
135
|
+
* are the stored JSON values, built afresh on every read.
|
|
136
|
+
*/
|
|
137
|
+
export interface SealedMapReader {
|
|
138
|
+
/** Entries in the map. */
|
|
139
|
+
size(): number;
|
|
140
|
+
has(key: string): boolean;
|
|
141
|
+
get(key: string): unknown;
|
|
142
|
+
/** Every key, in stored order. */
|
|
143
|
+
keys(): string[];
|
|
144
|
+
/** Every entry, in stored order. */
|
|
145
|
+
forEach(visit: (key: string, value: unknown) => void): void;
|
|
146
|
+
/** Every entry, in stored order, decoded as the iteration reaches it: a
|
|
147
|
+
* caller that stops early pays for the records it read. */
|
|
148
|
+
entries(): IterableIterator<[string, unknown]>;
|
|
149
|
+
/** The JSON text of every value, in bytes. Reads every value. */
|
|
150
|
+
valueBytes(): number;
|
|
151
|
+
/** The whole map as the object it was stored from. */
|
|
152
|
+
toJSON(): Record<string, unknown>;
|
|
153
|
+
/** Drop what keyed reads decoded and kept. */
|
|
154
|
+
releaseDecoded(): void;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// --- Encoding ---------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
/** Whether a value is a record (`{ type, params }` and nothing else). */
|
|
160
|
+
function isRecordShaped(value: unknown): value is { type: string; params: unknown } {
|
|
161
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
164
|
+
const o = value as Record<string, unknown>;
|
|
165
|
+
let count = 0;
|
|
166
|
+
for (const key in o) {
|
|
167
|
+
if (!Object.prototype.hasOwnProperty.call(o, key)) continue;
|
|
168
|
+
count += 1;
|
|
169
|
+
if (key !== 'type' && key !== 'params') return false;
|
|
170
|
+
}
|
|
171
|
+
return count === 2 && typeof o['type'] === 'string';
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** A value split the way a record store reports it. */
|
|
175
|
+
export function sealedRecordOf(value: unknown): SealedRecord {
|
|
176
|
+
return isRecordShaped(value)
|
|
177
|
+
? { tag: value.type, body: value.params }
|
|
178
|
+
: { tag: null, body: value };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** A map root as the encoder walks it: its keys in stored order, and the
|
|
182
|
+
* value each one holds. `release` drops a value once it is encoded. */
|
|
183
|
+
/** @internal */
|
|
184
|
+
export interface MapEntries {
|
|
185
|
+
readonly keys: readonly string[];
|
|
186
|
+
value(index: number): unknown;
|
|
187
|
+
release(index: number): void;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function entriesFromTriples(records: PackedRecord<unknown>[]): MapEntries {
|
|
191
|
+
return {
|
|
192
|
+
keys: records.map((r) => r[0]),
|
|
193
|
+
value: (i) => recordValueOf(records[i]![1], records[i]![2]),
|
|
194
|
+
release: (i) => {
|
|
195
|
+
(records[i] as unknown[])[2] = undefined;
|
|
196
|
+
},
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function entriesFromObject(
|
|
201
|
+
object: Record<string, unknown>,
|
|
202
|
+
keys: readonly string[],
|
|
203
|
+
): MapEntries {
|
|
204
|
+
return {
|
|
205
|
+
keys,
|
|
206
|
+
value: (i) => object[keys[i]!],
|
|
207
|
+
release: (i) => {
|
|
208
|
+
object[keys[i]!] = undefined;
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** @internal */
|
|
214
|
+
export type RootPlan =
|
|
215
|
+
| { kind: 'map'; name: string; entries: MapEntries }
|
|
216
|
+
| { kind: 'list'; name: string; items: readonly unknown[] }
|
|
217
|
+
| { kind: 'list'; name: string; keysOf: string }
|
|
218
|
+
| { kind: 'text'; name: string; text: string };
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* The roots `content` is stored as, in its order: an ordered map's key list
|
|
222
|
+
* folded into the map it orders, and every map's entries in stored order.
|
|
223
|
+
*
|
|
224
|
+
* Throws for a root that is not a map, a list or a string.
|
|
225
|
+
*
|
|
226
|
+
* @internal
|
|
227
|
+
*/
|
|
228
|
+
export function planSealedContent(content: SealedContentJson): RootPlan[] {
|
|
229
|
+
const maps = new Map<string, Record<string, unknown>>();
|
|
230
|
+
const lists: { name: string; items: readonly unknown[] }[] = [];
|
|
231
|
+
for (const name of Object.keys(content)) {
|
|
232
|
+
const value = content[name];
|
|
233
|
+
const kind = sealedRootKindOf(value);
|
|
234
|
+
if (kind === null) {
|
|
235
|
+
throw new TypeError(
|
|
236
|
+
`Content root "${name}" is neither an object, an array, nor a string.`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
if (kind === 'map') maps.set(name, value as Record<string, unknown>);
|
|
240
|
+
else if (kind === 'list') lists.push({ name, items: value as unknown[] });
|
|
241
|
+
}
|
|
242
|
+
// An ordered map: a list that is exactly a map's keys orders that map and
|
|
243
|
+
// is not stored. One list per map, first come.
|
|
244
|
+
const orderOf = new Map<string, readonly string[]>();
|
|
245
|
+
const listKeysOf = new Map<string, string>();
|
|
246
|
+
for (const { name, items } of lists) {
|
|
247
|
+
for (const [mapName, object] of maps) {
|
|
248
|
+
if (orderOf.has(mapName)) continue;
|
|
249
|
+
if (isKeyPermutation(items, object)) {
|
|
250
|
+
orderOf.set(mapName, items as readonly string[]);
|
|
251
|
+
listKeysOf.set(name, mapName);
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
const roots: RootPlan[] = [];
|
|
257
|
+
for (const name of Object.keys(content)) {
|
|
258
|
+
const value = content[name];
|
|
259
|
+
const kind = sealedRootKindOf(value)!;
|
|
260
|
+
if (kind === 'map') {
|
|
261
|
+
const object = value as Record<string, unknown>;
|
|
262
|
+
const keys = orderOf.get(name) ?? ownDefinedKeys(object);
|
|
263
|
+
roots.push({ kind: 'map', name, entries: entriesFromObject(object, keys) });
|
|
264
|
+
} else if (kind === 'list') {
|
|
265
|
+
const mapName = listKeysOf.get(name);
|
|
266
|
+
roots.push(
|
|
267
|
+
mapName !== undefined
|
|
268
|
+
? { kind: 'list', name, keysOf: mapName }
|
|
269
|
+
: { kind: 'list', name, items: value as unknown[] },
|
|
270
|
+
);
|
|
271
|
+
} else {
|
|
272
|
+
roots.push({ kind: 'text', name, text: value as string });
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return roots;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* The roots a record store handed over as triples is stored as.
|
|
280
|
+
*
|
|
281
|
+
* @internal
|
|
282
|
+
*/
|
|
283
|
+
export function planSealedStore(source: SealedStoreSource): RootPlan[] {
|
|
284
|
+
return [
|
|
285
|
+
{
|
|
286
|
+
kind: 'map',
|
|
287
|
+
name: META_KEY,
|
|
288
|
+
entries: entriesFromObject({ [SCHEMA_VERSION_KEY]: source.schemaVersion }, [
|
|
289
|
+
SCHEMA_VERSION_KEY,
|
|
290
|
+
]),
|
|
291
|
+
},
|
|
292
|
+
{
|
|
293
|
+
kind: 'list',
|
|
294
|
+
name: HEADER_KEY,
|
|
295
|
+
items: source.header.map((e) => recordValueOf(e.tag, e.body)),
|
|
296
|
+
},
|
|
297
|
+
{ kind: 'map', name: RECORDS_KEY, entries: entriesFromTriples(source.records) },
|
|
298
|
+
{ kind: 'list', name: ORDER_KEY, keysOf: RECORDS_KEY },
|
|
299
|
+
];
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** The keys of an object whose value is not `undefined`, which JSON would
|
|
303
|
+
* drop and which `release` leaves behind. */
|
|
304
|
+
function ownDefinedKeys(object: Record<string, unknown>): string[] {
|
|
305
|
+
return Object.keys(object).filter((k) => object[k] !== undefined);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function isKeyPermutation(
|
|
309
|
+
items: readonly unknown[],
|
|
310
|
+
object: Record<string, unknown>,
|
|
311
|
+
): boolean {
|
|
312
|
+
const keys = ownDefinedKeys(object);
|
|
313
|
+
if (keys.length !== items.length) return false;
|
|
314
|
+
const seen = new Set<string>();
|
|
315
|
+
for (const item of items) {
|
|
316
|
+
if (typeof item !== 'string' || seen.has(item)) return false;
|
|
317
|
+
if (!Object.prototype.hasOwnProperty.call(object, item)) return false;
|
|
318
|
+
if (object[item] === undefined) return false;
|
|
319
|
+
seen.add(item);
|
|
320
|
+
}
|
|
321
|
+
return true;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// --- Decoding ---------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
/** Whether a byte run opens as frozen content THIS reader can read. Cheap
|
|
327
|
+
* enough to ask of any object, since it reads the first five bytes. */
|
|
328
|
+
export function isSealedStore(bytes: Uint8Array): boolean {
|
|
329
|
+
const version = frameVersion(bytes);
|
|
330
|
+
return version === VERSION_ARROW || version === VERSION_ARROW_STAGED;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** The frame version stamped on frozen content, or null for bytes that are
|
|
334
|
+
* not frozen content. */
|
|
335
|
+
export function frameVersion(bytes: Uint8Array): number | null {
|
|
336
|
+
if (bytes.byteLength < PROLOGUE_BYTES) return null;
|
|
337
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
338
|
+
return view.getUint32(0) === MAGIC ? view.getUint8(4) : null;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** Frozen content that could not be read. The message names the content
|
|
342
|
+
* rather than the field that failed, because every way this throws means the
|
|
343
|
+
* bytes are not content this reader can open. */
|
|
344
|
+
export class SealedStoreError extends Error {
|
|
345
|
+
constructor(
|
|
346
|
+
message?: string,
|
|
347
|
+
/** The error the decoder raised, where this wraps one. */
|
|
348
|
+
readonly underlying?: unknown,
|
|
349
|
+
) {
|
|
350
|
+
super(message);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** `out[key] = value`, as an own property even where the key is `__proto__`. */
|
|
355
|
+
function setOwn(out: Record<string, unknown>, key: string, value: unknown): void {
|
|
356
|
+
if (key === '__proto__') {
|
|
357
|
+
Object.defineProperty(out, key, { value, enumerable: true, writable: true, configurable: true });
|
|
358
|
+
} else {
|
|
359
|
+
out[key] = value;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** A map root held in memory: its keys in stored order and each value's JSON
|
|
364
|
+
* text, parsed afresh on every read. */
|
|
365
|
+
class SealedJsonMap implements SealedMapReader {
|
|
366
|
+
private _index: Map<string, number> | null = null;
|
|
367
|
+
|
|
368
|
+
constructor(
|
|
369
|
+
private readonly _keys: readonly string[],
|
|
370
|
+
private readonly _lines: readonly string[],
|
|
371
|
+
) {}
|
|
372
|
+
|
|
373
|
+
size(): number {
|
|
374
|
+
return this._keys.length;
|
|
375
|
+
}
|
|
376
|
+
has(key: string): boolean {
|
|
377
|
+
return this._at(key) >= 0;
|
|
378
|
+
}
|
|
379
|
+
get(key: string): unknown {
|
|
380
|
+
const at = this._at(key);
|
|
381
|
+
return at < 0 ? undefined : JSON.parse(this._lines[at]!);
|
|
382
|
+
}
|
|
383
|
+
keys(): string[] {
|
|
384
|
+
return [...this._keys];
|
|
385
|
+
}
|
|
386
|
+
forEach(visit: (key: string, value: unknown) => void): void {
|
|
387
|
+
for (let i = 0; i < this._keys.length; i += 1) visit(this._keys[i]!, JSON.parse(this._lines[i]!));
|
|
388
|
+
}
|
|
389
|
+
*entries(): IterableIterator<[string, unknown]> {
|
|
390
|
+
for (let i = 0; i < this._keys.length; i += 1) {
|
|
391
|
+
yield [this._keys[i]!, JSON.parse(this._lines[i]!)];
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
valueBytes(): number {
|
|
395
|
+
let bytes = 0;
|
|
396
|
+
for (const line of this._lines) bytes += line.length;
|
|
397
|
+
return bytes;
|
|
398
|
+
}
|
|
399
|
+
toJSON(): Record<string, unknown> {
|
|
400
|
+
const out: Record<string, unknown> = {};
|
|
401
|
+
this.forEach((key, value) => setOwn(out, key, value));
|
|
402
|
+
return out;
|
|
403
|
+
}
|
|
404
|
+
releaseDecoded(): void {}
|
|
405
|
+
|
|
406
|
+
private _at(key: string): number {
|
|
407
|
+
if (this._index === null) {
|
|
408
|
+
this._index = new Map();
|
|
409
|
+
for (let i = 0; i < this._keys.length; i += 1) this._index.set(this._keys[i]!, i);
|
|
410
|
+
}
|
|
411
|
+
return this._index.get(key) ?? -1;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/** A map's entries as `SealedJsonMap` holds them. A key repeated keeps its
|
|
416
|
+
* first entry. CONSUMES the values. */
|
|
417
|
+
function jsonMapOf(entries: MapEntries): SealedJsonMap {
|
|
418
|
+
const keys: string[] = [];
|
|
419
|
+
const lines: string[] = [];
|
|
420
|
+
const seen = new Set<string>();
|
|
421
|
+
for (let i = 0; i < entries.keys.length; i += 1) {
|
|
422
|
+
const key = entries.keys[i]!;
|
|
423
|
+
if (!seen.has(key)) {
|
|
424
|
+
seen.add(key);
|
|
425
|
+
keys.push(key);
|
|
426
|
+
lines.push(JSON.stringify(entries.value(i)) ?? 'null');
|
|
427
|
+
}
|
|
428
|
+
entries.release(i);
|
|
429
|
+
}
|
|
430
|
+
return new SealedJsonMap(keys, lines);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/** Content held in memory, from the roots it is stored as. */
|
|
434
|
+
function contentOfPlan(plan: readonly RootPlan[], builtBy: SealedStoreBuiltBy | null): SealedContent {
|
|
435
|
+
const roots = new Map<string, DecodedRoot>();
|
|
436
|
+
for (const root of plan) {
|
|
437
|
+
if (root.kind === 'map') roots.set(root.name, { kind: 'map', map: jsonMapOf(root.entries) });
|
|
438
|
+
else if (root.kind === 'list') {
|
|
439
|
+
roots.set(
|
|
440
|
+
root.name,
|
|
441
|
+
'keysOf' in root
|
|
442
|
+
? { kind: 'list', keysOf: root.keysOf }
|
|
443
|
+
: { kind: 'list', items: JSON.parse(listJson(root.items)) as unknown[] },
|
|
444
|
+
);
|
|
445
|
+
} else roots.set(root.name, { kind: 'text', value: root.text });
|
|
446
|
+
}
|
|
447
|
+
return SealedContent._of(new Uint8Array(0), roots, builtBy);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** A list's items as one JSON array, an `undefined` item read as null. */
|
|
451
|
+
function listJson(items: readonly unknown[]): string {
|
|
452
|
+
return '[' + items.map((item) => JSON.stringify(item === undefined ? null : item) ?? 'null').join(',') + ']';
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
type DecodedRoot =
|
|
456
|
+
| { kind: 'map'; map: SealedMapReader }
|
|
457
|
+
| { kind: 'list'; keysOf: string }
|
|
458
|
+
| { kind: 'list'; items: unknown[] }
|
|
459
|
+
| { kind: 'text'; value: string };
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Decoded frozen content: the frame read and every root located, or content
|
|
463
|
+
* held in memory (`fromJSON`, `fromRecords`) with the same roots, so a reader
|
|
464
|
+
* never asks which one it holds.
|
|
465
|
+
*/
|
|
466
|
+
export class SealedContent {
|
|
467
|
+
private constructor(
|
|
468
|
+
private readonly _bytes: Uint8Array,
|
|
469
|
+
private readonly _roots: Map<string, DecodedRoot>,
|
|
470
|
+
private readonly _builtBy: SealedStoreBuiltBy | null,
|
|
471
|
+
) {}
|
|
472
|
+
|
|
473
|
+
/** Read the frame. Throws `SealedStoreError` for bytes that are not frozen
|
|
474
|
+
* content, or whose frame does not fit inside them. */
|
|
475
|
+
static decode(bytes: Uint8Array): SealedContent {
|
|
476
|
+
const version = frameVersion(bytes);
|
|
477
|
+
if (version === null) throw new SealedStoreError('Not frozen content');
|
|
478
|
+
if (version === VERSION_ARROW || version === VERSION_ARROW_STAGED) {
|
|
479
|
+
return decodeArrowFrame(bytes, version);
|
|
480
|
+
}
|
|
481
|
+
throw new SealedStoreError(
|
|
482
|
+
`Frozen content version ${version}, which this build reads up to ` +
|
|
483
|
+
`${NEWEST_VERSION}`,
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* Content held in memory, as `decode` reads the same content stored.
|
|
489
|
+
* CONSUMES the map values, as `sealedTablesOfImport` does. Throws for a
|
|
490
|
+
* root that is not a map, a list or a string.
|
|
491
|
+
*/
|
|
492
|
+
static fromJSON(content: SealedContentJson, builtBy?: SealedStoreBuiltBy): SealedContent {
|
|
493
|
+
return contentOfPlan(planSealedContent(content), builtBy ?? null);
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/** A record store handed over as triples, held in memory. CONSUMES
|
|
497
|
+
* `source.records`. */
|
|
498
|
+
static fromRecords(source: SealedStoreSource): SealedContent {
|
|
499
|
+
return contentOfPlan(planSealedStore(source), source.builtBy ?? null);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/** @internal */
|
|
503
|
+
static fromPlan(plan: readonly RootPlan[], builtBy: SealedStoreBuiltBy | null): SealedContent {
|
|
504
|
+
return contentOfPlan(plan, builtBy);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** @internal */
|
|
508
|
+
static _of(
|
|
509
|
+
bytes: Uint8Array,
|
|
510
|
+
roots: Map<string, DecodedRoot>,
|
|
511
|
+
builtBy: SealedStoreBuiltBy | null,
|
|
512
|
+
): SealedContent {
|
|
513
|
+
return new SealedContent(bytes, roots, builtBy);
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** The content's own size, which is what it costs to hold and transfer. */
|
|
517
|
+
get byteLength(): number {
|
|
518
|
+
return this._bytes.byteLength;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/** The importer that produced the content, or null for content that names
|
|
522
|
+
* none. */
|
|
523
|
+
builtBy(): SealedStoreBuiltBy | null {
|
|
524
|
+
return this._builtBy;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
rootNames(): string[] {
|
|
528
|
+
return [...this._roots.keys()];
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
kindOf(name: string): SealedRootKind | undefined {
|
|
532
|
+
return this._roots.get(name)?.kind;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/** A map root as a lazy reader, or null when the content holds no such
|
|
536
|
+
* map. */
|
|
537
|
+
mapRoot(name: string): SealedMapReader | null {
|
|
538
|
+
const root = this._roots.get(name);
|
|
539
|
+
return root?.kind === 'map' ? root.map : null;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** The map a list root is the key walk of (an ordered map's order list), or
|
|
543
|
+
* null for a list stored with items of its own. */
|
|
544
|
+
keyWalkOf(name: string): string | null {
|
|
545
|
+
const root = this._roots.get(name);
|
|
546
|
+
return root?.kind === 'list' && 'keysOf' in root ? root.keysOf : null;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/** Every entry of a map root, in stored order, without building the
|
|
550
|
+
* object. */
|
|
551
|
+
forEachMapEntry(name: string, visit: (key: string, value: unknown) => void): void {
|
|
552
|
+
this.mapRoot(name)?.forEach(visit);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/** A map root as an object, or null when the content holds no such map. */
|
|
556
|
+
map(name: string): Record<string, unknown> | null {
|
|
557
|
+
return this.mapRoot(name)?.toJSON() ?? null;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** A list root's items, or null when the content holds no such list. */
|
|
561
|
+
list(name: string): unknown[] | null {
|
|
562
|
+
const root = this._roots.get(name);
|
|
563
|
+
if (root?.kind !== 'list') return null;
|
|
564
|
+
if ('items' in root) return [...root.items];
|
|
565
|
+
return [...(this.mapRoot(root.keysOf)?.keys() ?? [])];
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/** A text root, or null when the content holds no such text. */
|
|
569
|
+
text(name: string): string | null {
|
|
570
|
+
const root = this._roots.get(name);
|
|
571
|
+
return root?.kind === 'text' ? root.value : null;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/** The whole content as the JSON it was encoded from. */
|
|
575
|
+
toJSON(): SealedContentJson {
|
|
576
|
+
const out: SealedContentJson = {};
|
|
577
|
+
for (const name of this._roots.keys()) {
|
|
578
|
+
const kind = this.kindOf(name);
|
|
579
|
+
if (kind === 'map') setOwn(out, name, this.map(name)!);
|
|
580
|
+
else if (kind === 'list') setOwn(out, name, this.list(name)!);
|
|
581
|
+
else if (kind === 'text') setOwn(out, name, this.text(name)!);
|
|
582
|
+
}
|
|
583
|
+
return out;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// --- Frame 4 ---------------------------------------------------------------
|
|
588
|
+
//
|
|
589
|
+
// The prologue, then one ZSTD frame. Inside it, named parts
|
|
590
|
+
// (`sealed-arrow.ts` `unpackParts`): first the manifest, `[builtBy, roots]`,
|
|
591
|
+
// where a root is `['map', name]`, `['list', name, items]`,
|
|
592
|
+
// `['list', name, null, keysOf]` or `['text', name, text]`; then one Arrow
|
|
593
|
+
// IPC file per table, each naming its map in its metadata. Decoding needs the
|
|
594
|
+
// Arrow reader and a ZSTD decoder, which load on the first frame-4 open
|
|
595
|
+
// (`loadSealedCodec`) so a page that never opens one never fetches them.
|
|
596
|
+
|
|
597
|
+
/** What a frame-4 decode calls. Each decompressor refuses output past `max`. */
|
|
598
|
+
export interface SealedCodec {
|
|
599
|
+
arrow: ArrowReader;
|
|
600
|
+
unzstd(bytes: Uint8Array, max: number): Uint8Array;
|
|
601
|
+
gunzip(bytes: Uint8Array, max: number): Uint8Array;
|
|
602
|
+
reader: typeof import('./sealed-arrow.js');
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* The most a frame-4 or staging-frame body expands to. The encoder refuses to
|
|
607
|
+
* write tables past it and the decoder refuses a body that would expand past
|
|
608
|
+
* it, so every stored line is one every reader decodes, and a body an
|
|
609
|
+
* uploader shaped to expand further is refused before it is decompressed.
|
|
610
|
+
* The densest import measured (2.85M IFC geometry records) expands to 68 MB,
|
|
611
|
+
* and a 128 MiB CSV of single-digit cells to about 285 MB.
|
|
612
|
+
*/
|
|
613
|
+
export const SEALED_INNER_MAX_BYTES = 256 * 1024 * 1024;
|
|
614
|
+
|
|
615
|
+
/** Content whose tables would expand past `SEALED_INNER_MAX_BYTES`. */
|
|
616
|
+
export class SealedContentTooLargeError extends SealedStoreError {
|
|
617
|
+
constructor() {
|
|
618
|
+
super(`Sealed content expands past ${SEALED_INNER_MAX_BYTES} bytes`);
|
|
619
|
+
this.name = 'SealedContentTooLargeError';
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
const ZSTD_MAGIC = 0xfd2fb528;
|
|
624
|
+
const ZSTD_BLOCK_MAX = 128 * 1024;
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* The most a ZSTD stream can write or allocate, read from its frame and block
|
|
628
|
+
* headers without decompressing it: per frame its window, and its declared
|
|
629
|
+
* content size, which a decoder holds the frame's output to. A frame that
|
|
630
|
+
* declares none is bounded by its blocks: each raw or RLE block's stated size
|
|
631
|
+
* plus the format's 128 KiB ceiling per compressed block, which overstates a
|
|
632
|
+
* level-19 frame (its blocks average a few KB). Both encoders here declare the
|
|
633
|
+
* size. Throws on a stream that is not ZSTD frames end to end.
|
|
634
|
+
*/
|
|
635
|
+
export function zstdOutputBound(bytes: Uint8Array): number {
|
|
636
|
+
let at = 0;
|
|
637
|
+
let output = 0;
|
|
638
|
+
let largest = 0;
|
|
639
|
+
const need = (n: number): void => {
|
|
640
|
+
if (at + n > bytes.length) throw new RangeError('zstd stream is truncated');
|
|
641
|
+
};
|
|
642
|
+
const le = (n: number): number => {
|
|
643
|
+
need(n);
|
|
644
|
+
let v = 0;
|
|
645
|
+
for (let i = n - 1; i >= 0; i -= 1) v = v * 256 + bytes[at + i]!;
|
|
646
|
+
at += n;
|
|
647
|
+
return v;
|
|
648
|
+
};
|
|
649
|
+
while (at < bytes.length) {
|
|
650
|
+
const magic = le(4);
|
|
651
|
+
if ((magic & 0xfffffff0) >>> 0 === 0x184d2a50) {
|
|
652
|
+
const size = le(4);
|
|
653
|
+
need(size);
|
|
654
|
+
at += size;
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
657
|
+
if (magic !== ZSTD_MAGIC) throw new RangeError('not a zstd frame');
|
|
658
|
+
const fhd = le(1);
|
|
659
|
+
if (fhd & 8) throw new RangeError('zstd frame header sets a reserved bit');
|
|
660
|
+
const fcsFlag = fhd >> 6;
|
|
661
|
+
const single = (fhd >> 5) & 1;
|
|
662
|
+
const checksum = (fhd >> 2) & 1;
|
|
663
|
+
let window = 0;
|
|
664
|
+
if (single === 0) {
|
|
665
|
+
const wd = le(1);
|
|
666
|
+
const base = 2 ** (10 + (wd >> 3));
|
|
667
|
+
window = base + (base / 8) * (wd & 7);
|
|
668
|
+
}
|
|
669
|
+
le([0, 1, 2, 4][fhd & 3]!);
|
|
670
|
+
const fcsBytes = fcsFlag === 0 ? single : 2 ** fcsFlag;
|
|
671
|
+
const fcs = fcsBytes === 0 ? 0 : le(fcsBytes) + (fcsBytes === 2 ? 256 : 0);
|
|
672
|
+
largest = Math.max(largest, window, fcs);
|
|
673
|
+
let blocks = 0;
|
|
674
|
+
for (;;) {
|
|
675
|
+
const header = le(3);
|
|
676
|
+
const type = (header >> 1) & 3;
|
|
677
|
+
const size = header >>> 3;
|
|
678
|
+
if (type === 0) {
|
|
679
|
+
need(size);
|
|
680
|
+
at += size;
|
|
681
|
+
blocks += size;
|
|
682
|
+
} else if (type === 1) {
|
|
683
|
+
need(1);
|
|
684
|
+
at += 1;
|
|
685
|
+
blocks += size;
|
|
686
|
+
} else if (type === 2) {
|
|
687
|
+
need(size);
|
|
688
|
+
at += size;
|
|
689
|
+
blocks += ZSTD_BLOCK_MAX;
|
|
690
|
+
} else {
|
|
691
|
+
throw new RangeError('zstd block type is reserved');
|
|
692
|
+
}
|
|
693
|
+
if (header & 1) break;
|
|
694
|
+
}
|
|
695
|
+
output += fcsBytes === 0 ? blocks : fcs;
|
|
696
|
+
if (checksum) le(4);
|
|
697
|
+
}
|
|
698
|
+
return Math.max(output, largest);
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
/** A ZSTD body decompressed, refused before it is decompressed where its
|
|
702
|
+
* headers allow more than `max`. */
|
|
703
|
+
function unzstdCapped(unzstd: (bytes: Uint8Array, max: number) => Uint8Array, body: Uint8Array, max: number): Uint8Array {
|
|
704
|
+
let bound: number;
|
|
705
|
+
try {
|
|
706
|
+
bound = zstdOutputBound(body);
|
|
707
|
+
} catch (cause) {
|
|
708
|
+
throw new SealedStoreError('Frozen content is unreadable', cause);
|
|
709
|
+
}
|
|
710
|
+
if (bound > max) throw new SealedContentTooLargeError();
|
|
711
|
+
return unzstd(body, max);
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
/** A gzip body inflated a slice at a time, refused once it passes `max`.
|
|
715
|
+
* A slice inflates to at most 1032 times its own size. */
|
|
716
|
+
function gunzipStreamed(body: Uint8Array, max: number): Uint8Array {
|
|
717
|
+
const chunks: Uint8Array[] = [];
|
|
718
|
+
let total = 0;
|
|
719
|
+
const inflate = new Gunzip((chunk) => {
|
|
720
|
+
total += chunk.byteLength;
|
|
721
|
+
if (total > max) throw new SealedContentTooLargeError();
|
|
722
|
+
chunks.push(chunk);
|
|
723
|
+
});
|
|
724
|
+
const step = 16 * 1024;
|
|
725
|
+
for (let at = 0; at < body.length; at += step) {
|
|
726
|
+
inflate.push(body.subarray(at, at + step), at + step >= body.length);
|
|
727
|
+
}
|
|
728
|
+
const out = new Uint8Array(total);
|
|
729
|
+
let at = 0;
|
|
730
|
+
for (const chunk of chunks) {
|
|
731
|
+
out.set(chunk, at);
|
|
732
|
+
at += chunk.byteLength;
|
|
733
|
+
}
|
|
734
|
+
return out;
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
let sealedCodec: SealedCodec | null = null;
|
|
738
|
+
let sealedCodecLoading: Promise<void> | null = null;
|
|
739
|
+
|
|
740
|
+
/** Whether decoding these bytes needs `loadSealedCodec` first. */
|
|
741
|
+
export function needsSealedCodec(bytes: Uint8Array): boolean {
|
|
742
|
+
const version = frameVersion(bytes);
|
|
743
|
+
return (version === VERSION_ARROW || version === VERSION_ARROW_STAGED) && sealedCodec === null;
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
interface NativeZstd {
|
|
747
|
+
zstdDecompressSync?(bytes: Uint8Array, options: { maxOutputLength: number }): Uint8Array;
|
|
748
|
+
gunzipSync?(bytes: Uint8Array, options: { maxOutputLength: number }): Uint8Array;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/** Node's error for output past `maxOutputLength`. */
|
|
752
|
+
function overMax(err: unknown): boolean {
|
|
753
|
+
return (err as { code?: unknown } | null)?.code === 'ERR_BUFFER_TOO_LARGE';
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/** Load what a frame-4 decode needs. Under a Node that carries zstd, the
|
|
757
|
+
* built-in decompresses; anywhere else, `fzstd`. */
|
|
758
|
+
export function loadSealedCodec(): Promise<void> {
|
|
759
|
+
if (sealedCodec !== null) return Promise.resolve();
|
|
760
|
+
sealedCodecLoading ??= (async () => {
|
|
761
|
+
let native: NativeZstd | undefined;
|
|
762
|
+
try {
|
|
763
|
+
const proc = (
|
|
764
|
+
globalThis as { process?: { getBuiltinModule?: (id: string) => unknown } }
|
|
765
|
+
).process;
|
|
766
|
+
native = proc?.getBuiltinModule?.('node:zlib') as NativeZstd | undefined;
|
|
767
|
+
} catch {
|
|
768
|
+
native = undefined;
|
|
769
|
+
}
|
|
770
|
+
const nativeUnzstd = native?.zstdDecompressSync?.bind(native);
|
|
771
|
+
const nativeGunzip = native?.gunzipSync?.bind(native);
|
|
772
|
+
const [arrow, unzstd, reader] = await Promise.all([
|
|
773
|
+
import('apache-arrow'),
|
|
774
|
+
nativeUnzstd !== undefined
|
|
775
|
+
? Promise.resolve((bytes: Uint8Array, max: number) => {
|
|
776
|
+
try {
|
|
777
|
+
return new Uint8Array(nativeUnzstd(bytes, { maxOutputLength: max }));
|
|
778
|
+
} catch (err) {
|
|
779
|
+
if (overMax(err)) throw new SealedContentTooLargeError();
|
|
780
|
+
throw err;
|
|
781
|
+
}
|
|
782
|
+
})
|
|
783
|
+
: // The frame headers bound what `fzstd` allocates and writes
|
|
784
|
+
// (`unzstdCapped` checks them first), so it runs uncapped.
|
|
785
|
+
import('fzstd').then((m) => (bytes: Uint8Array) => m.decompress(bytes)),
|
|
786
|
+
import('./sealed-arrow.js'),
|
|
787
|
+
]);
|
|
788
|
+
const gunzip =
|
|
789
|
+
nativeGunzip !== undefined
|
|
790
|
+
? (bytes: Uint8Array, max: number) => {
|
|
791
|
+
try {
|
|
792
|
+
return new Uint8Array(nativeGunzip(bytes, { maxOutputLength: max }));
|
|
793
|
+
} catch (err) {
|
|
794
|
+
if (overMax(err)) throw new SealedContentTooLargeError();
|
|
795
|
+
throw err;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
: gunzipStreamed;
|
|
799
|
+
sealedCodec = { arrow: arrow as unknown as ArrowReader, unzstd, gunzip, reader };
|
|
800
|
+
})().catch((err: unknown) => {
|
|
801
|
+
sealedCodecLoading = null;
|
|
802
|
+
throw err;
|
|
803
|
+
});
|
|
804
|
+
return sealedCodecLoading;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
/** Decode frozen bytes, loading the frame-4 codec first when they need it.
|
|
808
|
+
* A frame-4 decode awaits `pause` before each table it parses, so a large
|
|
809
|
+
* decode leaves the page responsive. */
|
|
810
|
+
export async function decodeSealedContent(
|
|
811
|
+
bytes: Uint8Array,
|
|
812
|
+
pause: () => Promise<void> = slicedYield(),
|
|
813
|
+
): Promise<SealedContent> {
|
|
814
|
+
if (needsSealedCodec(bytes)) await loadSealedCodec();
|
|
815
|
+
const version = frameVersion(bytes);
|
|
816
|
+
if (version !== VERSION_ARROW && version !== VERSION_ARROW_STAGED) return SealedContent.decode(bytes);
|
|
817
|
+
const steps = arrowFrameSteps(bytes, version);
|
|
818
|
+
for (let step = steps.next(); ; step = steps.next()) {
|
|
819
|
+
if (step.done === true) return step.value;
|
|
820
|
+
await pause();
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/** Work between yields. A yield costs a few milliseconds on a phone, and a
|
|
825
|
+
* frame holds hundreds of tables, so yielding before every one adds up. */
|
|
826
|
+
const YIELD_SLICE_MS = 25;
|
|
827
|
+
|
|
828
|
+
/** A pause that hands the event loop back once `sliceMs` of work has run
|
|
829
|
+
* since the last time it did. */
|
|
830
|
+
export function slicedYield(sliceMs = YIELD_SLICE_MS): () => Promise<void> {
|
|
831
|
+
let since = performance.now();
|
|
832
|
+
return async () => {
|
|
833
|
+
if (performance.now() - since < sliceMs) return;
|
|
834
|
+
await yieldToEventLoop();
|
|
835
|
+
since = performance.now();
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
type Scheduler = { yield?: () => Promise<void> };
|
|
840
|
+
|
|
841
|
+
/** Hands the event loop back for one turn: `scheduler.yield` where the
|
|
842
|
+
* platform has it, else a message, which no timer clamp delays. */
|
|
843
|
+
export function yieldToEventLoop(): Promise<void> {
|
|
844
|
+
const scheduler = (globalThis as { scheduler?: Scheduler }).scheduler;
|
|
845
|
+
if (typeof scheduler?.yield === 'function') return scheduler.yield();
|
|
846
|
+
if (typeof MessageChannel === 'function') {
|
|
847
|
+
return new Promise((resolve) => {
|
|
848
|
+
const channel = new MessageChannel();
|
|
849
|
+
channel.port1.onmessage = () => {
|
|
850
|
+
channel.port1.close();
|
|
851
|
+
resolve();
|
|
852
|
+
};
|
|
853
|
+
channel.port2.postMessage(null);
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/** A frame-4 object around its ZSTD-compressed parts, or with
|
|
860
|
+
* `VERSION_ARROW_STAGED`, the staging frame around its gzipped parts. */
|
|
861
|
+
export function arrowFrameBytes(compressed: Uint8Array, version = VERSION_ARROW): Uint8Array {
|
|
862
|
+
const out = new Uint8Array(PROLOGUE_BYTES + compressed.byteLength);
|
|
863
|
+
const view = new DataView(out.buffer);
|
|
864
|
+
view.setUint32(0, MAGIC);
|
|
865
|
+
view.setUint8(4, version);
|
|
866
|
+
out.set(compressed, PROLOGUE_BYTES);
|
|
867
|
+
return out;
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/** A frame-4 or staging-frame object's compressed parts. */
|
|
871
|
+
export function arrowFrameBody(bytes: Uint8Array): Uint8Array {
|
|
872
|
+
return bytes.subarray(PROLOGUE_BYTES);
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
/**
|
|
876
|
+
* A frame-4 or staging-frame object's parts, decompressed. Throws
|
|
877
|
+
* `SealedContentTooLargeError` for a body that expands past
|
|
878
|
+
* `SEALED_INNER_MAX_BYTES`, and needs `loadSealedCodec` first.
|
|
879
|
+
*/
|
|
880
|
+
export function arrowFrameInner(bytes: Uint8Array): Uint8Array {
|
|
881
|
+
const version = frameVersion(bytes);
|
|
882
|
+
if (sealedCodec === null) {
|
|
883
|
+
throw new SealedStoreError(`Frozen content version ${version} needs its codec loaded first`);
|
|
884
|
+
}
|
|
885
|
+
const body = bytes.subarray(PROLOGUE_BYTES);
|
|
886
|
+
try {
|
|
887
|
+
return version === VERSION_ARROW_STAGED
|
|
888
|
+
? sealedCodec.gunzip(body, SEALED_INNER_MAX_BYTES)
|
|
889
|
+
: unzstdCapped(sealedCodec.unzstd, body, SEALED_INNER_MAX_BYTES);
|
|
890
|
+
} catch (err) {
|
|
891
|
+
if (err instanceof SealedStoreError) throw err;
|
|
892
|
+
throw new SealedStoreError('Frozen content is unreadable', err);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
/** The manifest part's name; table parts are named by their position. */
|
|
897
|
+
export const ARROW_MANIFEST_PART = 'manifest';
|
|
898
|
+
|
|
899
|
+
export type ArrowManifestRoot =
|
|
900
|
+
| ['map', string]
|
|
901
|
+
| ['list', string, unknown[]]
|
|
902
|
+
| ['list', string, null, string]
|
|
903
|
+
| ['text', string, string];
|
|
904
|
+
|
|
905
|
+
export type ArrowManifest = [[string, string] | null, ArrowManifestRoot[]];
|
|
906
|
+
|
|
907
|
+
function decodeArrowFrame(bytes: Uint8Array, version: number): SealedContent {
|
|
908
|
+
const steps = arrowFrameSteps(bytes, version);
|
|
909
|
+
for (let step = steps.next(); ; step = steps.next()) if (step.done === true) return step.value;
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
/** A frame-4 decode, stepping before each table it parses. */
|
|
913
|
+
function* arrowFrameSteps(bytes: Uint8Array, version: number): Generator<void, SealedContent> {
|
|
914
|
+
if (sealedCodec === null) {
|
|
915
|
+
throw new SealedStoreError(`Frozen content version ${version} needs its codec loaded first`);
|
|
916
|
+
}
|
|
917
|
+
const codec = sealedCodec;
|
|
918
|
+
const inner = arrowFrameInner(bytes);
|
|
919
|
+
let parts: [string, Uint8Array][];
|
|
920
|
+
try {
|
|
921
|
+
parts = codec.reader.unpackParts(inner);
|
|
922
|
+
} catch (cause) {
|
|
923
|
+
throw new SealedStoreError('Frozen content is unreadable', cause);
|
|
924
|
+
}
|
|
925
|
+
const [manifestName, manifestBytes] = parts[0] ?? [];
|
|
926
|
+
if (manifestName !== ARROW_MANIFEST_PART || manifestBytes === undefined) {
|
|
927
|
+
throw new SealedStoreError('Frozen content descriptor is unreadable');
|
|
928
|
+
}
|
|
929
|
+
let manifest: ArrowManifest;
|
|
930
|
+
try {
|
|
931
|
+
manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as ArrowManifest;
|
|
932
|
+
} catch (cause) {
|
|
933
|
+
throw new SealedStoreError('Frozen content descriptor is unreadable', cause);
|
|
934
|
+
}
|
|
935
|
+
if (!Array.isArray(manifest) || !Array.isArray(manifest[1])) {
|
|
936
|
+
throw new SealedStoreError('Frozen content descriptor is unreadable');
|
|
937
|
+
}
|
|
938
|
+
const tablesOf = new Map<string, StoredArrowTable[]>();
|
|
939
|
+
for (const [, ipc] of parts.slice(1)) {
|
|
940
|
+
yield;
|
|
941
|
+
let table: ReturnType<ArrowReader['tableFromIPC']>;
|
|
942
|
+
let meta: TableMeta;
|
|
943
|
+
try {
|
|
944
|
+
codec.reader.checkIpcFile(ipc);
|
|
945
|
+
table = codec.arrow.tableFromIPC(ipc);
|
|
946
|
+
meta = JSON.parse(table.schema.metadata.get(codec.reader.TABLE_META_KEY) ?? '') as TableMeta;
|
|
947
|
+
for (const batch of table.batches) codec.reader.checkTableData(batch.data);
|
|
948
|
+
} catch (cause) {
|
|
949
|
+
throw new SealedStoreError('Frozen table is unreadable', cause);
|
|
950
|
+
}
|
|
951
|
+
let list = tablesOf.get(meta.root);
|
|
952
|
+
if (list === undefined) tablesOf.set(meta.root, (list = []));
|
|
953
|
+
for (const batch of table.batches) list.push({ meta, data: batch.data });
|
|
954
|
+
}
|
|
955
|
+
const by = manifest[0];
|
|
956
|
+
const builtBy =
|
|
957
|
+
Array.isArray(by) && typeof by[0] === 'string' && typeof by[1] === 'string'
|
|
958
|
+
? { id: by[0], version: by[1] }
|
|
959
|
+
: null;
|
|
960
|
+
const roots = new Map<string, DecodedRoot>();
|
|
961
|
+
for (const d of manifest[1]) {
|
|
962
|
+
if (!Array.isArray(d) || typeof d[1] !== 'string') {
|
|
963
|
+
throw new SealedStoreError('Frozen content descriptor is unreadable');
|
|
964
|
+
}
|
|
965
|
+
const name = d[1];
|
|
966
|
+
if (d[0] === 'map') {
|
|
967
|
+
roots.set(name, { kind: 'map', map: new SealedArrowMap(codec.reader, tablesOf.get(name) ?? []) });
|
|
968
|
+
} else if (d[0] === 'list') {
|
|
969
|
+
if (typeof d[3] === 'string') roots.set(name, { kind: 'list', keysOf: d[3] });
|
|
970
|
+
else if (Array.isArray(d[2])) roots.set(name, { kind: 'list', items: d[2] });
|
|
971
|
+
else throw new SealedStoreError('Frozen content descriptor is unreadable');
|
|
972
|
+
} else if (d[0] === 'text' && typeof d[2] === 'string') {
|
|
973
|
+
roots.set(name, { kind: 'text', value: d[2] });
|
|
974
|
+
} else {
|
|
975
|
+
throw new SealedStoreError(`Frozen content root "${name}" has an unknown kind`);
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
return SealedContent._of(bytes, roots, builtBy);
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
interface StoredArrowTable {
|
|
982
|
+
meta: TableMeta;
|
|
983
|
+
data: ArrowData;
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/** A row's value, with any failure to build it read as unreadable content. */
|
|
987
|
+
function readValue(t: ReadyTable, row: number): unknown {
|
|
988
|
+
try {
|
|
989
|
+
return t.value(row);
|
|
990
|
+
} catch (cause) {
|
|
991
|
+
throw new SealedStoreError('Frozen table is unreadable', cause);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
/** A row's key, with any failure to read it as a string read as unreadable
|
|
996
|
+
* content. */
|
|
997
|
+
function readKey(t: ReadyTable, row: number): string {
|
|
998
|
+
let key: unknown;
|
|
999
|
+
try {
|
|
1000
|
+
key = t.key(row);
|
|
1001
|
+
} catch (cause) {
|
|
1002
|
+
throw new SealedStoreError('Frozen table is unreadable', cause);
|
|
1003
|
+
}
|
|
1004
|
+
if (typeof key !== 'string') throw new SealedStoreError('Frozen table is unreadable');
|
|
1005
|
+
return key;
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
/**
|
|
1009
|
+
* Records a hash per key as a walk meets them, in position order, and throws
|
|
1010
|
+
* from `finish` if two keys were the same: the hashes are sorted, and only
|
|
1011
|
+
* where two agree are the keys themselves rebuilt and compared. Kept to a
|
|
1012
|
+
* typed array, so the check costs one sort of four bytes a key.
|
|
1013
|
+
*/
|
|
1014
|
+
class KeyCheck {
|
|
1015
|
+
private readonly _hashes: Uint32Array;
|
|
1016
|
+
|
|
1017
|
+
constructor(
|
|
1018
|
+
count: number,
|
|
1019
|
+
private readonly _keyAt: (at: number) => string,
|
|
1020
|
+
) {
|
|
1021
|
+
this._hashes = new Uint32Array(count);
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
add(key: string, at: number): void {
|
|
1025
|
+
this._hashes[at] = keyHash(key);
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
finish(): void {
|
|
1029
|
+
const sorted = this._hashes.slice().sort();
|
|
1030
|
+
for (let i = 1; i < sorted.length; i += 1) {
|
|
1031
|
+
if (sorted[i] !== sorted[i - 1]) continue;
|
|
1032
|
+
const h = sorted[i]!;
|
|
1033
|
+
const seen = new Set<string>();
|
|
1034
|
+
for (let at = 0; at < this._hashes.length; at += 1) {
|
|
1035
|
+
if (this._hashes[at] !== h) continue;
|
|
1036
|
+
const key = this._keyAt(at);
|
|
1037
|
+
if (seen.has(key)) throw new SealedStoreError('Frozen table is unreadable');
|
|
1038
|
+
seen.add(key);
|
|
1039
|
+
}
|
|
1040
|
+
while (i + 1 < sorted.length && sorted[i + 1] === h) i += 1;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
function keyHash(key: string): number {
|
|
1046
|
+
let h = 0x811c9dc5;
|
|
1047
|
+
for (let i = 0; i < key.length; i += 1) h = Math.imul(h ^ key.charCodeAt(i), 0x01000193);
|
|
1048
|
+
return h >>> 0;
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/** Keyed reads a frame-4 map answers by scanning its key columns before it
|
|
1052
|
+
* builds an index of every key. A scan touches only typed arrays and costs
|
|
1053
|
+
* milliseconds at a million records; the index costs a string and a map
|
|
1054
|
+
* entry per record, which a few keyed reads never repay. */
|
|
1055
|
+
const SCANNED_LOOKUPS = 64;
|
|
1056
|
+
|
|
1057
|
+
/** A version 4 map root. Its tables are prepared on first use; a value is
|
|
1058
|
+
* built when it is read, so a keyed read builds the rows it asks for. Until a
|
|
1059
|
+
* walk prepares every table, a keyed read prepares only the table it hits,
|
|
1060
|
+
* decoding its dictionaries' entries as they are read. */
|
|
1061
|
+
class SealedArrowMap implements SealedMapReader {
|
|
1062
|
+
private readonly _count: number;
|
|
1063
|
+
private readonly _cx: ReadContext;
|
|
1064
|
+
private readonly _ready: (ReadyTable | undefined)[];
|
|
1065
|
+
private _lazyCx: ReadContext | null = null;
|
|
1066
|
+
private _lazyReady: (ReadyTable | undefined)[] | null = null;
|
|
1067
|
+
private _tables: ReadyTable[] | null = null;
|
|
1068
|
+
private _tableAt: Int32Array | null = null;
|
|
1069
|
+
private _rowAt: Int32Array | null = null;
|
|
1070
|
+
private _keys: string[] | null = null;
|
|
1071
|
+
private _index: Map<string, number> | null = null;
|
|
1072
|
+
private _lookups = 0;
|
|
1073
|
+
/** Whether a whole walk has found every key distinct. */
|
|
1074
|
+
private _unique = false;
|
|
1075
|
+
|
|
1076
|
+
constructor(
|
|
1077
|
+
private readonly _reader: SealedCodec['reader'],
|
|
1078
|
+
private readonly _stored: readonly StoredArrowTable[],
|
|
1079
|
+
) {
|
|
1080
|
+
let count = 0;
|
|
1081
|
+
for (const t of _stored) count += t.data.length;
|
|
1082
|
+
this._count = count;
|
|
1083
|
+
this._cx = new _reader.ReadContext();
|
|
1084
|
+
this._ready = Array.from({ length: _stored.length });
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
size(): number {
|
|
1088
|
+
return this._count;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
has(key: string): boolean {
|
|
1092
|
+
return this._find(key) !== null;
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
get(key: string): unknown {
|
|
1096
|
+
const found = this._find(key);
|
|
1097
|
+
return found === null ? undefined : readValue(this._keyedTable(found[0]), found[1]);
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
keys(): string[] {
|
|
1101
|
+
if (this._keys === null) {
|
|
1102
|
+
this._order();
|
|
1103
|
+
const check = this._unique ? null : new KeyCheck(this._count, (at) => this._keyAt(at));
|
|
1104
|
+
const keys: string[] = Array.from({ length: this._count });
|
|
1105
|
+
for (let at = 0; at < this._count; at += 1) {
|
|
1106
|
+
const key = this._keyAt(at);
|
|
1107
|
+
check?.add(key, at);
|
|
1108
|
+
keys[at] = key;
|
|
1109
|
+
}
|
|
1110
|
+
check?.finish();
|
|
1111
|
+
this._unique = true;
|
|
1112
|
+
this._keys = keys;
|
|
1113
|
+
}
|
|
1114
|
+
return this._keys;
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
forEach(visit: (key: string, value: unknown) => void): void {
|
|
1118
|
+
for (const [key, value] of this.entries()) visit(key, value);
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
*entries(): IterableIterator<[string, unknown]> {
|
|
1122
|
+
this._order();
|
|
1123
|
+
const tables = this._tables!;
|
|
1124
|
+
const tableAt = this._tableAt!;
|
|
1125
|
+
const rowAt = this._rowAt!;
|
|
1126
|
+
// A repeated key is reported once the walk has met both.
|
|
1127
|
+
const check = this._unique ? null : new KeyCheck(this._count, (at) => this._keyAt(at));
|
|
1128
|
+
for (let at = 0; at < this._count; at += 1) {
|
|
1129
|
+
const t = tables[tableAt[at]!]!;
|
|
1130
|
+
const row = rowAt[at]!;
|
|
1131
|
+
const key = readKey(t, row);
|
|
1132
|
+
check?.add(key, at);
|
|
1133
|
+
yield [key, readValue(t, row)];
|
|
1134
|
+
}
|
|
1135
|
+
check?.finish();
|
|
1136
|
+
this._unique = true;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
valueBytes(): number {
|
|
1140
|
+
let bytes = 0;
|
|
1141
|
+
for (const [, value] of this.entries()) bytes += (JSON.stringify(value) ?? 'null').length;
|
|
1142
|
+
return bytes;
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
toJSON(): Record<string, unknown> {
|
|
1146
|
+
const out: Record<string, unknown> = {};
|
|
1147
|
+
for (const [key, value] of this.entries()) setOwn(out, key, value);
|
|
1148
|
+
return out;
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
releaseDecoded(): void {
|
|
1152
|
+
this._keys = null;
|
|
1153
|
+
this._index = null;
|
|
1154
|
+
this._lookups = 0;
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
/** The table and row holding `key`. */
|
|
1158
|
+
private _find(key: string): [number, number] | null {
|
|
1159
|
+
if (this._index === null && this._lookups >= SCANNED_LOOKUPS) {
|
|
1160
|
+
const keys = this.keys();
|
|
1161
|
+
const index = new Map<string, number>();
|
|
1162
|
+
for (let at = 0; at < keys.length; at += 1) index.set(keys[at]!, at);
|
|
1163
|
+
this._index = index;
|
|
1164
|
+
}
|
|
1165
|
+
if (this._index !== null) {
|
|
1166
|
+
const at = this._index.get(key);
|
|
1167
|
+
return at === undefined ? null : [this._tableAt![at]!, this._rowAt![at]!];
|
|
1168
|
+
}
|
|
1169
|
+
this._lookups += 1;
|
|
1170
|
+
const cx = this._tables === null ? this._lazy() : this._cx;
|
|
1171
|
+
const matches: [number, number][] = [];
|
|
1172
|
+
for (let ti = 0; ti < this._stored.length; ti += 1) {
|
|
1173
|
+
const t = this._stored[ti]!;
|
|
1174
|
+
let rows: number[];
|
|
1175
|
+
try {
|
|
1176
|
+
rows = this._reader.rowsWithKey(t.meta, t.data.children[0]!, key, cx);
|
|
1177
|
+
} catch (cause) {
|
|
1178
|
+
throw new SealedStoreError('Frozen table is unreadable', cause);
|
|
1179
|
+
}
|
|
1180
|
+
for (const row of rows) matches.push([ti, row]);
|
|
1181
|
+
}
|
|
1182
|
+
// The encoder writes each key once, so tables repeating one are not its.
|
|
1183
|
+
if (matches.length > 1) throw new SealedStoreError('Frozen table is unreadable');
|
|
1184
|
+
return matches[0] ?? null;
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
private _keyAt(at: number): string {
|
|
1188
|
+
return readKey(this._tables![this._tableAt![at]!]!, this._rowAt![at]!);
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
private _lazy(): ReadContext {
|
|
1192
|
+
return (this._lazyCx ??= new this._reader.ReadContext(true));
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
/** A table for a keyed read: the walk's where it is prepared, else one
|
|
1196
|
+
* prepared lazily on its own. */
|
|
1197
|
+
private _keyedTable(ti: number): ReadyTable {
|
|
1198
|
+
const ready = this._ready[ti];
|
|
1199
|
+
if (ready !== undefined) return ready;
|
|
1200
|
+
const lazy = (this._lazyReady ??= Array.from({ length: this._stored.length }));
|
|
1201
|
+
let t = lazy[ti];
|
|
1202
|
+
if (t === undefined) {
|
|
1203
|
+
const stored = this._stored[ti]!;
|
|
1204
|
+
try {
|
|
1205
|
+
t = this._reader.readyTable(stored.meta, stored.data, this._lazy());
|
|
1206
|
+
} catch (cause) {
|
|
1207
|
+
throw new SealedStoreError('Frozen table is unreadable', cause);
|
|
1208
|
+
}
|
|
1209
|
+
lazy[ti] = t;
|
|
1210
|
+
}
|
|
1211
|
+
return t;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
private _table(ti: number): ReadyTable {
|
|
1215
|
+
let t = this._ready[ti];
|
|
1216
|
+
if (t === undefined) {
|
|
1217
|
+
const stored = this._stored[ti]!;
|
|
1218
|
+
try {
|
|
1219
|
+
t = this._reader.readyTable(stored.meta, stored.data, this._cx);
|
|
1220
|
+
} catch (cause) {
|
|
1221
|
+
throw new SealedStoreError('Frozen table is unreadable', cause);
|
|
1222
|
+
}
|
|
1223
|
+
this._ready[ti] = t;
|
|
1224
|
+
}
|
|
1225
|
+
return t;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
/** Each position's table and row. Every position in the map is held by
|
|
1229
|
+
* exactly one row, or the tables do not describe the map. */
|
|
1230
|
+
private _order(): void {
|
|
1231
|
+
if (this._tables !== null) return;
|
|
1232
|
+
const tables = this._stored.map((_, ti) => this._table(ti));
|
|
1233
|
+
const tableAt = new Int32Array(this._count).fill(-1);
|
|
1234
|
+
const rowAt = new Int32Array(this._count);
|
|
1235
|
+
tables.forEach((t, ti) => {
|
|
1236
|
+
const positions = t.positions;
|
|
1237
|
+
for (let row = 0; row < positions.length; row += 1) {
|
|
1238
|
+
const at = positions[row]!;
|
|
1239
|
+
if (!(at >= 0 && at < this._count) || tableAt[at] !== -1) {
|
|
1240
|
+
throw new SealedStoreError('Frozen table is unreadable');
|
|
1241
|
+
}
|
|
1242
|
+
tableAt[at] = ti;
|
|
1243
|
+
rowAt[at] = row;
|
|
1244
|
+
}
|
|
1245
|
+
});
|
|
1246
|
+
this._tableAt = tableAt;
|
|
1247
|
+
this._rowAt = rowAt;
|
|
1248
|
+
this._tables = tables;
|
|
1249
|
+
this._lazyCx = null;
|
|
1250
|
+
this._lazyReady = null;
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
/**
|
|
1255
|
+
* Frozen content read as a record store: the `instances` map, its order, the
|
|
1256
|
+
* schema version and the header. Every server-side reader that walks records
|
|
1257
|
+
* (an exporter, a view) reads through this, and it answers for any frozen
|
|
1258
|
+
* content: a root the content does not hold reads as empty.
|
|
1259
|
+
*/
|
|
1260
|
+
export class SealedStore {
|
|
1261
|
+
private readonly _records: SealedMapReader | null;
|
|
1262
|
+
|
|
1263
|
+
private constructor(private readonly _content: SealedContent) {
|
|
1264
|
+
this._records = _content.mapRoot(RECORDS_KEY);
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
static decode(bytes: Uint8Array): SealedStore {
|
|
1268
|
+
return new SealedStore(SealedContent.decode(bytes));
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
static of(content: SealedContent): SealedStore {
|
|
1272
|
+
return new SealedStore(content);
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
/** The general form this view reads. */
|
|
1276
|
+
get content(): SealedContent {
|
|
1277
|
+
return this._content;
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
get byteLength(): number {
|
|
1281
|
+
return this._content.byteLength;
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
/** The schema version, the `meta` root's one key. */
|
|
1285
|
+
schemaVersion(): string {
|
|
1286
|
+
const v = this._content.mapRoot(META_KEY)?.get(SCHEMA_VERSION_KEY);
|
|
1287
|
+
return typeof v === 'string' ? v : '';
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
/** The header entries in the record shape, in order. */
|
|
1291
|
+
header(): SealedHeaderEntry[] {
|
|
1292
|
+
const out: SealedHeaderEntry[] = [];
|
|
1293
|
+
for (const item of this._content.list(HEADER_KEY) ?? []) {
|
|
1294
|
+
if (isRecordShaped(item)) out.push({ tag: item.type, body: item.params });
|
|
1295
|
+
}
|
|
1296
|
+
return out;
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
builtBy(): SealedStoreBuiltBy | null {
|
|
1300
|
+
return this._content.builtBy();
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
keys(): string[] {
|
|
1304
|
+
return this._records?.keys() ?? [];
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
recordCount(): number {
|
|
1308
|
+
return this._records?.size() ?? 0;
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
has(key: string): boolean {
|
|
1312
|
+
return this._records?.has(key) ?? false;
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
get(key: string): SealedRecord | undefined {
|
|
1316
|
+
if (this._records === null || !this._records.has(key)) return undefined;
|
|
1317
|
+
return sealedRecordOf(this._records.get(key));
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
forEachRecord(visit: (key: string, tag: string | null, body: unknown) => void): void {
|
|
1321
|
+
this._records?.forEach((key, value) => {
|
|
1322
|
+
const record = sealedRecordOf(value);
|
|
1323
|
+
visit(key, record.tag, record.body);
|
|
1324
|
+
});
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
releaseDecoded(): void {
|
|
1328
|
+
this._records?.releaseDecoded();
|
|
1329
|
+
}
|
|
1330
|
+
}
|