@genesislcap/foundation-comms 15.79.2 → 15.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/asconfig.json +15 -0
- package/assembly/scdb/README.md +74 -0
- package/assembly/scdb/bench.mjs +32 -0
- package/assembly/scdb/decoder.ts +217 -0
- package/assembly/scdb/decoder.wasm +0 -0
- package/assembly/scdb/inline-wasm-base64.mjs +24 -0
- package/dist/dts/connect/connect.d.ts +2 -0
- package/dist/dts/connect/connect.d.ts.map +1 -1
- package/dist/dts/connect/http.connect.d.ts +10 -0
- package/dist/dts/connect/http.connect.d.ts.map +1 -1
- package/dist/dts/connect/message.d.ts +18 -0
- package/dist/dts/connect/message.d.ts.map +1 -1
- package/dist/dts/connect/scdbStreamDecoder.d.ts +66 -0
- package/dist/dts/connect/scdbStreamDecoder.d.ts.map +1 -0
- package/dist/dts/connect/socket.d.ts +23 -0
- package/dist/dts/connect/socket.d.ts.map +1 -1
- package/dist/dts/connect/socket.types.d.ts +14 -0
- package/dist/dts/connect/socket.types.d.ts.map +1 -1
- package/dist/dts/datasource/datasource.d.ts.map +1 -1
- package/dist/dts/datasource/datasource.types.d.ts +8 -0
- package/dist/dts/datasource/datasource.types.d.ts.map +1 -1
- package/dist/dts/datasource/index.d.ts +1 -0
- package/dist/dts/datasource/index.d.ts.map +1 -1
- package/dist/dts/datasource/scdb/base64.d.ts +6 -0
- package/dist/dts/datasource/scdb/base64.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/dictionaryHash.d.ts +40 -0
- package/dist/dts/datasource/scdb/dictionaryHash.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/index.d.ts +6 -0
- package/dist/dts/datasource/scdb/index.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdb-js-bench.d.mts +2 -0
- package/dist/dts/datasource/scdb/scdb-js-bench.d.mts.map +1 -0
- package/dist/dts/datasource/scdb/scdbDecode.d.ts +24 -0
- package/dist/dts/datasource/scdb/scdbDecode.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdbDecoder.d.ts +42 -0
- package/dist/dts/datasource/scdb/scdbDecoder.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdbWasmDecoder.d.ts +27 -0
- package/dist/dts/datasource/scdb/scdbWasmDecoder.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdbjDecoder.d.ts +35 -0
- package/dist/dts/datasource/scdb/scdbjDecoder.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/wasm/decoder.wasm.base64.d.ts +7 -0
- package/dist/dts/datasource/scdb/wasm/decoder.wasm.base64.d.ts.map +1 -0
- package/dist/dts/testing/mocks/connect/socket.d.ts +1 -0
- package/dist/dts/testing/mocks/connect/socket.d.ts.map +1 -1
- package/dist/esm/connect/connect.js +22 -3
- package/dist/esm/connect/http.connect.js +33 -1
- package/dist/esm/connect/message.js +3 -1
- package/dist/esm/connect/scdbStreamDecoder.js +160 -0
- package/dist/esm/connect/socket.js +72 -2
- package/dist/esm/datasource/datasource.js +3 -0
- package/dist/esm/datasource/index.js +1 -0
- package/dist/esm/datasource/scdb/base64.js +17 -0
- package/dist/esm/datasource/scdb/dictionaryHash.js +45 -0
- package/dist/esm/datasource/scdb/index.js +5 -0
- package/dist/esm/datasource/scdb/scdb-golden-v1.json +15 -0
- package/dist/esm/datasource/scdb/scdb-js-bench.mjs +461 -0
- package/dist/esm/datasource/scdb/scdbDecode.js +67 -0
- package/dist/esm/datasource/scdb/scdbDecoder.js +279 -0
- package/dist/esm/datasource/scdb/scdbWasmDecoder.js +213 -0
- package/dist/esm/datasource/scdb/scdbjDecoder.js +60 -0
- package/dist/esm/datasource/scdb/wasm/decoder.wasm.base64.js +6 -0
- package/dist/esm/testing/mocks/connect/socket.js +3 -0
- package/dist/foundation-comms.api.json +1453 -8
- package/dist/foundation-comms.d.ts +251 -0
- package/package.json +14 -12
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
/* oxlint-disable no-magic-numbers, no-plusplus, max-depth -- binary SCDB codec: byte offsets, bit masks and index-loop counters read clearer inline than as named constants */
|
|
2
|
+
/**
|
|
3
|
+
* SCDB v1 (Sparse Columnar Delta Block) decoder.
|
|
4
|
+
*
|
|
5
|
+
* Decodes the binary `RBUF` payload that a Genesis dataserver sends when a client negotiates
|
|
6
|
+
* `DETAILS.ENCODING = SCDB_V1`. The wire format is defined, language-neutrally, in the server repo
|
|
7
|
+
* at `genesis-columnar/docs/scdb-v1.md` and is pinned by the shared golden vectors in
|
|
8
|
+
* `scdb-golden-v1.json` (this directory). The Kotlin `ScdbReader` is the reference implementation;
|
|
9
|
+
* this decoder must stay byte-for-byte compatible with it.
|
|
10
|
+
*
|
|
11
|
+
* Performance notes (see scdb-js-bench.mjs):
|
|
12
|
+
* - single pass straight into the output objects — no per-column intermediate arrays;
|
|
13
|
+
* - dense columns (flagged by the encoder) skip the presence bitmap and the per-row presence test;
|
|
14
|
+
* - varints decode in Number space (exact for |v| < 2^53), using BigInt only past 49 bits;
|
|
15
|
+
* - strings take a manual ASCII fast-path (TextDecoder has a fixed per-call overhead).
|
|
16
|
+
*
|
|
17
|
+
* Browser contract:
|
|
18
|
+
* - operates over the `Uint8Array` the existing msgpack envelope decoder already produces for `RBUF`;
|
|
19
|
+
* - `ROW_REF` decodes to a `string` (sent as decimal text) — matches the existing contract, no BigInt;
|
|
20
|
+
* - 64-bit LONG *values* decode to `number` when safe, else `bigint` (the customNumberParser rule).
|
|
21
|
+
*/
|
|
22
|
+
const MAGIC = [0x53, 0x43, 0x44, 0x31]; // 'S','C','D','1'
|
|
23
|
+
const ENC_STRING_DICT = 1;
|
|
24
|
+
const OPS = ['INSERT', 'MODIFY', 'DELETE'];
|
|
25
|
+
// BigInt(...) (not 1n literals) so this compiles at the package's ES2015 target.
|
|
26
|
+
const BIG_ONE = BigInt(1);
|
|
27
|
+
const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER);
|
|
28
|
+
const SAFE_MIN = -SAFE_MAX;
|
|
29
|
+
const sharedDecoder = new TextDecoder();
|
|
30
|
+
function bitSet(b, i) {
|
|
31
|
+
return ((b[i >> 3] >> (i & 7)) & 1) === 1;
|
|
32
|
+
}
|
|
33
|
+
function byteLength(bits) {
|
|
34
|
+
return (bits + 7) >> 3;
|
|
35
|
+
}
|
|
36
|
+
class Cursor {
|
|
37
|
+
constructor(bytes) {
|
|
38
|
+
this.bytes = bytes;
|
|
39
|
+
this.pos = 0;
|
|
40
|
+
this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
41
|
+
}
|
|
42
|
+
byte() {
|
|
43
|
+
return this.bytes[this.pos++];
|
|
44
|
+
}
|
|
45
|
+
slice(len) {
|
|
46
|
+
const out = this.bytes.subarray(this.pos, this.pos + len);
|
|
47
|
+
this.pos += len;
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
uvarint() {
|
|
51
|
+
let r = 0;
|
|
52
|
+
let s = 0;
|
|
53
|
+
for (;;) {
|
|
54
|
+
const x = this.byte();
|
|
55
|
+
if (s < 28)
|
|
56
|
+
r |= (x & 0x7f) << s;
|
|
57
|
+
else
|
|
58
|
+
r += (x & 0x7f) * Math.pow(2, s);
|
|
59
|
+
if ((x & 0x80) === 0)
|
|
60
|
+
return r >>> 0;
|
|
61
|
+
s += 7;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
zigzag() {
|
|
65
|
+
let r = 0;
|
|
66
|
+
let shift = 0;
|
|
67
|
+
let big = null;
|
|
68
|
+
for (;;) {
|
|
69
|
+
const b = this.byte();
|
|
70
|
+
if (shift < 49)
|
|
71
|
+
r += (b & 0x7f) * Math.pow(2, shift);
|
|
72
|
+
else {
|
|
73
|
+
if (big === null)
|
|
74
|
+
big = BigInt(r);
|
|
75
|
+
big += BigInt(b & 0x7f) << BigInt(shift);
|
|
76
|
+
}
|
|
77
|
+
shift += 7;
|
|
78
|
+
if ((b & 0x80) === 0)
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
if (big === null)
|
|
82
|
+
return r % 2 ? -(r + 1) / 2 : r / 2;
|
|
83
|
+
const s = (big >> BIG_ONE) ^ -(big & BIG_ONE);
|
|
84
|
+
return s >= SAFE_MIN && s <= SAFE_MAX ? Number(s) : s;
|
|
85
|
+
}
|
|
86
|
+
doubleBE() {
|
|
87
|
+
const v = this.view.getFloat64(this.pos, false);
|
|
88
|
+
this.pos += 8;
|
|
89
|
+
return v;
|
|
90
|
+
}
|
|
91
|
+
string() {
|
|
92
|
+
const len = this.uvarint();
|
|
93
|
+
const b = this.bytes;
|
|
94
|
+
const start = this.pos;
|
|
95
|
+
const end = start + len;
|
|
96
|
+
this.pos = end;
|
|
97
|
+
for (let i = start; i < end; i++) {
|
|
98
|
+
if (b[i] >= 0x80)
|
|
99
|
+
return sharedDecoder.decode(b.subarray(start, end));
|
|
100
|
+
}
|
|
101
|
+
if (len <= 4096)
|
|
102
|
+
return String.fromCharCode.apply(null, b.subarray(start, end));
|
|
103
|
+
let out = '';
|
|
104
|
+
for (let i = start; i < end; i += 4096) {
|
|
105
|
+
out += String.fromCharCode.apply(null, b.subarray(i, Math.min(end, i + 4096)));
|
|
106
|
+
}
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
function readTaggedValue(cur) {
|
|
111
|
+
const tag = cur.byte();
|
|
112
|
+
switch (tag) {
|
|
113
|
+
case 0:
|
|
114
|
+
return cur.string(); // STRING
|
|
115
|
+
case 1:
|
|
116
|
+
return cur.zigzag(); // INT
|
|
117
|
+
case 2:
|
|
118
|
+
return cur.zigzag(); // LONG (number when safe, else bigint)
|
|
119
|
+
case 3:
|
|
120
|
+
return cur.doubleBE(); // DOUBLE
|
|
121
|
+
case 4:
|
|
122
|
+
return cur.byte() !== 0; // BOOLEAN
|
|
123
|
+
case 5:
|
|
124
|
+
return cur.zigzag(); // SHORT
|
|
125
|
+
case 6:
|
|
126
|
+
return cur.string(); // BIGDECIMAL (kept as string)
|
|
127
|
+
case 7:
|
|
128
|
+
return cur.slice(cur.uvarint()); // BYTES
|
|
129
|
+
case 8:
|
|
130
|
+
return cur.zigzag(); // DATETIME (epoch millis)
|
|
131
|
+
case 9:
|
|
132
|
+
return null; // NULL (inline; replaces the null bitmap)
|
|
133
|
+
default:
|
|
134
|
+
throw new Error(`Unknown SCDB value tag ${tag}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Decodes an SCDB `RBUF` blob into delta records, using the column dictionary (ordered field-name
|
|
139
|
+
* list) the client derives from the resource's metadata and validates against the server's
|
|
140
|
+
* `DICT_HASH`. Each record's `cells` is a fresh target object passed to [decodeColumns].
|
|
141
|
+
*/
|
|
142
|
+
export function decodeScdb(bytes, dictionary) {
|
|
143
|
+
const cur = readHeader(bytes);
|
|
144
|
+
const flags = cur.byte();
|
|
145
|
+
const n = cur.uvarint();
|
|
146
|
+
const rowRefs = readRowRefs(cur, n);
|
|
147
|
+
const ops = readOps(cur, n, flags);
|
|
148
|
+
const records = new Array(n);
|
|
149
|
+
const targets = new Array(n);
|
|
150
|
+
for (let i = 0; i < n; i++) {
|
|
151
|
+
const cells = {};
|
|
152
|
+
records[i] = { rowRef: rowRefs[i], operation: ops[i], cells };
|
|
153
|
+
targets[i] = cells;
|
|
154
|
+
}
|
|
155
|
+
decodeColumns(cur, n, dictionary, targets);
|
|
156
|
+
return records;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Reconstructs the legacy `ROW` array shape directly (the hot path used in the live socket flow):
|
|
160
|
+
* `[{ ...cells, DETAILS: { OPERATION, ROW_REF } }]`. Builds straight into the row objects with no
|
|
161
|
+
* intermediate record/cells allocation.
|
|
162
|
+
*/
|
|
163
|
+
export function decodeScdbToRows(bytes, dictionary) {
|
|
164
|
+
const cur = readHeader(bytes);
|
|
165
|
+
const flags = cur.byte();
|
|
166
|
+
const n = cur.uvarint();
|
|
167
|
+
const rowRefs = readRowRefs(cur, n);
|
|
168
|
+
const ops = readOps(cur, n, flags);
|
|
169
|
+
const rows = new Array(n);
|
|
170
|
+
for (let i = 0; i < n; i++) {
|
|
171
|
+
rows[i] = { DETAILS: { OPERATION: ops[i], ROW_REF: rowRefs[i] } };
|
|
172
|
+
}
|
|
173
|
+
decodeColumns(cur, n, dictionary, rows);
|
|
174
|
+
return rows;
|
|
175
|
+
}
|
|
176
|
+
/** @internal */
|
|
177
|
+
export function assertScdbMagic(bytes) {
|
|
178
|
+
for (let i = 0; i < MAGIC.length; i++) {
|
|
179
|
+
if (bytes[i] !== MAGIC[i])
|
|
180
|
+
throw new Error('Not an SCDB block (bad magic)');
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
/** Validates the magic and leaves the cursor positioned at the flags byte (read by the caller). */
|
|
184
|
+
function readHeader(bytes) {
|
|
185
|
+
assertScdbMagic(bytes);
|
|
186
|
+
const cur = new Cursor(bytes);
|
|
187
|
+
cur.pos = MAGIC.length;
|
|
188
|
+
return cur;
|
|
189
|
+
}
|
|
190
|
+
function readRowRefs(cur, n) {
|
|
191
|
+
const refs = new Array(n);
|
|
192
|
+
for (let i = 0; i < n; i++)
|
|
193
|
+
refs[i] = cur.string();
|
|
194
|
+
return refs;
|
|
195
|
+
}
|
|
196
|
+
const FLAG_UNIFORM_OPS = 0x02;
|
|
197
|
+
function readOps(cur, n, flags) {
|
|
198
|
+
const ops = new Array(n);
|
|
199
|
+
if ((flags & FLAG_UNIFORM_OPS) !== 0) {
|
|
200
|
+
const op = OPS[cur.byte()];
|
|
201
|
+
for (let i = 0; i < n; i++)
|
|
202
|
+
ops[i] = op;
|
|
203
|
+
return ops;
|
|
204
|
+
}
|
|
205
|
+
const packed = cur.slice(byteLength(n * 2));
|
|
206
|
+
for (let i = 0; i < n; i++) {
|
|
207
|
+
const lo = bitSet(packed, i * 2) ? 1 : 0;
|
|
208
|
+
const hi = bitSet(packed, i * 2 + 1) ? 2 : 0;
|
|
209
|
+
ops[i] = OPS[lo | hi];
|
|
210
|
+
}
|
|
211
|
+
return ops;
|
|
212
|
+
}
|
|
213
|
+
const COL_FLAG_DENSE = 0x01;
|
|
214
|
+
const ENC_DELTA_ZIGZAG = 2;
|
|
215
|
+
/** Single-pass column decode straight into the supplied per-row target objects. Nulls are inline in
|
|
216
|
+
* the value stream (TAG_NULL for RAW_TAGGED, dict index 0 for STRING_DICT) — no null bitmap. */
|
|
217
|
+
function decodeColumns(cur, n, dictionary, targets) {
|
|
218
|
+
const columnCount = cur.uvarint();
|
|
219
|
+
for (let c = 0; c < columnCount; c++) {
|
|
220
|
+
const colId = cur.uvarint();
|
|
221
|
+
const field = dictionary[colId];
|
|
222
|
+
if (field === undefined)
|
|
223
|
+
throw new Error(`Unknown SCDB column id ${colId}`);
|
|
224
|
+
const encoding = cur.byte();
|
|
225
|
+
const dense = (cur.byte() & COL_FLAG_DENSE) !== 0;
|
|
226
|
+
const presence = dense ? null : cur.slice(byteLength(n));
|
|
227
|
+
if (encoding === ENC_STRING_DICT) {
|
|
228
|
+
const dictSize = cur.uvarint();
|
|
229
|
+
const dict = new Array(dictSize);
|
|
230
|
+
for (let i = 0; i < dictSize; i++)
|
|
231
|
+
dict[i] = cur.string();
|
|
232
|
+
for (let r = 0; r < n; r++) {
|
|
233
|
+
if (dense || bitSet(presence, r)) {
|
|
234
|
+
const idx = cur.uvarint();
|
|
235
|
+
targets[r][field] = idx === 0 ? null : dict[idx - 1];
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
else if (encoding === ENC_DELTA_ZIGZAG) {
|
|
240
|
+
// Cumulative sum of zigzag deltas. LONG (typeTag 2) accumulates in BigInt and narrows per cell
|
|
241
|
+
// (number when safe, else bigint), so a running total that crosses 2^53 stays exact — matching the
|
|
242
|
+
// WASM/Kotlin i64 accumulators and the customNumberParser rule. Deciding number-vs-bigint from the
|
|
243
|
+
// first value alone would silently lose precision once the cumulative sum passes 2^53.
|
|
244
|
+
// INT/SHORT/DATETIME are always < 2^53, so they accumulate in Number space.
|
|
245
|
+
const typeTag = cur.byte();
|
|
246
|
+
let present = 0;
|
|
247
|
+
if (typeTag === 2) {
|
|
248
|
+
let acc = BigInt(0);
|
|
249
|
+
for (let r = 0; r < n; r++) {
|
|
250
|
+
if (dense || bitSet(presence, r)) {
|
|
251
|
+
const v = cur.zigzag();
|
|
252
|
+
const delta = typeof v === 'bigint' ? v : BigInt(v);
|
|
253
|
+
acc = present > 0 ? acc + delta : delta;
|
|
254
|
+
targets[r][field] = acc >= SAFE_MIN && acc <= SAFE_MAX ? Number(acc) : acc;
|
|
255
|
+
present++;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
else {
|
|
260
|
+
let acc = 0;
|
|
261
|
+
for (let r = 0; r < n; r++) {
|
|
262
|
+
if (dense || bitSet(presence, r)) {
|
|
263
|
+
const v = cur.zigzag();
|
|
264
|
+
acc = present > 0 ? acc + Number(v) : Number(v);
|
|
265
|
+
targets[r][field] = acc;
|
|
266
|
+
present++;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
else {
|
|
272
|
+
for (let r = 0; r < n; r++) {
|
|
273
|
+
if (dense || bitSet(presence, r)) {
|
|
274
|
+
targets[r][field] = readTaggedValue(cur);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/* oxlint-disable no-magic-numbers, no-plusplus -- wasm memory layout: region byte-offsets/sizes and index-loop counters read clearer inline than as named constants */
|
|
2
|
+
/**
|
|
3
|
+
* WASM fast-path decoder for the binary SCDB (`SCDB_V1`) payload.
|
|
4
|
+
*
|
|
5
|
+
* Instantiates the AssemblyScript decoder (`wasm/decoder.wasm`), which parses the binary blob at
|
|
6
|
+
* near-native speed into a columnar result in linear memory; this module then assembles the legacy
|
|
7
|
+
* `ROW` array of objects that ag-Grid consumes. Measured ~1.5x faster than native `JSON.parse` on a
|
|
8
|
+
* full snapshot and ~2.5x on deltas, while the payload stays ~3x smaller on the wire. See
|
|
9
|
+
* `wasm/README.md` and the benchmark for numbers.
|
|
10
|
+
*
|
|
11
|
+
* If wasm is unavailable (or instantiation fails), callers fall back to the pure-JS `decodeScdbToRows`
|
|
12
|
+
* from `scdbDecoder.ts` — same output, no toolchain.
|
|
13
|
+
*/
|
|
14
|
+
import { __awaiter } from "tslib";
|
|
15
|
+
import { base64ToBytes } from './base64';
|
|
16
|
+
import { assertScdbMagic } from './scdbDecoder';
|
|
17
|
+
import { SCDB_DECODER_WASM_BASE64 } from './wasm/decoder.wasm.base64';
|
|
18
|
+
const OPS = ['INSERT', 'MODIFY', 'DELETE'];
|
|
19
|
+
const PAGE = 65536;
|
|
20
|
+
// LONG (kind 3) columns carry a raw i64; reconstruct it exactly, then narrow to a JS number when it
|
|
21
|
+
// fits the safe-integer range and keep a bigint otherwise — the same rule as the pure-JS decoder and
|
|
22
|
+
// the row-mode customNumberParser. BigInt(...) (not literals) to compile at the ES2015 target.
|
|
23
|
+
const BIG_32 = BigInt(32);
|
|
24
|
+
const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER);
|
|
25
|
+
const SAFE_MIN = -SAFE_MAX;
|
|
26
|
+
let defaultWasm;
|
|
27
|
+
/**
|
|
28
|
+
* Lazily instantiates (once) the bundled decoder from inlined base64 — no asset/fetch config, works
|
|
29
|
+
* in any bundler and SSR. This is what the socket layer uses for the `SCDB_V1` fast-path. Rejects if
|
|
30
|
+
* WebAssembly is unavailable; callers fall back to the pure-JS decoder.
|
|
31
|
+
*/
|
|
32
|
+
export function getDefaultScdbWasm() {
|
|
33
|
+
if (!defaultWasm)
|
|
34
|
+
defaultWasm = initScdbWasm();
|
|
35
|
+
return defaultWasm;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Instantiates the decoder. Defaults to the inlined `decoder.wasm`; pass your own bytes (bundler
|
|
39
|
+
* import / fetch / fs.readFile) to override. Returns an object whose `decodeToRows` is synchronous.
|
|
40
|
+
*/
|
|
41
|
+
export function initScdbWasm() {
|
|
42
|
+
return __awaiter(this, arguments, void 0, function* (wasmBytes = base64ToBytes(SCDB_DECODER_WASM_BASE64)) {
|
|
43
|
+
// Generous initial memory; grown on demand per batch.
|
|
44
|
+
const memory = new WebAssembly.Memory({ initial: 32 });
|
|
45
|
+
const { instance } = yield WebAssembly.instantiate(wasmBytes, {
|
|
46
|
+
env: {
|
|
47
|
+
memory,
|
|
48
|
+
abort() {
|
|
49
|
+
throw new Error('scdb wasm abort');
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
const decode = instance.exports.decode;
|
|
54
|
+
const decoder = new TextDecoder();
|
|
55
|
+
// Layout: blob at BLOB_BASE, output above it (8-aligned). Memory grown per batch as needed.
|
|
56
|
+
const BLOB_BASE = PAGE; // 64 KB
|
|
57
|
+
function decodeToRows(blob, dictionary) {
|
|
58
|
+
// Pre-parse to size output, then place blob + out and decode. Sparse layout regions:
|
|
59
|
+
// nums n*c*8 + strOff/strLen 2*(n*c*4) + refOff/refLen 2*(n*4) + colIds c*4 + kinds c + ops n
|
|
60
|
+
// + counts c*4 + idx (<=n*c)*4 + scratch (string-dict tables, <= n*8).
|
|
61
|
+
// The wasm decoder does not validate its input and re-reads the previous blob left in memory.
|
|
62
|
+
assertScdbMagic(blob);
|
|
63
|
+
const { n, c } = preparse(blob);
|
|
64
|
+
const outBytes = 64 + n * c * 8 + n * c * 4 * 2 + n * 4 * 2 + c * 4 + c + n + c * 4 + n * c * 4 + n * 8 + 4096;
|
|
65
|
+
const outBase = align8(BLOB_BASE + blob.length + 64);
|
|
66
|
+
const needed = outBase + outBytes;
|
|
67
|
+
if (needed > memory.buffer.byteLength) {
|
|
68
|
+
memory.grow(Math.ceil((needed - memory.buffer.byteLength) / PAGE));
|
|
69
|
+
}
|
|
70
|
+
const u8 = new Uint8Array(memory.buffer);
|
|
71
|
+
u8.set(blob, BLOB_BASE);
|
|
72
|
+
decode(BLOB_BASE, blob.length, outBase);
|
|
73
|
+
return assemble(memory.buffer, u8, BLOB_BASE, outBase, dictionary, decoder);
|
|
74
|
+
}
|
|
75
|
+
return { decodeToRows };
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
function align8(x) {
|
|
79
|
+
return (x + 7) & ~7;
|
|
80
|
+
}
|
|
81
|
+
/** Reads only n (row count) and c (column count) by skipping refs + ops. No allocation. */
|
|
82
|
+
function preparse(blob) {
|
|
83
|
+
const flags = blob[4];
|
|
84
|
+
let p = 5; // magic + flags
|
|
85
|
+
let r = 0;
|
|
86
|
+
let s = 0;
|
|
87
|
+
const uv = () => {
|
|
88
|
+
r = 0;
|
|
89
|
+
s = 0;
|
|
90
|
+
for (;;) {
|
|
91
|
+
const b = blob[p++];
|
|
92
|
+
if (s < 28)
|
|
93
|
+
r |= (b & 0x7f) << s;
|
|
94
|
+
else
|
|
95
|
+
r += (b & 0x7f) * Math.pow(2, s);
|
|
96
|
+
if ((b & 0x80) === 0)
|
|
97
|
+
return r >>> 0;
|
|
98
|
+
s += 7;
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
const n = uv();
|
|
102
|
+
for (let i = 0; i < n; i++) {
|
|
103
|
+
const l = uv();
|
|
104
|
+
p += l;
|
|
105
|
+
}
|
|
106
|
+
p += (flags & 0x02) !== 0 ? 1 : (n * 2 + 7) >> 3; // UNIFORM_OPS -> single op byte
|
|
107
|
+
const c = uv();
|
|
108
|
+
return { n, c };
|
|
109
|
+
}
|
|
110
|
+
function assemble(buffer, u8, blobBase, outBase, dictionary, decoder) {
|
|
111
|
+
const dv = new DataView(buffer);
|
|
112
|
+
const n = dv.getInt32(outBase, true);
|
|
113
|
+
const c = dv.getInt32(outBase + 4, true);
|
|
114
|
+
const base = outBase + 64;
|
|
115
|
+
const numsOff = base;
|
|
116
|
+
const strOffOff = numsOff + n * c * 8;
|
|
117
|
+
const strLenOff = strOffOff + n * c * 4;
|
|
118
|
+
const refOffOff = strLenOff + n * c * 4;
|
|
119
|
+
const refLenOff = refOffOff + n * 4;
|
|
120
|
+
const colIdsOff = refLenOff + n * 4;
|
|
121
|
+
// i32 regions (counts, idx) precede the i8 ones (kinds, ops) so these offsets stay 4-aligned.
|
|
122
|
+
const countsOff = colIdsOff + c * 4;
|
|
123
|
+
const idxOff = countsOff + c * 4;
|
|
124
|
+
const kindsOff = idxOff + n * c * 4;
|
|
125
|
+
const opsOff = kindsOff + c;
|
|
126
|
+
const nums = new Float64Array(buffer, numsOff, n * c);
|
|
127
|
+
const strOff = new Int32Array(buffer, strOffOff, n * c);
|
|
128
|
+
const strLen = new Int32Array(buffer, strLenOff, n * c);
|
|
129
|
+
const refOff = new Int32Array(buffer, refOffOff, n);
|
|
130
|
+
const refLen = new Int32Array(buffer, refLenOff, n);
|
|
131
|
+
const colIds = new Int32Array(buffer, colIdsOff, c);
|
|
132
|
+
const kinds = u8.subarray(kindsOff, kindsOff + c);
|
|
133
|
+
const ops = u8.subarray(opsOff, opsOff + n);
|
|
134
|
+
const counts = new Int32Array(buffer, countsOff, c);
|
|
135
|
+
const idx = new Int32Array(buffer, idxOff, n * c);
|
|
136
|
+
const rows = new Array(n);
|
|
137
|
+
for (let i = 0; i < n; i++) {
|
|
138
|
+
const ro = blobBase + refOff[i];
|
|
139
|
+
rows[i] = {
|
|
140
|
+
DETAILS: { OPERATION: OPS[ops[i]], ROW_REF: decoder.decode(u8.subarray(ro, ro + refLen[i])) },
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
// Walk only the present cells (sum of counts), in column order, via the idx list. A negative idx
|
|
144
|
+
// (high bit set) is an explicit change-to-null; otherwise the low 31 bits are the row index.
|
|
145
|
+
let cursor = 0;
|
|
146
|
+
for (let ci = 0; ci < c; ci++) {
|
|
147
|
+
const field = dictionary[colIds[ci]];
|
|
148
|
+
// Match the pure-JS decoder: a column id outside the dictionary is a hard error, not a silent
|
|
149
|
+
// write to `rows[r][undefined]` (which would corrupt the row shape without any signal).
|
|
150
|
+
if (field === undefined)
|
|
151
|
+
throw new Error(`Unknown SCDB column id ${colIds[ci]}`);
|
|
152
|
+
const kind = kinds[ci];
|
|
153
|
+
const cb = ci * n;
|
|
154
|
+
const p = counts[ci];
|
|
155
|
+
if (kind === 1) {
|
|
156
|
+
const cache = new Map();
|
|
157
|
+
for (let k = 0; k < p; k++) {
|
|
158
|
+
const raw = idx[cursor++];
|
|
159
|
+
const r = raw & 0x7fffffff;
|
|
160
|
+
if (raw < 0) {
|
|
161
|
+
rows[r][field] = null;
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const off = strOff[cb + r];
|
|
165
|
+
let s = cache.get(off);
|
|
166
|
+
if (s === undefined) {
|
|
167
|
+
const a = blobBase + off;
|
|
168
|
+
s = decoder.decode(u8.subarray(a, a + strLen[cb + r]));
|
|
169
|
+
cache.set(off, s);
|
|
170
|
+
}
|
|
171
|
+
rows[r][field] = s;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
else if (kind === 4) {
|
|
175
|
+
// BYTES: raw payload copied out of wasm memory as a Uint8Array (matches the pure-JS decoder's
|
|
176
|
+
// `cur.slice`), NOT UTF-8-decoded like kind 1. `slice` copies so the value survives wasm memory reuse.
|
|
177
|
+
for (let k = 0; k < p; k++) {
|
|
178
|
+
const raw = idx[cursor++];
|
|
179
|
+
const r = raw & 0x7fffffff;
|
|
180
|
+
if (raw < 0) {
|
|
181
|
+
rows[r][field] = null;
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
const a = blobBase + strOff[cb + r];
|
|
185
|
+
rows[r][field] = u8.slice(a, a + strLen[cb + r]);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
else if (kind === 3) {
|
|
189
|
+
// LONG: 8 bytes hold a little-endian i64 (not an f64). Reconstruct via signed-high/unsigned-low
|
|
190
|
+
// words, then narrow to number when safe else keep bigint — matches pure-JS + customNumberParser.
|
|
191
|
+
for (let k = 0; k < p; k++) {
|
|
192
|
+
const raw = idx[cursor++];
|
|
193
|
+
const r = raw & 0x7fffffff;
|
|
194
|
+
if (raw < 0) {
|
|
195
|
+
rows[r][field] = null;
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
const off = numsOff + (cb + r) * 8;
|
|
199
|
+
const v = (BigInt(dv.getInt32(off + 4, true)) << BIG_32) + BigInt(dv.getUint32(off, true));
|
|
200
|
+
rows[r][field] = v >= SAFE_MIN && v <= SAFE_MAX ? Number(v) : v;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
const bool = kind === 2;
|
|
205
|
+
for (let k = 0; k < p; k++) {
|
|
206
|
+
const raw = idx[cursor++];
|
|
207
|
+
const r = raw & 0x7fffffff;
|
|
208
|
+
rows[r][field] = raw < 0 ? null : bool ? nums[cb + r] !== 0 : nums[cb + r];
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return rows;
|
|
213
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/* oxlint-disable no-plusplus -- columnar transpose uses index-loop counters */
|
|
2
|
+
/**
|
|
3
|
+
* SCDBJ (JSON-columnar) decoder.
|
|
4
|
+
*
|
|
5
|
+
* The no-WASM peer of the binary `scdbDecoder`. The server sends a `QUERY_UPDATE` whose payload is a
|
|
6
|
+
* plain JSON object (decoded for free by the existing envelope `JSON.parse`); this transposes it back
|
|
7
|
+
* into the legacy `ROW` array of row objects that ag-Grid consumes. Because native `JSON.parse` does
|
|
8
|
+
* the bulk work and only the transpose is JS, this is at parity with row-JSON on a full snapshot and
|
|
9
|
+
* faster on deltas, while being ~2x smaller on the wire (no repeated field-name keys).
|
|
10
|
+
*
|
|
11
|
+
* Wire shape (short keys; see genesis-columnar/docs/scdb-v1.md):
|
|
12
|
+
* { n, r:[rowRef strings], o: opCode | [opCodes], c:[ { i:colId, x?:[presentIdx], v:[values] } ] }
|
|
13
|
+
* `x` omitted => dense column. Values are aligned to the present cells; a `null` entry is an explicit
|
|
14
|
+
* change-to-null. Column ids index the metadata-derived dictionary.
|
|
15
|
+
*/
|
|
16
|
+
const OPS = ['INSERT', 'MODIFY', 'DELETE'];
|
|
17
|
+
/** Transposes the per-column arrays into the supplied per-row target objects (shared core). */
|
|
18
|
+
function transpose(payload, dictionary, rows) {
|
|
19
|
+
for (const col of payload.c) {
|
|
20
|
+
const field = dictionary[col.i];
|
|
21
|
+
if (field === undefined)
|
|
22
|
+
throw new Error(`Unknown SCDBJ column id ${col.i}`);
|
|
23
|
+
const idx = col.x;
|
|
24
|
+
const vals = col.v;
|
|
25
|
+
const presentCount = idx ? idx.length : payload.n;
|
|
26
|
+
for (let present = 0; present < presentCount; present++) {
|
|
27
|
+
const rowIndex = idx ? idx[present] : present;
|
|
28
|
+
rows[rowIndex][field] = vals[present]; // a null entry is an explicit change-to-null
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return rows;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Transposes a parsed SCDBJ payload into the legacy dataserver ROW shape:
|
|
35
|
+
* `[{ ...cells, DETAILS: { OPERATION, ROW_REF } }]`, using the column dictionary.
|
|
36
|
+
*/
|
|
37
|
+
export function decodeScdbjToRows(payload, dictionary) {
|
|
38
|
+
const { n, r: refs, o: ops } = payload;
|
|
39
|
+
const opsArr = Array.isArray(ops);
|
|
40
|
+
const rows = new Array(n);
|
|
41
|
+
for (let i = 0; i < n; i++) {
|
|
42
|
+
rows[i] = {
|
|
43
|
+
DETAILS: {
|
|
44
|
+
OPERATION: OPS[opsArr ? ops[i] : ops],
|
|
45
|
+
ROW_REF: refs[i],
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
return transpose(payload, dictionary, rows);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Transposes a parsed SCDBJ payload into plain result rows (cells only, no `DETAILS`). Used for
|
|
53
|
+
* req/rep replies, which are full result sets without per-row operation / rowRef semantics.
|
|
54
|
+
*/
|
|
55
|
+
export function decodeScdbjToCells(payload, dictionary) {
|
|
56
|
+
const rows = new Array(payload.n);
|
|
57
|
+
for (let i = 0; i < payload.n; i++)
|
|
58
|
+
rows[i] = {};
|
|
59
|
+
return transpose(payload, dictionary, rows);
|
|
60
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The SCDB decoder `decoder.wasm` inlined as base64 so it loads in any bundler/runtime
|
|
3
|
+
* (webpack, Vite, Next, esbuild, SSR) with no asset-resolution or fetch config.
|
|
4
|
+
* Generated from assembly/scdb/decoder.ts — do not edit by hand; run `npm run build:wasm`.
|
|
5
|
+
*/
|
|
6
|
+
export const SCDB_DECODER_WASM_BASE64 = 'AGFzbQEAAAABCAFgA39/fwF/Ag8BA2VudgZtZW1vcnkCAAADAgEABgsCfwFBAAt/AUEACwcTAgZkZWNvZGUAAAZtZW1vcnkCAAq5EgG2EgIRfwR+IAAkACMALQAEQQJxQQBHIQ5BBSQBA0AjASIAQQFqJAEgFCMAIABqLQAAIgBB/wBxrCAHrYaEIRQgAEGAAXEEQCAHQQdqIQcMAQsLIBSnIQEDQCABIAhKBEBCACEUQQAhAANAIwEiB0EBaiQBIBQjACAHai0AACIHQf8AcawgAK2GhCEUIAdBgAFxBEAgAEEHaiEADAELCyMBIBSnaiQBIAhBAWohCAwBCwsjASEAIA4Ef0EBBSABQQF0QQdqQQN1CyAAaiQBA0AjASIAQQFqJAEgFSMAIABqLQAAIgBB/wBxrCAKrYaEIRUgAEGAAXEEQCAKQQdqIQoMAQsLIAIgATYCACACIBWnIgA2AgQgAkFAayIKIAAgAWwiAkEDdGoiDCACQQJ0IgJqIg8gAmoiECABQQJ0IgdqIREgByARaiIHIABBAnQiCGohEiACIAggEmoiDWoiCCAAaiIAIAFqQQNqQXxxIRNBBSQBA0AjASICQQFqJAEgFyMAIAJqLQAAIgJB/wBxrCAGrYaEIRcgAkGAAXEEQCAGQQdqIQYMAQsLA0AgASAESgRAQgAhFEEAIQIDQCMBIgZBAWokASAUIwAgBmotAAAiBkH/AHGsIAKthoQhFCAGQYABcQRAIAJBB2ohAgwBCwsgBEECdCICIBBqIwE2AgAgAiARaiAUpyICNgIAIwEgAmokASAEQQFqIQQMAQsLIA4EQCMBIgJBAWokASMAIAJqLQAAIQIDQCABIAVKBEAgACAFaiACOgAAIAVBAWohBQwBCwsFIwEhAgNAIAEgBUoEQCAAIAVqIwAgAmogBUEBdCIEQQN1ai0AACAEQQdxdUEDcToAACAFQQFqIQUMAQsLIwEgAUEBdEEHakEDdWokAQsDQCMBIgBBAWokASAWIwAgAGotAAAiAEH/AHGsIAuthoQhFiAAQYABcQRAIAtBB2ohCwwBCwsgFqchCwNAIAkgC0gEQEIAIRRBACEAA0AjASICQQFqJAEgFCMAIAJqLQAAIgJB/wBxrCAArYaEIRQgAkGAAXEEQCAAQQdqIQAMAQsLIAcgCUECdGogFD4CACMBIgBBAWokASMAIABqLQAAIQIjASIAQQFqJAFBfyEGIwAgAGotAABBAXEiDkUEQCMBIQYjASABQQdqQQN1aiQBC0EAIQUgAyEAIAJBAkYEQCMBIgJBAWokASMAIAJqLQAAQQJGIRBCACEUQQEhBEEAIQIDQCABIAJKBEAgDgR/QQEFIwAgBmogAkEDdWotAAAgAkEHcXVBAXELBEAgBAR+QgAhFEEAIQUDQCMBIgRBAWokASAUIwAgBGotAAAiBEH/AHGsIAWthoQhFCAEQYABcQRAIAVBB2ohBQwBCwtBACEEQgAgFEIBg30gFEIBiIUFQgAhFUEAIQUDQCMBIhFBAWokASAVIwAgEWotAAAiEUH/AHGsIAWthoQhFSARQYABcQRAIAVBB2ohBQwBCwsgFEIAIBVCAYN9IBVCAYiFfAshFCAQBEAgCiABIAlsIAJqQQN0aiAUNwMABSAKIAEgCWwgAmpBA3RqIBS5OQMACyANIANBAnRqIAI2AgAgA0EBaiEDCyACQQFqIQIMAQsLQQNBACAQGyEFBSACQQFGBEBCACEUQQAhAgNAIwEiBEEBaiQBIBQjACAEai0AACIEQf8AcawgAq2GhCEUIARBgAFxBEAgAkEHaiECDAELCyATIBSnIgRBAnRqIRBBACECA0AgAiAESARAQgAhFEEAIQUDQCMBIhFBAWokASAUIwAgEWotAAAiEUH/AHGsIAWthoQhFCARQYABcQRAIAVBB2ohBQwBCwsgAkECdCIFIBNqIwE2AgAgBSAQaiAUpyIFNgIAIwEgBWokASACQQFqIQIMAQsLQQAhAgNAIAEgAkoEQCAOBH9BAQUjACAGaiACQQN1ai0AACACQQdxdUEBcQsEQEIAIRRBACEFA0AjASIEQQFqJAEgFCMAIARqLQAAIgRB/wBxrCAFrYaEIRQgBEGAAXEEQCAFQQdqIQUMAQsLIBSnIgQEQCABIAlsIAJqQQJ0IgUgDGogBEEBa0ECdCIEIBNqKAIANgIAIAUgD2ogBCAQaigCADYCACANIANBAnRqIAI2AgAFIA0gA0ECdGogAkGAgICAeHI2AgALIANBAWohAwsgAkEBaiECDAELC0EBIQUFQQAhAgNAIAEgAkoEQCAOBH9BAQUjACAGaiACQQN1ai0AACACQQdxdUEBcQsEQCMBIgRBAWokASMAIARqLQAAIgRBCUYEQCANIANBAnRqIAJBgICAgHhyNgIABSABIAlsIAJqIRAgBEUgBEEGRnIEQEEBIQVCACEUQQAhBANAIwEiEUEBaiQBIBQjACARai0AACIRQf8AcawgBK2GhCEUIBFBgAFxBEAgBEEHaiEEDAELCyAQQQJ0IgQgDGojATYCACAEIA9qIBSnIgQ2AgAjASAEaiQBBSAEQQdGBEBBBCEFQgAhFEEAIQQDQCMBIhFBAWokASAUIwAgEWotAAAiEUH/AHGsIASthoQhFCARQYABcQRAIARBB2ohBAwBCwsgEEECdCIEIAxqIwE2AgAgBCAPaiAUpyIENgIAIwEgBGokAQUgBEEDRgRAQgAhFEEAIQQDQCAEQQhIBEAjACMBaiAEajEAACAUQgiGhCEUIARBAWohBAwBCwsjAUEIaiQBIAogEEEDdGogFDcDAAUgBEEERgRAQQIhBSMBIgRBAWokASAKIBBBA3RqIwAgBGotAAC3OQMABSAEQQJGBEBCACEUQQAhBQNAIwEiBEEBaiQBIBQjACAEai0AACIEQf8AcawgBa2GhCEUIARBgAFxBEAgBUEHaiEFDAELCyAKIBBBA3RqQgAgFEIBg30gFEIBiIU3AwBBAyEFBUIAIRRBACEEA0AjASIRQQFqJAEgFCMAIBFqLQAAIhFB/wBxrCAErYaEIRQgEUGAAXEEQCAEQQdqIQQMAQsLIAogEEEDdGpCACAUQgGDfSAUQgGIhbk5AwALCwsLCyANIANBAnRqIAI2AgALIANBAWohAwsgAkEBaiECDAELCwsLIBIgCUECdGogAyAAazYCACAIIAlqIAU6AAAgCUEBaiEJDAELCyABCw==';
|