@genesislcap/foundation-comms 15.79.2 → 15.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/asconfig.json +15 -0
- package/assembly/scdb/README.md +74 -0
- package/assembly/scdb/bench.mjs +32 -0
- package/assembly/scdb/decoder.ts +217 -0
- package/assembly/scdb/decoder.wasm +0 -0
- package/assembly/scdb/inline-wasm-base64.mjs +24 -0
- package/dist/dts/connect/connect.d.ts +2 -0
- package/dist/dts/connect/connect.d.ts.map +1 -1
- package/dist/dts/connect/http.connect.d.ts +10 -0
- package/dist/dts/connect/http.connect.d.ts.map +1 -1
- package/dist/dts/connect/message.d.ts +18 -0
- package/dist/dts/connect/message.d.ts.map +1 -1
- package/dist/dts/connect/scdbStreamDecoder.d.ts +66 -0
- package/dist/dts/connect/scdbStreamDecoder.d.ts.map +1 -0
- package/dist/dts/connect/socket.d.ts +23 -0
- package/dist/dts/connect/socket.d.ts.map +1 -1
- package/dist/dts/connect/socket.types.d.ts +14 -0
- package/dist/dts/connect/socket.types.d.ts.map +1 -1
- package/dist/dts/datasource/datasource.d.ts.map +1 -1
- package/dist/dts/datasource/datasource.types.d.ts +8 -0
- package/dist/dts/datasource/datasource.types.d.ts.map +1 -1
- package/dist/dts/datasource/index.d.ts +1 -0
- package/dist/dts/datasource/index.d.ts.map +1 -1
- package/dist/dts/datasource/scdb/base64.d.ts +6 -0
- package/dist/dts/datasource/scdb/base64.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/dictionaryHash.d.ts +40 -0
- package/dist/dts/datasource/scdb/dictionaryHash.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/index.d.ts +6 -0
- package/dist/dts/datasource/scdb/index.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdb-js-bench.d.mts +2 -0
- package/dist/dts/datasource/scdb/scdb-js-bench.d.mts.map +1 -0
- package/dist/dts/datasource/scdb/scdbDecode.d.ts +24 -0
- package/dist/dts/datasource/scdb/scdbDecode.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdbDecoder.d.ts +42 -0
- package/dist/dts/datasource/scdb/scdbDecoder.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdbWasmDecoder.d.ts +27 -0
- package/dist/dts/datasource/scdb/scdbWasmDecoder.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/scdbjDecoder.d.ts +35 -0
- package/dist/dts/datasource/scdb/scdbjDecoder.d.ts.map +1 -0
- package/dist/dts/datasource/scdb/wasm/decoder.wasm.base64.d.ts +7 -0
- package/dist/dts/datasource/scdb/wasm/decoder.wasm.base64.d.ts.map +1 -0
- package/dist/dts/testing/mocks/connect/socket.d.ts +1 -0
- package/dist/dts/testing/mocks/connect/socket.d.ts.map +1 -1
- package/dist/esm/connect/connect.js +22 -3
- package/dist/esm/connect/http.connect.js +33 -1
- package/dist/esm/connect/message.js +3 -1
- package/dist/esm/connect/scdbStreamDecoder.js +160 -0
- package/dist/esm/connect/socket.js +72 -2
- package/dist/esm/datasource/datasource.js +3 -0
- package/dist/esm/datasource/index.js +1 -0
- package/dist/esm/datasource/scdb/base64.js +17 -0
- package/dist/esm/datasource/scdb/dictionaryHash.js +45 -0
- package/dist/esm/datasource/scdb/index.js +5 -0
- package/dist/esm/datasource/scdb/scdb-golden-v1.json +15 -0
- package/dist/esm/datasource/scdb/scdb-js-bench.mjs +461 -0
- package/dist/esm/datasource/scdb/scdbDecode.js +67 -0
- package/dist/esm/datasource/scdb/scdbDecoder.js +279 -0
- package/dist/esm/datasource/scdb/scdbWasmDecoder.js +213 -0
- package/dist/esm/datasource/scdb/scdbjDecoder.js +60 -0
- package/dist/esm/datasource/scdb/wasm/decoder.wasm.base64.js +6 -0
- package/dist/esm/testing/mocks/connect/socket.js +3 -0
- package/dist/foundation-comms.api.json +1453 -8
- package/dist/foundation-comms.d.ts +251 -0
- package/package.json +14 -12
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JS decode micro-benchmark for SCDB (Phase 3c).
|
|
3
|
+
*
|
|
4
|
+
* Times decoding the *real* server wire bytes (dumped by the Kotlin `ScdbBenchDataDumpTest`, run
|
|
5
|
+
* with `SCDB_DUMP=true`) and reconstructing the legacy ROW shape, against two baselines:
|
|
6
|
+
* - MessagePack decode of the row-mode payload ← the REAL current wire path (also JS)
|
|
7
|
+
* - JSON.parse of the row-mode JSON ← native C++ reference (lower bound)
|
|
8
|
+
*
|
|
9
|
+
* Usage: node scdb-js-bench.mjs [/path/to/scdb-bench-data.json]
|
|
10
|
+
*
|
|
11
|
+
* The SCDB decode logic mirrors the optimized `scdbDecoder.ts` (kept as a standalone copy so the
|
|
12
|
+
* benchmark runs with plain Node, no build step). Keep them in sync: nulls are INLINE in the value
|
|
13
|
+
* stream (TAG_NULL / dict index 0 — no null bitmap), colFlags bit0 = DENSE (presence bitmap omitted),
|
|
14
|
+
* flags bit1 = UNIFORM_OPS. On startup it self-checks against the shared golden vectors so drift from
|
|
15
|
+
* the shipped decoder fails loudly instead of silently misreading bytes.
|
|
16
|
+
*/
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import { fileURLToPath } from 'node:url';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
const dec = new TextDecoder();
|
|
21
|
+
const OPS = ['INSERT', 'MODIFY', 'DELETE'];
|
|
22
|
+
const FLAG_UNIFORM_OPS = 0x02;
|
|
23
|
+
const COL_FLAG_DENSE = 0x01;
|
|
24
|
+
function bit(b, i) {
|
|
25
|
+
return ((b[i >> 3] >> (i & 7)) & 1) === 1;
|
|
26
|
+
}
|
|
27
|
+
function byteLen(n) {
|
|
28
|
+
return (n + 7) >> 3;
|
|
29
|
+
}
|
|
30
|
+
// Manual ASCII fast-path string decode (TextDecoder has ~100ns/call fixed overhead).
|
|
31
|
+
function readString(cur) {
|
|
32
|
+
const len = cur.uv();
|
|
33
|
+
const b = cur.b;
|
|
34
|
+
const p = cur.p;
|
|
35
|
+
const end = p + len;
|
|
36
|
+
cur.p = end;
|
|
37
|
+
let ascii = true;
|
|
38
|
+
for (let i = p; i < end; i++) {
|
|
39
|
+
if (b[i] >= 0x80) {
|
|
40
|
+
ascii = false;
|
|
41
|
+
break;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (ascii) {
|
|
45
|
+
if (len <= 4096)
|
|
46
|
+
return String.fromCharCode.apply(null, b.subarray(p, end));
|
|
47
|
+
let s = '';
|
|
48
|
+
for (let i = p; i < end; i += 4096)
|
|
49
|
+
s += String.fromCharCode.apply(null, b.subarray(i, Math.min(end, i + 4096)));
|
|
50
|
+
return s;
|
|
51
|
+
}
|
|
52
|
+
return dec.decode(b.subarray(p, end));
|
|
53
|
+
}
|
|
54
|
+
class Cursor {
|
|
55
|
+
constructor(b) {
|
|
56
|
+
this.b = b;
|
|
57
|
+
this.p = 0;
|
|
58
|
+
this.v = new DataView(b.buffer, b.byteOffset, b.byteLength);
|
|
59
|
+
}
|
|
60
|
+
byte() {
|
|
61
|
+
return this.b[this.p++];
|
|
62
|
+
}
|
|
63
|
+
slice(n) {
|
|
64
|
+
const s = this.b.subarray(this.p, this.p + n);
|
|
65
|
+
this.p += n;
|
|
66
|
+
return s;
|
|
67
|
+
}
|
|
68
|
+
uv() {
|
|
69
|
+
let r = 0, s = 0;
|
|
70
|
+
for (;;) {
|
|
71
|
+
const x = this.byte();
|
|
72
|
+
if (s < 28)
|
|
73
|
+
r |= (x & 0x7f) << s;
|
|
74
|
+
else
|
|
75
|
+
r += (x & 0x7f) * Math.pow(2, s);
|
|
76
|
+
if ((x & 0x80) === 0)
|
|
77
|
+
return r >>> 0;
|
|
78
|
+
s += 7;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Signed zigzag: Number fast-path (exact for |v| < 2^53), BigInt only past 49 bits.
|
|
82
|
+
zz() {
|
|
83
|
+
let r = 0, shift = 0, b, big = null;
|
|
84
|
+
for (;;) {
|
|
85
|
+
b = this.byte();
|
|
86
|
+
if (shift < 49)
|
|
87
|
+
r += (b & 0x7f) * Math.pow(2, shift);
|
|
88
|
+
else {
|
|
89
|
+
if (big === null)
|
|
90
|
+
big = BigInt(r);
|
|
91
|
+
big += BigInt(b & 0x7f) << BigInt(shift);
|
|
92
|
+
}
|
|
93
|
+
shift += 7;
|
|
94
|
+
if ((b & 0x80) === 0)
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
if (big === null)
|
|
98
|
+
return r % 2 ? -(r + 1) / 2 : r / 2;
|
|
99
|
+
const s = (big >> 1n) ^ -(big & 1n);
|
|
100
|
+
return s >= -9007199254740991n && s <= 9007199254740991n ? Number(s) : s;
|
|
101
|
+
}
|
|
102
|
+
dbe() {
|
|
103
|
+
const v = this.v.getFloat64(this.p, false);
|
|
104
|
+
this.p += 8;
|
|
105
|
+
return v;
|
|
106
|
+
}
|
|
107
|
+
str() {
|
|
108
|
+
return readString(this);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function tagged(c) {
|
|
112
|
+
const t = c.byte();
|
|
113
|
+
switch (t) {
|
|
114
|
+
case 0:
|
|
115
|
+
return c.str();
|
|
116
|
+
case 1:
|
|
117
|
+
return c.zz();
|
|
118
|
+
case 2:
|
|
119
|
+
return c.zz();
|
|
120
|
+
case 3:
|
|
121
|
+
return c.dbe();
|
|
122
|
+
case 4:
|
|
123
|
+
return c.byte() !== 0;
|
|
124
|
+
case 5:
|
|
125
|
+
return c.zz();
|
|
126
|
+
case 6:
|
|
127
|
+
return c.str();
|
|
128
|
+
case 7:
|
|
129
|
+
return c.slice(c.uv());
|
|
130
|
+
case 8:
|
|
131
|
+
return c.zz();
|
|
132
|
+
case 9:
|
|
133
|
+
return null; // inline NULL (replaces the old null bitmap)
|
|
134
|
+
default:
|
|
135
|
+
throw new Error('tag ' + t);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
// Mirrors the shipped scdbDecoder.ts decodeScdbToRows: inline nulls, DENSE flag, UNIFORM_OPS,
|
|
139
|
+
// delta-zigzag — single pass straight into the row objects, no intermediate column arrays.
|
|
140
|
+
function decodeToRows(bytes, dict) {
|
|
141
|
+
const c = new Cursor(bytes);
|
|
142
|
+
for (const m of [0x53, 0x43, 0x44, 0x31])
|
|
143
|
+
if (c.byte() !== m)
|
|
144
|
+
throw new Error('magic');
|
|
145
|
+
const flags = c.byte();
|
|
146
|
+
const n = c.uv();
|
|
147
|
+
const rows = new Array(n);
|
|
148
|
+
const refs = new Array(n);
|
|
149
|
+
for (let i = 0; i < n; i++)
|
|
150
|
+
refs[i] = c.str();
|
|
151
|
+
let ops;
|
|
152
|
+
if ((flags & FLAG_UNIFORM_OPS) !== 0) {
|
|
153
|
+
const op = OPS[c.byte()];
|
|
154
|
+
ops = new Array(n).fill(op);
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
const ob = c.slice(byteLen(n * 2));
|
|
158
|
+
ops = new Array(n);
|
|
159
|
+
for (let i = 0; i < n; i++) {
|
|
160
|
+
const lo = bit(ob, i * 2) ? 1 : 0, hi = bit(ob, i * 2 + 1) ? 2 : 0;
|
|
161
|
+
ops[i] = OPS[lo | hi];
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
for (let i = 0; i < n; i++)
|
|
165
|
+
rows[i] = { DETAILS: { OPERATION: ops[i], ROW_REF: refs[i] } };
|
|
166
|
+
const cc = c.uv();
|
|
167
|
+
for (let k = 0; k < cc; k++) {
|
|
168
|
+
const field = dict[c.uv()];
|
|
169
|
+
const enc = c.byte();
|
|
170
|
+
const dense = (c.byte() & COL_FLAG_DENSE) !== 0; // presence bitmap omitted when dense
|
|
171
|
+
const presence = dense ? null : c.slice(byteLen(n));
|
|
172
|
+
if (enc === 1) {
|
|
173
|
+
// STRING_DICT: dict then per-present-row index (0 = NULL, k = dict[k-1]).
|
|
174
|
+
const ds = c.uv();
|
|
175
|
+
const d = new Array(ds);
|
|
176
|
+
for (let i = 0; i < ds; i++)
|
|
177
|
+
d[i] = c.str();
|
|
178
|
+
for (let r = 0; r < n; r++) {
|
|
179
|
+
if (dense || bit(presence, r)) {
|
|
180
|
+
const idx = c.uv();
|
|
181
|
+
rows[r][field] = idx === 0 ? null : d[idx - 1];
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
else if (enc === 2) {
|
|
186
|
+
// DELTA_ZIGZAG: type tag + first + (P-1) deltas; cumulative sum, materialised per type tag.
|
|
187
|
+
const typeTag = c.byte();
|
|
188
|
+
let acc = c.zz();
|
|
189
|
+
const useBig = typeof acc === 'bigint';
|
|
190
|
+
let present = 0;
|
|
191
|
+
for (let r = 0; r < n; r++) {
|
|
192
|
+
if (dense || bit(presence, r)) {
|
|
193
|
+
if (present > 0) {
|
|
194
|
+
const d = c.zz();
|
|
195
|
+
acc = useBig
|
|
196
|
+
? acc + (typeof d === 'bigint' ? d : BigInt(d))
|
|
197
|
+
: acc + (typeof d === 'number' ? d : Number(d));
|
|
198
|
+
}
|
|
199
|
+
rows[r][field] = typeTag === 2 ? acc : Number(acc); // LONG keeps number|bigint; others -> number
|
|
200
|
+
present++;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
// RAW_TAGGED: inline self-describing values; TAG_NULL (9) decodes to null.
|
|
206
|
+
for (let r = 0; r < n; r++) {
|
|
207
|
+
if (dense || bit(presence, r))
|
|
208
|
+
rows[r][field] = tagged(c);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return rows;
|
|
213
|
+
}
|
|
214
|
+
// --- Minimal MessagePack decoder: enough for the row-mode payload (the real wire baseline). ---
|
|
215
|
+
function mpDecode(bytes) {
|
|
216
|
+
const v = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
217
|
+
let p = 0;
|
|
218
|
+
const str = (len) => {
|
|
219
|
+
const s = readStringAt(bytes, p, len);
|
|
220
|
+
p += len;
|
|
221
|
+
return s;
|
|
222
|
+
};
|
|
223
|
+
function readStringAt(b, off, len) {
|
|
224
|
+
const end = off + len;
|
|
225
|
+
let ascii = true;
|
|
226
|
+
for (let i = off; i < end; i++)
|
|
227
|
+
if (b[i] >= 0x80) {
|
|
228
|
+
ascii = false;
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
if (ascii)
|
|
232
|
+
return len <= 4096
|
|
233
|
+
? String.fromCharCode.apply(null, b.subarray(off, end))
|
|
234
|
+
: dec.decode(b.subarray(off, end));
|
|
235
|
+
return dec.decode(b.subarray(off, end));
|
|
236
|
+
}
|
|
237
|
+
function val() {
|
|
238
|
+
const c = bytes[p++];
|
|
239
|
+
if (c < 0x80)
|
|
240
|
+
return c; // positive fixint
|
|
241
|
+
if (c >= 0xe0)
|
|
242
|
+
return c - 256; // negative fixint
|
|
243
|
+
if (c >= 0x80 && c <= 0x8f)
|
|
244
|
+
return map(c & 0x0f);
|
|
245
|
+
if (c >= 0x90 && c <= 0x9f)
|
|
246
|
+
return arr(c & 0x0f);
|
|
247
|
+
if (c >= 0xa0 && c <= 0xbf)
|
|
248
|
+
return str(c & 0x1f);
|
|
249
|
+
switch (c) {
|
|
250
|
+
case 0xc0:
|
|
251
|
+
return null;
|
|
252
|
+
case 0xc2:
|
|
253
|
+
return false;
|
|
254
|
+
case 0xc3:
|
|
255
|
+
return true;
|
|
256
|
+
case 0xcc:
|
|
257
|
+
return bytes[p++];
|
|
258
|
+
case 0xcd: {
|
|
259
|
+
const x = v.getUint16(p);
|
|
260
|
+
p += 2;
|
|
261
|
+
return x;
|
|
262
|
+
}
|
|
263
|
+
case 0xce: {
|
|
264
|
+
const x = v.getUint32(p);
|
|
265
|
+
p += 4;
|
|
266
|
+
return x;
|
|
267
|
+
}
|
|
268
|
+
case 0xcf: {
|
|
269
|
+
const x = v.getBigUint64(p);
|
|
270
|
+
p += 8;
|
|
271
|
+
return x <= 9007199254740991n ? Number(x) : x;
|
|
272
|
+
}
|
|
273
|
+
case 0xd0:
|
|
274
|
+
return v.getInt8(p++);
|
|
275
|
+
case 0xd1: {
|
|
276
|
+
const x = v.getInt16(p);
|
|
277
|
+
p += 2;
|
|
278
|
+
return x;
|
|
279
|
+
}
|
|
280
|
+
case 0xd2: {
|
|
281
|
+
const x = v.getInt32(p);
|
|
282
|
+
p += 4;
|
|
283
|
+
return x;
|
|
284
|
+
}
|
|
285
|
+
case 0xd3: {
|
|
286
|
+
const x = v.getBigInt64(p);
|
|
287
|
+
p += 8;
|
|
288
|
+
return x >= -9007199254740991n && x <= 9007199254740991n ? Number(x) : x;
|
|
289
|
+
}
|
|
290
|
+
case 0xca: {
|
|
291
|
+
const x = v.getFloat32(p);
|
|
292
|
+
p += 4;
|
|
293
|
+
return x;
|
|
294
|
+
}
|
|
295
|
+
case 0xcb: {
|
|
296
|
+
const x = v.getFloat64(p);
|
|
297
|
+
p += 8;
|
|
298
|
+
return x;
|
|
299
|
+
}
|
|
300
|
+
case 0xd9: {
|
|
301
|
+
const l = bytes[p++];
|
|
302
|
+
return str(l);
|
|
303
|
+
}
|
|
304
|
+
case 0xda: {
|
|
305
|
+
const l = v.getUint16(p);
|
|
306
|
+
p += 2;
|
|
307
|
+
return str(l);
|
|
308
|
+
}
|
|
309
|
+
case 0xdb: {
|
|
310
|
+
const l = v.getUint32(p);
|
|
311
|
+
p += 4;
|
|
312
|
+
return str(l);
|
|
313
|
+
}
|
|
314
|
+
case 0xdc: {
|
|
315
|
+
const l = v.getUint16(p);
|
|
316
|
+
p += 2;
|
|
317
|
+
return arr(l);
|
|
318
|
+
}
|
|
319
|
+
case 0xdd: {
|
|
320
|
+
const l = v.getUint32(p);
|
|
321
|
+
p += 4;
|
|
322
|
+
return arr(l);
|
|
323
|
+
}
|
|
324
|
+
case 0xde: {
|
|
325
|
+
const l = v.getUint16(p);
|
|
326
|
+
p += 2;
|
|
327
|
+
return map(l);
|
|
328
|
+
}
|
|
329
|
+
case 0xdf: {
|
|
330
|
+
const l = v.getUint32(p);
|
|
331
|
+
p += 4;
|
|
332
|
+
return map(l);
|
|
333
|
+
}
|
|
334
|
+
default:
|
|
335
|
+
throw new Error('msgpack byte 0x' + c.toString(16));
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
function arr(len) {
|
|
339
|
+
const a = new Array(len);
|
|
340
|
+
for (let i = 0; i < len; i++)
|
|
341
|
+
a[i] = val();
|
|
342
|
+
return a;
|
|
343
|
+
}
|
|
344
|
+
function map(len) {
|
|
345
|
+
const o = {};
|
|
346
|
+
for (let i = 0; i < len; i++) {
|
|
347
|
+
const k = val();
|
|
348
|
+
o[k] = val();
|
|
349
|
+
}
|
|
350
|
+
return o;
|
|
351
|
+
}
|
|
352
|
+
return val();
|
|
353
|
+
}
|
|
354
|
+
function timeNs(warmup, iters, fn) {
|
|
355
|
+
let sink = 0;
|
|
356
|
+
for (let i = 0; i < warmup; i++)
|
|
357
|
+
sink ^= fn().length | 0;
|
|
358
|
+
const start = process.hrtime.bigint();
|
|
359
|
+
for (let i = 0; i < iters; i++)
|
|
360
|
+
sink ^= fn().length | 0;
|
|
361
|
+
const elapsed = process.hrtime.bigint() - start;
|
|
362
|
+
if (sink === 42)
|
|
363
|
+
console.log('');
|
|
364
|
+
return Number(elapsed) / iters;
|
|
365
|
+
}
|
|
366
|
+
// --- Self-check against the shared golden vectors: fails loudly if this decoder drifts from the
|
|
367
|
+
// wire format (the bug this file previously shipped — a stale separate-null-bitmap layout). ---
|
|
368
|
+
function fromHex(hex) {
|
|
369
|
+
const out = new Uint8Array(hex.length / 2);
|
|
370
|
+
for (let i = 0; i < out.length; i++)
|
|
371
|
+
out[i] = parseInt(hex.substr(i * 2, 2), 16);
|
|
372
|
+
return out;
|
|
373
|
+
}
|
|
374
|
+
// Key-order-insensitive canonical form (cells are emitted in column-id order, not literal order).
|
|
375
|
+
function canon(x) {
|
|
376
|
+
if (Array.isArray(x))
|
|
377
|
+
return '[' + x.map(canon).join(',') + ']';
|
|
378
|
+
if (x && typeof x === 'object')
|
|
379
|
+
return ('{' +
|
|
380
|
+
Object.keys(x)
|
|
381
|
+
.sort()
|
|
382
|
+
.map((k) => JSON.stringify(k) + ':' + canon(x[k]))
|
|
383
|
+
.join(',') +
|
|
384
|
+
'}');
|
|
385
|
+
return JSON.stringify(x);
|
|
386
|
+
}
|
|
387
|
+
function goldenSelfCheck() {
|
|
388
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
389
|
+
const golden = JSON.parse(fs.readFileSync(path.join(here, 'scdb-golden-v1.json'), 'utf8'));
|
|
390
|
+
const DICT = ['QUANTITY', 'PRICE', 'SYMBOL', 'STATUS', 'ACTIVE'];
|
|
391
|
+
const EXPECTED = {
|
|
392
|
+
'mixed-tristate': [
|
|
393
|
+
{ DETAILS: { OPERATION: 'MODIFY', ROW_REF: '1' }, PRICE: 10, QUANTITY: 100 },
|
|
394
|
+
{ DETAILS: { OPERATION: 'MODIFY', ROW_REF: '2' }, PRICE: null },
|
|
395
|
+
{ DETAILS: { OPERATION: 'MODIFY', ROW_REF: '9007199254740993' }, SYMBOL: 'VOD.L' },
|
|
396
|
+
],
|
|
397
|
+
'string-dict': [
|
|
398
|
+
{ DETAILS: { OPERATION: 'MODIFY', ROW_REF: '1' }, STATUS: 'ACTIVE' },
|
|
399
|
+
{ DETAILS: { OPERATION: 'MODIFY', ROW_REF: '2' }, STATUS: 'ACTIVE' },
|
|
400
|
+
{ DETAILS: { OPERATION: 'MODIFY', ROW_REF: '3' }, STATUS: 'CLOSED' },
|
|
401
|
+
],
|
|
402
|
+
'deletes-only': [
|
|
403
|
+
{ DETAILS: { OPERATION: 'DELETE', ROW_REF: '5' } },
|
|
404
|
+
{ DETAILS: { OPERATION: 'DELETE', ROW_REF: '6' } },
|
|
405
|
+
],
|
|
406
|
+
empty: [],
|
|
407
|
+
'delta-longs': [
|
|
408
|
+
{ DETAILS: { OPERATION: 'INSERT', ROW_REF: '1' }, QUANTITY: 1000000000000 },
|
|
409
|
+
{ DETAILS: { OPERATION: 'INSERT', ROW_REF: '2' }, QUANTITY: 1000000000005 },
|
|
410
|
+
{ DETAILS: { OPERATION: 'INSERT', ROW_REF: '3' }, QUANTITY: 1000000000011 },
|
|
411
|
+
],
|
|
412
|
+
};
|
|
413
|
+
for (const [name, expected] of Object.entries(EXPECTED)) {
|
|
414
|
+
const got = decodeToRows(fromHex(golden.vectors[name]), DICT);
|
|
415
|
+
const a = canon(got), b = canon(expected);
|
|
416
|
+
if (a !== b) {
|
|
417
|
+
console.error(`GOLDEN MISMATCH '${name}':\n got ${a}\n expected ${b}`);
|
|
418
|
+
process.exit(1);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
console.log('golden vectors: OK (decoder matches the shared conformance fixture)');
|
|
422
|
+
}
|
|
423
|
+
goldenSelfCheck();
|
|
424
|
+
const fixturePath = process.argv[2] || '/tmp/scdb-bench-data.json';
|
|
425
|
+
const data = JSON.parse(fs.readFileSync(fixturePath, 'utf8'));
|
|
426
|
+
console.log('\n=== JS decode ns/op: SCDB vs MessagePack(row) vs JSON.parse(row) ===');
|
|
427
|
+
console.log([
|
|
428
|
+
'shape'.padEnd(22),
|
|
429
|
+
'rows'.padStart(6),
|
|
430
|
+
'cols'.padStart(5),
|
|
431
|
+
'scdb B'.padStart(9),
|
|
432
|
+
'mp B'.padStart(9),
|
|
433
|
+
'scdb ns'.padStart(10),
|
|
434
|
+
'mp ns'.padStart(10),
|
|
435
|
+
'json ns'.padStart(10),
|
|
436
|
+
].join(' '));
|
|
437
|
+
for (const s of data.shapes) {
|
|
438
|
+
const dict = s.dictionary || data.dictionary; // per-shape dictionary (new fixtures); top-level fallback
|
|
439
|
+
const sb = Buffer.from(s.scdbBase64, 'base64');
|
|
440
|
+
const u8 = new Uint8Array(sb.buffer, sb.byteOffset, sb.byteLength);
|
|
441
|
+
const hasMp = !!s.rowMsgpackBase64;
|
|
442
|
+
const mb = hasMp ? Buffer.from(s.rowMsgpackBase64, 'base64') : null;
|
|
443
|
+
const mp = hasMp ? new Uint8Array(mb.buffer, mb.byteOffset, mb.byteLength) : null;
|
|
444
|
+
const big = s.rows >= 10000; // keep wall-time sane on large batches
|
|
445
|
+
const warmup = big ? 10 : 50;
|
|
446
|
+
const iters = big ? 50 : 500;
|
|
447
|
+
const scdbNs = timeNs(warmup, iters, () => decodeToRows(u8, dict));
|
|
448
|
+
const mpNs = hasMp ? timeNs(warmup, iters, () => mpDecode(mp).ROW) : NaN;
|
|
449
|
+
const jsonNs = timeNs(warmup, iters, () => JSON.parse(s.rowJson).ROW);
|
|
450
|
+
console.log([
|
|
451
|
+
s.name.padEnd(22),
|
|
452
|
+
String(s.rows).padStart(6),
|
|
453
|
+
String(dict.length).padStart(5),
|
|
454
|
+
String(u8.length).padStart(9),
|
|
455
|
+
String(hasMp ? mp.length : 0).padStart(9),
|
|
456
|
+
scdbNs.toFixed(0).padStart(10),
|
|
457
|
+
(hasMp ? mpNs.toFixed(0) : '-').padStart(10),
|
|
458
|
+
jsonNs.toFixed(0).padStart(10),
|
|
459
|
+
].join(' '));
|
|
460
|
+
}
|
|
461
|
+
console.log('\nMessagePack(row) is the real current wire path (JS-decoded). JSON.parse is a native lower bound.\n');
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { __rest } from "tslib";
|
|
2
|
+
/* oxlint-disable no-plusplus -- index-loop counters */
|
|
3
|
+
/**
|
|
4
|
+
* Single entry point the comms socket path uses to turn a columnar QUERY_UPDATE into the legacy
|
|
5
|
+
* `ROW` array of row objects, regardless of negotiated encoding. Downstream (`updateState`, ag-Grid
|
|
6
|
+
* datasources) is unchanged.
|
|
7
|
+
*
|
|
8
|
+
* Wiring (see README):
|
|
9
|
+
* 1. Derive the column dictionary from the resource's metadata (id = index into the ordered field
|
|
10
|
+
* list) and validate it against the server's `DICT_HASH`; the dictionary is NOT sent on the wire.
|
|
11
|
+
* 2. On each `QUERY_UPDATE`, call `decodeColumnarQueryUpdate(msg, dictionary, wasm?)` and use the
|
|
12
|
+
* returned array as `msg.ROW`.
|
|
13
|
+
*/
|
|
14
|
+
import { base64ToBytes } from './base64';
|
|
15
|
+
import { decodeScdbToRows } from './scdbDecoder';
|
|
16
|
+
import { decodeScdbjToCells, decodeScdbjToRows } from './scdbjDecoder';
|
|
17
|
+
/**
|
|
18
|
+
* Decodes a QUERY_UPDATE to row objects. Chooses by `ENCODING`:
|
|
19
|
+
* - `SCDB_V1` -> binary blob: WASM decoder when provided (fastest), else pure-JS.
|
|
20
|
+
* - `SCDBJ_V1` -> JSON-columnar transpose.
|
|
21
|
+
* - otherwise -> the message's existing `ROW` array (legacy).
|
|
22
|
+
*/
|
|
23
|
+
export function decodeColumnarQueryUpdate(msg, dictionary, wasm) {
|
|
24
|
+
var _a;
|
|
25
|
+
switch (msg.ENCODING) {
|
|
26
|
+
case 'SCDBJ_V1':
|
|
27
|
+
return decodeScdbjToRows(msg.SCDBJ, dictionary);
|
|
28
|
+
case 'SCDB_V1': {
|
|
29
|
+
const bytes = toBytes(msg.RBUF);
|
|
30
|
+
return wasm ? wasm.decodeToRows(bytes, dictionary) : decodeScdbToRows(bytes, dictionary);
|
|
31
|
+
}
|
|
32
|
+
default:
|
|
33
|
+
return (_a = msg.ROW) !== null && _a !== void 0 ? _a : [];
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/** Normalises an RBUF payload: Uint8Array (msgpack), base64 string or number[] (JSON transports). */
|
|
37
|
+
function toBytes(rbuf) {
|
|
38
|
+
if (rbuf instanceof Uint8Array)
|
|
39
|
+
return rbuf;
|
|
40
|
+
if (Array.isArray(rbuf))
|
|
41
|
+
return Uint8Array.from(rbuf);
|
|
42
|
+
if (typeof rbuf === 'string')
|
|
43
|
+
return base64ToBytes(rbuf);
|
|
44
|
+
throw new Error('SCDB_V1 message has no RBUF payload');
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Decodes a self-describing columnar req/rep reply (dictionary inline) into plain result rows
|
|
48
|
+
* (cells only, no synthetic `DETAILS`). req/rep replies are full result sets, so unlike a dataserver
|
|
49
|
+
* `QUERY_UPDATE` they carry no per-row operation / rowRef.
|
|
50
|
+
*/
|
|
51
|
+
export function decodeColumnarReply(msg, dictionary) {
|
|
52
|
+
var _a;
|
|
53
|
+
switch (msg.ENCODING) {
|
|
54
|
+
case 'SCDBJ_V1':
|
|
55
|
+
return decodeScdbjToCells(msg.SCDBJ, dictionary);
|
|
56
|
+
case 'SCDB_V1': {
|
|
57
|
+
// Not currently emitted for req/rep; decode then drop the synthetic DETAILS for parity.
|
|
58
|
+
const bytes = toBytes(msg.RBUF);
|
|
59
|
+
return decodeScdbToRows(bytes, dictionary).map((_a) => {
|
|
60
|
+
var { DETAILS } = _a, cells = __rest(_a, ["DETAILS"]);
|
|
61
|
+
return cells;
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
default:
|
|
65
|
+
return (_a = msg.REPLY) !== null && _a !== void 0 ? _a : [];
|
|
66
|
+
}
|
|
67
|
+
}
|