@genome-spy/app 0.88.1 → 0.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agentApi-BIpg3JKh.js → agentApi-Z65dw-sD.js} +5976 -6711
- package/dist/agentApi.es.js +3580 -4470
- package/dist/agentShared.es.js +742 -683
- package/dist/{canvas2d-BVQ7-2eJ.js → canvas2d-DiPQvNtZ.js} +122 -109
- package/dist/{canvas2d-D7SZK3xv.js → canvas2d-t_N10CeW.js} +114 -101
- package/dist/{canvasSizeHelper-7h_8rOcx.js → canvasSizeHelper-foL2YvGL.js} +15 -8
- package/dist/{clipOptions-DTMcB_9z.js → clipOptions-B5PibDq3.js} +1610 -1342
- package/dist/{clipOptions-BWHr3aTa.js → clipOptions-Cba4I0AO.js} +1872 -1644
- package/dist/{componentStyles-FsNmIb22.js → componentStyles-DGtQXvwc.js} +17 -15
- package/dist/{debug-CVpoEOWx.js → debug-CVAlrcZt.js} +2 -2
- package/dist/{debug-DYAMZZKl.js → debug-DP4WzTNo.js} +2 -2
- package/dist/dialog.es.js +4 -3
- package/dist/esm-BiqrFjMh.js +443 -0
- package/dist/{esm-lwYrRdbg.js → esm-Bv7Frz7C.js} +142 -79
- package/dist/{esm-VvpZ9hsq.js → esm-C3A-AuRi.js} +1 -1
- package/dist/esm-CvpR1OcU.js +322 -0
- package/dist/esm-DFHFX1sH.js +1526 -0
- package/dist/esm-Dndm-NaA.js +1193 -0
- package/dist/{esm-CcDLRSvC.js → esm-DtNElqRr.js} +64 -63
- package/dist/esm-L53qjcYH.js +1155 -0
- package/dist/esm-VFJeyk9i.js +249 -0
- package/dist/{esm-BdLYkz-m.js → esm-_xCtpDwm.js} +1 -1
- package/dist/index.es.js +4203 -3910
- package/dist/index.js +971 -482
- package/dist/{indexer-CXjZDo2Q.js → indexer-DbFTKGlm.js} +1 -1
- package/dist/{indexer-DAH_ruVn.js → indexer-En7gpvLN.js} +1 -1
- package/dist/{lit-ByuJ0FJ4.js → lit-CoXp9NFX.js} +12 -10
- package/dist/order-BkYCnwD7.js +1062 -0
- package/dist/order-R043xTpC.js +1062 -0
- package/dist/{parquetRead-Cs2xRrvz.js → parquetRead-0hFIa3Cc.js} +532 -350
- package/dist/rasterExport-C_FcNBkC.js +52 -0
- package/dist/rasterExport-Z7gs5346.js +52 -0
- package/dist/rasterization-_-9JmSas.js +63 -0
- package/dist/rectangle-DE98gIxF.js +108 -0
- package/dist/{renderCanvas2D-DxturTEW.js → renderCanvas2D-DK1c9TFK.js} +68 -67
- package/dist/{renderCanvas2D-DtcJjqpB.js → renderCanvas2D-IM-A9DYG.js} +57 -56
- package/dist/schema.json +599 -116
- package/dist/{src-DjOlFqSn.js → src-DJtvThVC.js} +3 -5
- package/dist/style.css +1 -1
- package/dist/{svg-SDJMzL--.js → svg-DW3tfUmm.js} +339 -326
- package/dist/{svg-CyrLfa9e.js → svg-PabUdcDi.js} +365 -352
- package/dist/{svgRasterizer-f7DJ3UO0.js → svgRasterizer-B6YtWLJs.js} +6 -5
- package/dist/{svgRasterizer-mWK7HTkx.js → svgRasterizer-BEVDyyS3.js} +6 -5
- package/dist/{vega-scale-DLKCwWRs.js → vega-scale-Byaw1K8O.js} +823 -780
- package/dist/{vega-scale-CypF85iS.js → vega-scale-Ctz_wa3z.js} +963 -923
- package/dist/{viewIdentityRegistry-Dl-vMN9I.js → viewIdentityRegistry-BW415b3o.js} +1 -1
- package/dist/{viewIdentityRegistry-DIDo44XQ.js → viewIdentityRegistry-_F0E7nkA.js} +1 -1
- package/dist/{viewSelectors-CymbZyE6.js → viewSelectors-DYz-PhTO.js} +407 -340
- package/dist/{viewSelectors-DNiDip5e.js → viewSelectors-KtD-QSN0.js} +349 -282
- package/dist/{viewUtils-BrJLIj2o.js → viewUtils-BZGPseMx.js} +2052 -1871
- package/dist/{webgl-tZfZ8y3q.js → webgl-BKSGqHaz.js} +2144 -784
- package/dist/{webgl-CTJt7jQd.js → webgl-tdKf9SlZ.js} +2386 -1026
- package/package.json +5 -5
- package/dist/esm-80V4xF_D.js +0 -1098
- package/dist/esm-BA6yMcc-.js +0 -1222
- package/dist/esm-BdT7qUL-.js +0 -157
- package/dist/esm-CfYZhLHN.js +0 -1367
- package/dist/esm-D-iG_TIX.js +0 -441
- package/dist/esm-JAiZmFIS.js +0 -128
- package/dist/order-C5CzA37O.js +0 -918
- package/dist/order-DvY7KNcu.js +0 -918
- package/dist/rasterExport-BgZ98e6U.js +0 -44
- package/dist/rasterExport-DEr8fOLm.js +0 -44
- package/dist/rasterization-CFogVB3B.js +0 -50
- package/dist/rectangle-DKnrAmKs.js +0 -142
|
@@ -90,28 +90,32 @@ function s(e) {
|
|
|
90
90
|
type: "MultiPoint",
|
|
91
91
|
coordinates: n
|
|
92
92
|
};
|
|
93
|
-
}
|
|
93
|
+
}
|
|
94
|
+
if (t.type === 5) {
|
|
94
95
|
let n = [];
|
|
95
96
|
for (let r = 0; r < t.count; r++) n.push(u(e, c(e)));
|
|
96
97
|
return {
|
|
97
98
|
type: "MultiLineString",
|
|
98
99
|
coordinates: n
|
|
99
100
|
};
|
|
100
|
-
}
|
|
101
|
+
}
|
|
102
|
+
if (t.type === 6) {
|
|
101
103
|
let n = [];
|
|
102
104
|
for (let r = 0; r < t.count; r++) n.push(d(e, c(e)));
|
|
103
105
|
return {
|
|
104
106
|
type: "MultiPolygon",
|
|
105
107
|
coordinates: n
|
|
106
108
|
};
|
|
107
|
-
}
|
|
109
|
+
}
|
|
110
|
+
if (t.type === 7) {
|
|
108
111
|
let n = [];
|
|
109
112
|
for (let r = 0; r < t.count; r++) n.push(s(e));
|
|
110
113
|
return {
|
|
111
114
|
type: "GeometryCollection",
|
|
112
115
|
geometries: n
|
|
113
116
|
};
|
|
114
|
-
}
|
|
117
|
+
}
|
|
118
|
+
throw Error(`Unsupported geometry type: ${t.type}`);
|
|
115
119
|
}
|
|
116
120
|
function c(e) {
|
|
117
121
|
let { view: t } = e, n = t.getUint8(e.offset++) === 1, r = t.getUint32(e.offset, n);
|
|
@@ -168,6 +172,9 @@ var f = new TextDecoder(), p = {
|
|
|
168
172
|
stringFromBytes(e) {
|
|
169
173
|
return e && f.decode(e);
|
|
170
174
|
},
|
|
175
|
+
jsonFromBytes(e) {
|
|
176
|
+
return e && JSON.parse(f.decode(e));
|
|
177
|
+
},
|
|
171
178
|
geometryFromBytes(e) {
|
|
172
179
|
return e && s({
|
|
173
180
|
view: new DataView(e.buffer, e.byteOffset, e.byteLength),
|
|
@@ -192,7 +199,8 @@ function m(e, t, n, r) {
|
|
|
192
199
|
e instanceof Uint8Array && !(t instanceof Uint8Array) && (n = new t.constructor(e.length));
|
|
193
200
|
for (let r = 0; r < e.length; r++) n[r] = t[e[r]];
|
|
194
201
|
return n;
|
|
195
|
-
}
|
|
202
|
+
}
|
|
203
|
+
return h(e, r);
|
|
196
204
|
}
|
|
197
205
|
function h(e, t) {
|
|
198
206
|
let { element: n, parsers: r, utf8: i = !0, schemaPath: a } = t, { type: o, converted_type: s, logical_type: c } = n, l = n.repetition_type !== "REQUIRED";
|
|
@@ -206,7 +214,7 @@ function h(e, t) {
|
|
|
206
214
|
if (s === "DATE") return Array.from(e).map((e) => r.dateFromDays(e));
|
|
207
215
|
if (s === "TIMESTAMP_MILLIS") return Array.from(e).map((e) => r.timestampFromMilliseconds(e));
|
|
208
216
|
if (s === "TIMESTAMP_MICROS") return Array.from(e).map((e) => r.timestampFromMicroseconds(e));
|
|
209
|
-
if (s === "JSON") return e.map((e) =>
|
|
217
|
+
if (s === "JSON") return e.map((e) => r.jsonFromBytes(e));
|
|
210
218
|
if (s === "BSON") throw Error("parquet bson not supported");
|
|
211
219
|
if (s === "INTERVAL") throw Error("parquet interval not supported");
|
|
212
220
|
if (c?.type === "GEOMETRY") return e.map((e) => r.geometryFromBytes(e));
|
|
@@ -301,54 +309,54 @@ function ne(e) {
|
|
|
301
309
|
function re(e) {
|
|
302
310
|
if (!e || e.element.converted_type !== "MAP" || e.children.length > 1) return !1;
|
|
303
311
|
let t = e.children[0];
|
|
304
|
-
return
|
|
312
|
+
return t.children.length === 2 && t.element.repetition_type === "REPEATED" && t.children.find((e) => e.element.name === "key")?.element.repetition_type !== "REPEATED" && t.children.find((e) => e.element.name === "value")?.element.repetition_type !== "REPEATED";
|
|
305
313
|
}
|
|
306
|
-
function
|
|
314
|
+
function ie(e) {
|
|
307
315
|
if (e.length !== 2) return !1;
|
|
308
316
|
let [, t] = e;
|
|
309
317
|
return !(t.element.repetition_type === "REPEATED" || t.children.length);
|
|
310
318
|
}
|
|
311
319
|
//#endregion
|
|
312
320
|
//#region ../../node_modules/hyparquet/src/thrift.js
|
|
313
|
-
var
|
|
314
|
-
function
|
|
321
|
+
var ae = 0, oe = 1, se = 2, ce = 3, le = 4, ue = 5, de = 6, fe = 7, pe = 8, me = 9, he = 12;
|
|
322
|
+
function S(e) {
|
|
315
323
|
let t = {}, n = 0;
|
|
316
324
|
for (; e.offset < e.view.byteLength;) {
|
|
317
325
|
let r = e.view.getUint8(e.offset++), i = r & 15;
|
|
318
|
-
if (i ===
|
|
326
|
+
if (i === ae) break;
|
|
319
327
|
let a = r >> 4;
|
|
320
|
-
n = a ? n + a :
|
|
328
|
+
n = a ? n + a : T(e), t[`field_${n}`] = C(e, i);
|
|
321
329
|
}
|
|
322
330
|
return t;
|
|
323
331
|
}
|
|
324
|
-
function
|
|
332
|
+
function C(e, t) {
|
|
325
333
|
switch (t) {
|
|
326
|
-
case
|
|
327
|
-
case
|
|
328
|
-
case
|
|
329
|
-
case
|
|
330
|
-
case
|
|
331
|
-
case
|
|
332
|
-
case
|
|
334
|
+
case oe: return !0;
|
|
335
|
+
case se: return !1;
|
|
336
|
+
case ce: return e.view.getInt8(e.offset++);
|
|
337
|
+
case le:
|
|
338
|
+
case ue: return T(e);
|
|
339
|
+
case de: return E(e);
|
|
340
|
+
case fe: {
|
|
333
341
|
let t = e.view.getFloat64(e.offset, !0);
|
|
334
342
|
return e.offset += 8, t;
|
|
335
343
|
}
|
|
336
|
-
case
|
|
337
|
-
let t =
|
|
344
|
+
case pe: {
|
|
345
|
+
let t = w(e), n = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
338
346
|
return e.offset += t, n;
|
|
339
347
|
}
|
|
340
|
-
case
|
|
348
|
+
case me: {
|
|
341
349
|
let t = e.view.getUint8(e.offset++), n = t & 15, r = t >> 4;
|
|
342
|
-
r === 15 && (r =
|
|
343
|
-
let i = n ===
|
|
344
|
-
for (let t = 0; t < r; t++) a[t] = i ?
|
|
350
|
+
r === 15 && (r = w(e));
|
|
351
|
+
let i = n === oe || n === se, a = Array(r);
|
|
352
|
+
for (let t = 0; t < r; t++) a[t] = i ? C(e, ce) === 1 : C(e, n);
|
|
345
353
|
return a;
|
|
346
354
|
}
|
|
347
|
-
case
|
|
355
|
+
case he: return S(e);
|
|
348
356
|
default: throw Error(`thrift unhandled type: ${t}`);
|
|
349
357
|
}
|
|
350
358
|
}
|
|
351
|
-
function
|
|
359
|
+
function w(e) {
|
|
352
360
|
let t = 0, n = 0;
|
|
353
361
|
for (;;) {
|
|
354
362
|
let r = e.view.getUint8(e.offset++);
|
|
@@ -356,7 +364,7 @@ function T(e) {
|
|
|
356
364
|
n += 7;
|
|
357
365
|
}
|
|
358
366
|
}
|
|
359
|
-
function
|
|
367
|
+
function ge(e) {
|
|
360
368
|
let t = 0n, n = 0n;
|
|
361
369
|
for (;;) {
|
|
362
370
|
let r = e.view.getUint8(e.offset++);
|
|
@@ -364,12 +372,12 @@ function he(e) {
|
|
|
364
372
|
n += 7n;
|
|
365
373
|
}
|
|
366
374
|
}
|
|
367
|
-
function
|
|
368
|
-
let t =
|
|
375
|
+
function T(e) {
|
|
376
|
+
let t = w(e);
|
|
369
377
|
return t >>> 1 ^ -(t & 1);
|
|
370
378
|
}
|
|
371
379
|
function E(e) {
|
|
372
|
-
let t =
|
|
380
|
+
let t = ge(e);
|
|
373
381
|
return t >> 1n ^ -(t & 1n);
|
|
374
382
|
}
|
|
375
383
|
//#endregion
|
|
@@ -411,25 +419,26 @@ async function be(e, { parsers: t, initialFetchSize: n = ve, geoparquet: r = !0
|
|
|
411
419
|
parsers: t,
|
|
412
420
|
geoparquet: r
|
|
413
421
|
});
|
|
414
|
-
}
|
|
422
|
+
}
|
|
423
|
+
return xe(a, {
|
|
415
424
|
parsers: t,
|
|
416
425
|
geoparquet: r
|
|
417
426
|
});
|
|
418
427
|
}
|
|
419
428
|
function xe(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
420
429
|
if (!(o instanceof ArrayBuffer)) throw Error("parquet expected ArrayBuffer");
|
|
421
|
-
let l = new DataView(o)
|
|
422
|
-
if (s = {
|
|
430
|
+
let l = new DataView(o), u = {
|
|
423
431
|
...p,
|
|
424
432
|
...s
|
|
425
|
-
}
|
|
433
|
+
};
|
|
434
|
+
if (l.byteLength < 8) throw Error("parquet file is too short");
|
|
426
435
|
if (l.getUint32(l.byteLength - 4, !0) !== 827474256) throw Error("parquet file invalid (footer != PAR1)");
|
|
427
|
-
let
|
|
428
|
-
if (
|
|
429
|
-
let
|
|
436
|
+
let d = l.byteLength - 8, f = l.getUint32(d, !0);
|
|
437
|
+
if (f > l.byteLength - 8) throw Error(`parquet metadata length ${f} exceeds available buffer ${l.byteLength - 8}`);
|
|
438
|
+
let m = S({
|
|
430
439
|
view: l,
|
|
431
|
-
offset:
|
|
432
|
-
}),
|
|
440
|
+
offset: d - f
|
|
441
|
+
}), h = m.field_1, g = m.field_2.map((t) => ({
|
|
433
442
|
type: e[t.field_1],
|
|
434
443
|
type_length: t.field_2,
|
|
435
444
|
repetition_type: n[t.field_3],
|
|
@@ -440,7 +449,7 @@ function xe(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
|
440
449
|
precision: t.field_8,
|
|
441
450
|
field_id: t.field_9,
|
|
442
451
|
logical_type: Ce(t.field_10)
|
|
443
|
-
})),
|
|
452
|
+
})), _ = g.filter((e) => e.type), v = m.field_3, y = m.field_4.map((n) => ({
|
|
444
453
|
columns: n.field_1.map((n, r) => ({
|
|
445
454
|
file_path: D(n.field_1),
|
|
446
455
|
file_offset: n.field_2,
|
|
@@ -459,7 +468,7 @@ function xe(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
|
459
468
|
data_page_offset: n.field_3.field_9,
|
|
460
469
|
index_page_offset: n.field_3.field_10,
|
|
461
470
|
dictionary_page_offset: n.field_3.field_11,
|
|
462
|
-
statistics: Te(n.field_3.field_12,
|
|
471
|
+
statistics: Te(n.field_3.field_12, _[r], u),
|
|
463
472
|
encoding_stats: n.field_3.field_13?.map((e) => ({
|
|
464
473
|
page_type: a[e.field_1],
|
|
465
474
|
encoding: t[e.field_2],
|
|
@@ -503,18 +512,18 @@ function xe(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
|
503
512
|
file_offset: n.field_5,
|
|
504
513
|
total_compressed_size: n.field_6,
|
|
505
514
|
ordinal: n.field_7
|
|
506
|
-
})),
|
|
515
|
+
})), b = m.field_5?.map((e) => ({
|
|
507
516
|
key: D(e.field_1),
|
|
508
517
|
value: D(e.field_2)
|
|
509
|
-
})),
|
|
510
|
-
return c && _e(
|
|
511
|
-
version:
|
|
512
|
-
schema:
|
|
513
|
-
num_rows:
|
|
514
|
-
row_groups:
|
|
515
|
-
key_value_metadata:
|
|
516
|
-
created_by:
|
|
517
|
-
metadata_length:
|
|
518
|
+
})), ee = D(m.field_6);
|
|
519
|
+
return c && _e(g, b), {
|
|
520
|
+
version: h,
|
|
521
|
+
schema: g,
|
|
522
|
+
num_rows: v,
|
|
523
|
+
row_groups: y,
|
|
524
|
+
key_value_metadata: b,
|
|
525
|
+
created_by: ee,
|
|
526
|
+
metadata_length: f
|
|
518
527
|
};
|
|
519
528
|
}
|
|
520
529
|
function Se({ schema: e }) {
|
|
@@ -573,7 +582,18 @@ function O(e, t, n) {
|
|
|
573
582
|
if (r === "BOOLEAN") return e[0] === 1;
|
|
574
583
|
if (r === "BYTE_ARRAY") return n.stringFromBytes(e);
|
|
575
584
|
let o = new DataView(e.buffer, e.byteOffset, e.byteLength);
|
|
576
|
-
|
|
585
|
+
if (r === "FLOAT" && o.byteLength === 4) return o.getFloat32(0, !0);
|
|
586
|
+
if (r === "DOUBLE" && o.byteLength === 8) return o.getFloat64(0, !0);
|
|
587
|
+
if (r === "INT32" && i === "DECIMAL" && o.byteLength === 4) return o.getInt32(0, !0) * 10 ** -(t.scale || 0);
|
|
588
|
+
if (r === "INT64" && i === "DECIMAL" && o.byteLength === 8) return Number(o.getBigInt64(0, !0)) * 10 ** -(t.scale || 0);
|
|
589
|
+
if (r === "INT32" && i === "DATE") return n.dateFromDays(o.getInt32(0, !0));
|
|
590
|
+
if (r === "INT64" && i === "TIMESTAMP_MILLIS") return n.timestampFromMilliseconds(o.getBigInt64(0, !0));
|
|
591
|
+
if (r === "INT64" && i === "TIMESTAMP_MICROS") return n.timestampFromMicroseconds(o.getBigInt64(0, !0));
|
|
592
|
+
if (r === "INT64" && a?.type === "TIMESTAMP" && a?.unit === "NANOS") return n.timestampFromNanoseconds(o.getBigInt64(0, !0));
|
|
593
|
+
if (r === "INT64" && a?.type === "TIMESTAMP" && a?.unit === "MICROS") return n.timestampFromMicroseconds(o.getBigInt64(0, !0));
|
|
594
|
+
if (r === "INT64" && a?.type === "TIMESTAMP") return n.timestampFromMilliseconds(o.getBigInt64(0, !0));
|
|
595
|
+
let s = i?.startsWith("UINT_") || a?.type === "INTEGER" && !a.isSigned;
|
|
596
|
+
return r === "INT32" && s && o.byteLength === 4 ? o.getUint32(0, !0) : r === "INT64" && s && o.byteLength === 8 ? o.getBigUint64(0, !0) : r === "INT32" && o.byteLength === 4 ? o.getInt32(0, !0) : r === "INT64" && o.byteLength === 8 ? o.getBigInt64(0, !0) : i === "DECIMAL" ? g(e) * 10 ** -(t.scale || 0) : a?.type === "FLOAT16" ? v(e) : a?.type === "UUID" ? n.uuidFromBytes(e) : e;
|
|
577
597
|
}
|
|
578
598
|
//#endregion
|
|
579
599
|
//#region ../../node_modules/hyparquet/src/xxhash.js
|
|
@@ -668,6 +688,7 @@ function I(e, t, n = !0) {
|
|
|
668
688
|
for (let n = 0; n < e.length; n++) if (e[n] !== t[n]) return !1;
|
|
669
689
|
return !0;
|
|
670
690
|
}
|
|
691
|
+
if (e instanceof Date || t instanceof Date) return e instanceof Date && t instanceof Date && e.getTime() === t.getTime();
|
|
671
692
|
if (Array.isArray(e) && Array.isArray(t)) {
|
|
672
693
|
if (e.length !== t.length) return !1;
|
|
673
694
|
for (let r = 0; r < e.length; r++) if (!I(e[r], t[r], n)) return !1;
|
|
@@ -687,9 +708,19 @@ function Ie(e) {
|
|
|
687
708
|
}
|
|
688
709
|
//#endregion
|
|
689
710
|
//#region ../../node_modules/hyparquet/src/filter.js
|
|
690
|
-
|
|
711
|
+
var Le = new TextEncoder();
|
|
712
|
+
function L(e, t, n = !0) {
|
|
713
|
+
return "$and" in t && Array.isArray(t.$and) ? t.$and.every((t) => L(e, t, n)) : "$or" in t && Array.isArray(t.$or) ? t.$or.some((t) => L(e, t, n)) : "$nor" in t && Array.isArray(t.$nor) ? !t.$nor.some((t) => L(e, t, n)) : Object.entries(t).every(([t, r]) => {
|
|
714
|
+
let i = Ve(e, t);
|
|
715
|
+
return typeof r != "object" || !r || Array.isArray(r) ? I(i, r, n) : Object.entries(r || {}).every(([e, t]) => e === "$gt" ? i != null && i > t : e === "$gte" ? i != null && i >= t : e === "$lt" ? i != null && i < t : e === "$lte" ? i != null && i <= t : e === "$eq" ? I(i, t, n) : e === "$ne" ? !I(i, t, n) : e === "$in" ? Array.isArray(t) && Re(i, t, n) : e === "$nin" ? Array.isArray(t) && !Re(i, t, n) : e !== "$not" || !L({ value: i }, { value: t }, n));
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
function Re(e, t, n) {
|
|
719
|
+
return t.some((t) => I(e, t, n) || Array.isArray(e) && e.some((e) => I(e, t, n)));
|
|
720
|
+
}
|
|
721
|
+
function R({ rowGroup: e, physicalColumns: t, filter: n, strict: r = !0, bloomFilters: i, schemaElements: a }) {
|
|
691
722
|
if (!n) return !1;
|
|
692
|
-
if ("$and" in n && Array.isArray(n.$and)) return n.$and.some((n) =>
|
|
723
|
+
if ("$and" in n && Array.isArray(n.$and)) return n.$and.some((n) => R({
|
|
693
724
|
rowGroup: e,
|
|
694
725
|
physicalColumns: t,
|
|
695
726
|
filter: n,
|
|
@@ -697,7 +728,7 @@ function L({ rowGroup: e, physicalColumns: t, filter: n, strict: r = !0, bloomFi
|
|
|
697
728
|
bloomFilters: i,
|
|
698
729
|
schemaElements: a
|
|
699
730
|
}));
|
|
700
|
-
if ("$or" in n && Array.isArray(n.$or)) return n.$or.every((n) =>
|
|
731
|
+
if ("$or" in n && Array.isArray(n.$or)) return n.$or.every((n) => R({
|
|
701
732
|
rowGroup: e,
|
|
702
733
|
physicalColumns: t,
|
|
703
734
|
filter: n,
|
|
@@ -709,104 +740,228 @@ function L({ rowGroup: e, physicalColumns: t, filter: n, strict: r = !0, bloomFi
|
|
|
709
740
|
for (let [o, s] of Object.entries(n)) {
|
|
710
741
|
let n = t.indexOf(o);
|
|
711
742
|
if (n === -1) continue;
|
|
712
|
-
let { min: c, max: l, min_value: u, max_value: d } = e.columns[n].meta_data?.statistics || {},
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
if (
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
break;
|
|
727
|
-
}
|
|
743
|
+
let { min: c, max: l, min_value: u, max_value: d, null_count: f } = e.columns[n].meta_data?.statistics || {}, p = u === void 0 ? c : u, m = d === void 0 ? l : d, h = p !== void 0 && m !== void 0, g = i?.[o], _ = a?.[o], v = L({ value: null }, { value: s }, r) && (f === void 0 || f > 0);
|
|
744
|
+
if (h && !v && ze(s, p, m, r, _)) return !0;
|
|
745
|
+
for (let [e, t] of Object.entries(s || {})) if (g && _) {
|
|
746
|
+
if (e === "$eq") {
|
|
747
|
+
let e = Pe(t, _);
|
|
748
|
+
if (e !== void 0 && !Ne(g.blocks, e)) return !0;
|
|
749
|
+
}
|
|
750
|
+
if (e === "$in" && Array.isArray(t) && t.length > 0) {
|
|
751
|
+
let e = !0;
|
|
752
|
+
for (let n of t) {
|
|
753
|
+
let t = Pe(n, _);
|
|
754
|
+
if (t === void 0 || Ne(g.blocks, t)) {
|
|
755
|
+
e = !1;
|
|
756
|
+
break;
|
|
728
757
|
}
|
|
729
|
-
if (e) return !0;
|
|
730
758
|
}
|
|
759
|
+
if (e) return !0;
|
|
731
760
|
}
|
|
732
761
|
}
|
|
733
762
|
}
|
|
734
763
|
return !1;
|
|
735
764
|
}
|
|
765
|
+
function ze(e, t, n, r, i) {
|
|
766
|
+
if (t === void 0 || n === void 0) return !1;
|
|
767
|
+
let a = i?.type === "FLOAT" || i?.type === "DOUBLE" || i?.logical_type?.type === "FLOAT16";
|
|
768
|
+
for (let [o, s] of Object.entries(e || {})) {
|
|
769
|
+
let e = z(t, s, r, i), c = z(n, s, r, i), l = !(t instanceof Uint8Array || n instanceof Uint8Array) && (i?.type !== "BYTE_ARRAY" || typeof s == "string" && [...s].every((e) => e.charCodeAt(0) <= 127));
|
|
770
|
+
if (o === "$gt" && l && c !== void 0 && c <= 0 || o === "$gte" && l && c !== void 0 && c < 0 || o === "$lt" && l && e !== void 0 && e >= 0 || o === "$lte" && l && e !== void 0 && e > 0) return !0;
|
|
771
|
+
if (o === "$eq") {
|
|
772
|
+
let e = z(s, t, r, i), a = z(s, n, r, i);
|
|
773
|
+
if (e !== void 0 && e < 0 || a !== void 0 && a > 0) return !0;
|
|
774
|
+
}
|
|
775
|
+
if (o === "$ne" && !a && I(t, n, r) && I(t, s, r) || o === "$in" && Array.isArray(s) && s.every((e) => {
|
|
776
|
+
let a = z(e, t, r, i), o = z(e, n, r, i);
|
|
777
|
+
return a !== void 0 && a < 0 || o !== void 0 && o > 0;
|
|
778
|
+
}) || o === "$nin" && !a && Array.isArray(s) && I(t, n, r) && s.some((e) => I(t, e, r))) return !0;
|
|
779
|
+
}
|
|
780
|
+
return !1;
|
|
781
|
+
}
|
|
782
|
+
function z(e, t, n, r) {
|
|
783
|
+
if (r?.type === "BYTE_ARRAY") return typeof e != "string" || typeof t != "string" ? void 0 : Be(Le.encode(e), Le.encode(t));
|
|
784
|
+
if (e instanceof Uint8Array || t instanceof Uint8Array) return !(e instanceof Uint8Array) || !(t instanceof Uint8Array) ? void 0 : Be(e, t);
|
|
785
|
+
if (e < t) return -1;
|
|
786
|
+
if (e > t) return 1;
|
|
787
|
+
if (I(e, t, n)) return 0;
|
|
788
|
+
}
|
|
789
|
+
function Be(e, t) {
|
|
790
|
+
let n = Math.min(e.length, t.length);
|
|
791
|
+
for (let r = 0; r < n; r++) {
|
|
792
|
+
if (e[r] < t[r]) return -1;
|
|
793
|
+
if (e[r] > t[r]) return 1;
|
|
794
|
+
}
|
|
795
|
+
return e.length < t.length ? -1 : +(e.length > t.length);
|
|
796
|
+
}
|
|
797
|
+
function Ve(e, t) {
|
|
798
|
+
let n = e;
|
|
799
|
+
for (let e of t.split(".")) n = n?.[e];
|
|
800
|
+
return n;
|
|
801
|
+
}
|
|
802
|
+
//#endregion
|
|
803
|
+
//#region ../../node_modules/hyparquet/src/indexes.js
|
|
804
|
+
function He(e) {
|
|
805
|
+
let t = S(e);
|
|
806
|
+
return {
|
|
807
|
+
page_locations: t.field_1.map((e) => ({
|
|
808
|
+
offset: e.field_1,
|
|
809
|
+
compressed_page_size: e.field_2,
|
|
810
|
+
first_row_index: e.field_3
|
|
811
|
+
})),
|
|
812
|
+
unencoded_byte_array_data_bytes: t.field_2
|
|
813
|
+
};
|
|
814
|
+
}
|
|
736
815
|
//#endregion
|
|
737
816
|
//#region ../../node_modules/hyparquet/src/plan.js
|
|
738
|
-
var
|
|
739
|
-
function
|
|
817
|
+
var Ue = 1 << 21, We = 8192;
|
|
818
|
+
function Ge(e) {
|
|
819
|
+
let { metadata: t, rowStart: n = 0, columns: r, useOffsetIndex: i = !1 } = e;
|
|
820
|
+
if (!t) throw Error("parquetPlan requires metadata");
|
|
821
|
+
let a = [], o = [], s = [], c = Ke(e);
|
|
822
|
+
for (let e of c.groups) {
|
|
823
|
+
let t = qe({
|
|
824
|
+
...e,
|
|
825
|
+
columns: r,
|
|
826
|
+
useOffsetIndex: i
|
|
827
|
+
});
|
|
828
|
+
a.push(...t.groups), o.push(...t.fetches), s.push(...t.indexes);
|
|
829
|
+
}
|
|
830
|
+
return o.push(...s), {
|
|
831
|
+
metadata: t,
|
|
832
|
+
rowStart: n,
|
|
833
|
+
rowEnd: c.rowEnd,
|
|
834
|
+
columns: r,
|
|
835
|
+
fetches: o,
|
|
836
|
+
groups: a
|
|
837
|
+
};
|
|
838
|
+
}
|
|
839
|
+
function Ke({ metadata: e, rowStart: t = 0, rowEnd: n = Infinity, columns: r, filter: i, filterStrict: a = !0, bloomFiltersByGroup: o, schemaElements: s, pageRangesByGroup: c, pageLocationsByGroup: l }) {
|
|
740
840
|
if (!e) throw Error("parquetPlan requires metadata");
|
|
741
|
-
let
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
841
|
+
let u = Se(e), d = ee(u), f = i ? {
|
|
842
|
+
...Ze(u),
|
|
843
|
+
...s
|
|
844
|
+
} : s, p = [], m = 0;
|
|
845
|
+
for (let s = 0; s < e.row_groups.length; s++) {
|
|
846
|
+
let u = e.row_groups[s], h = Number(u.num_rows), g = m + h;
|
|
847
|
+
if (h > 0 && g > t && m < n && !R({
|
|
848
|
+
rowGroup: u,
|
|
849
|
+
physicalColumns: d,
|
|
747
850
|
filter: i,
|
|
748
851
|
strict: a,
|
|
749
|
-
bloomFilters:
|
|
750
|
-
schemaElements:
|
|
852
|
+
bloomFilters: o?.[s],
|
|
853
|
+
schemaElements: f
|
|
751
854
|
})) {
|
|
752
|
-
let i =
|
|
753
|
-
|
|
754
|
-
let
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
855
|
+
let e = Math.max(t - m, 0), i = Math.min(n - m, h), a = c?.[s], o = l?.[s], d = a ? a.map(([t, n]) => [Math.max(t, e), Math.min(n, i)]).filter(([e, t]) => e < t) : [[e, i]];
|
|
856
|
+
d.length > 1 && (d = u.columns.every((e) => {
|
|
857
|
+
let t = e.meta_data?.path_in_schema[0], n = e.meta_data?.path_in_schema.join(".");
|
|
858
|
+
return r && t && !r.includes(t) ? !0 : !!(e.offset_index_offset && e.offset_index_length) || !!(n && o?.[n]);
|
|
859
|
+
}) ? Je(d, u, r, o) : [[d[0][0], d[d.length - 1][1]]]), d.length && p.push({
|
|
860
|
+
rowGroup: u,
|
|
861
|
+
groupIndex: s,
|
|
862
|
+
groupStart: m,
|
|
863
|
+
groupRows: h,
|
|
864
|
+
ranges: d,
|
|
865
|
+
pageRanges: a,
|
|
866
|
+
pageLocations: o
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
m = g;
|
|
870
|
+
}
|
|
871
|
+
return {
|
|
872
|
+
groups: p,
|
|
873
|
+
rowEnd: isFinite(n) ? n : m
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
function qe({ rowGroup: e, groupStart: t, groupRows: n, ranges: r, columns: i, useOffsetIndex: a = !1, pageRanges: o, pageLocations: s }) {
|
|
877
|
+
let c = [], l = [], u = [], d = r.length > 1 || r[0][0] > 0 || r[0][1] < n;
|
|
878
|
+
for (let t of e.columns) {
|
|
879
|
+
let e = t.meta_data;
|
|
880
|
+
if (t.file_path) throw Error("parquet file_path not supported");
|
|
881
|
+
if (!e) throw Error("parquet column metadata is undefined");
|
|
882
|
+
if (i && !i.includes(e.path_in_schema[0])) continue;
|
|
883
|
+
let n = e.dictionary_page_offset || e.data_page_offset, r = Number(n), l = Number(n + e.total_compressed_size), u = s?.[e.path_in_schema.join(".")];
|
|
884
|
+
if (u && d) c.push({
|
|
885
|
+
columnMetadata: e,
|
|
886
|
+
pageLocations: u,
|
|
887
|
+
range: {
|
|
888
|
+
startByte: r,
|
|
889
|
+
endByte: l
|
|
780
890
|
}
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
891
|
+
});
|
|
892
|
+
else if ((a || o) && t.offset_index_offset && t.offset_index_length && d) {
|
|
893
|
+
let r = Number(t.offset_index_offset);
|
|
894
|
+
c.push({
|
|
895
|
+
columnMetadata: e,
|
|
896
|
+
offsetIndex: {
|
|
897
|
+
startByte: r,
|
|
898
|
+
endByte: r + t.offset_index_length
|
|
899
|
+
},
|
|
900
|
+
range: {
|
|
901
|
+
startByte: Number(n),
|
|
902
|
+
endByte: l
|
|
903
|
+
}
|
|
789
904
|
});
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
905
|
+
} else c.push({
|
|
906
|
+
columnMetadata: e,
|
|
907
|
+
range: {
|
|
908
|
+
startByte: r,
|
|
909
|
+
endByte: l
|
|
795
910
|
}
|
|
796
|
-
|
|
797
|
-
}
|
|
798
|
-
p = g, m++;
|
|
911
|
+
});
|
|
799
912
|
}
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
913
|
+
let f = [], p;
|
|
914
|
+
for (let e of c) "pageLocations" in e || ("offsetIndex" in e ? u.push(e.offsetIndex) : i ? f.push(e.range) : p && e.range.endByte - p.startByte <= Ue ? p.endByte = e.range.endByte : (p && l.push(p), p = { ...e.range }));
|
|
915
|
+
return p && l.push(p), l.push(...Xe(f, We, Ue)), {
|
|
916
|
+
groups: r.map(([r, i]) => ({
|
|
917
|
+
chunks: c,
|
|
918
|
+
rowGroup: e,
|
|
919
|
+
groupStart: t,
|
|
920
|
+
groupRows: n,
|
|
921
|
+
selectStart: r,
|
|
922
|
+
selectEnd: i
|
|
923
|
+
})),
|
|
924
|
+
fetches: l,
|
|
925
|
+
indexes: u
|
|
807
926
|
};
|
|
808
927
|
}
|
|
809
|
-
function
|
|
928
|
+
function Je(e, t, n, r) {
|
|
929
|
+
let i = t.columns.filter((e) => !n || n.includes(e.meta_data?.path_in_schema[0] || "")).map((e) => r?.[e.meta_data?.path_in_schema.join(".") || ""]), a = [];
|
|
930
|
+
for (let n of e) {
|
|
931
|
+
let e = a[a.length - 1], r = e && i.some((r) => {
|
|
932
|
+
if (!r) return !0;
|
|
933
|
+
let i = Ye(e, r, Number(t.num_rows)), a = Ye(n, r, Number(t.num_rows));
|
|
934
|
+
return i[0] <= a[1] && a[0] <= i[1];
|
|
935
|
+
});
|
|
936
|
+
e && r ? e[1] = n[1] : a.push([...n]);
|
|
937
|
+
}
|
|
938
|
+
return a;
|
|
939
|
+
}
|
|
940
|
+
function Ye([e, t], n, r) {
|
|
941
|
+
let i = Infinity, a = -Infinity;
|
|
942
|
+
for (let o = 0; o < n.length; o++) {
|
|
943
|
+
let s = Number(n[o].first_row_index);
|
|
944
|
+
(o + 1 < n.length ? Number(n[o + 1].first_row_index) : r) > e && s < t && (i = Math.min(i, o), a = o);
|
|
945
|
+
}
|
|
946
|
+
return [i, a];
|
|
947
|
+
}
|
|
948
|
+
function Xe(e, t = 0, n = Infinity) {
|
|
949
|
+
let r = e.map((e) => ({ ...e })).sort((e, t) => e.startByte - t.startByte || e.endByte - t.endByte), i = [];
|
|
950
|
+
for (let e of r) {
|
|
951
|
+
let r = i[i.length - 1];
|
|
952
|
+
r && e.startByte <= r.endByte + t && Math.max(r.endByte, e.endByte) - r.startByte <= n ? r.endByte = Math.max(r.endByte, e.endByte) : i.push(e);
|
|
953
|
+
}
|
|
954
|
+
return i;
|
|
955
|
+
}
|
|
956
|
+
function Ze(e) {
|
|
957
|
+
let t = {};
|
|
958
|
+
function n(e) {
|
|
959
|
+
if (e.children.length) for (let t of e.children) n(t);
|
|
960
|
+
else t[e.path.join(".")] = e.element;
|
|
961
|
+
}
|
|
962
|
+
return n(e), t;
|
|
963
|
+
}
|
|
964
|
+
function Qe(e, { fetches: t }) {
|
|
810
965
|
let n = t.map(({ startByte: t, endByte: n }) => e.slice(t, n));
|
|
811
966
|
return {
|
|
812
967
|
byteLength: e.byteLength,
|
|
@@ -816,18 +971,19 @@ function ze(e, { fetches: t }) {
|
|
|
816
971
|
if (t[a].startByte !== r || t[a].endByte !== i) {
|
|
817
972
|
let e = r - t[a].startByte, o = i - t[a].startByte;
|
|
818
973
|
return n[a] instanceof Promise ? n[a].then((t) => t.slice(e, o)) : n[a].slice(e, o);
|
|
819
|
-
}
|
|
974
|
+
}
|
|
975
|
+
return n[a];
|
|
820
976
|
}
|
|
821
977
|
};
|
|
822
978
|
}
|
|
823
979
|
//#endregion
|
|
824
980
|
//#region ../../node_modules/hyparquet/src/variant.js
|
|
825
|
-
var
|
|
826
|
-
function
|
|
827
|
-
if (Array.isArray(e)) return e.map((e) =>
|
|
981
|
+
var B = new TextDecoder(), $e = /* @__PURE__ */ new WeakMap();
|
|
982
|
+
function et(e, t = p) {
|
|
983
|
+
if (Array.isArray(e)) return e.map((e) => et(e, t));
|
|
828
984
|
if (typeof e != "object") return e;
|
|
829
985
|
if ("metadata" in e) {
|
|
830
|
-
let n =
|
|
986
|
+
let n = tt(e.metadata), r = e.typed_value && V(e.typed_value, n, t), i = e.value && W(H(e.value), n, t);
|
|
831
987
|
return r && i ? {
|
|
832
988
|
...i,
|
|
833
989
|
...r
|
|
@@ -853,9 +1009,9 @@ function H(e) {
|
|
|
853
1009
|
offset: 0
|
|
854
1010
|
};
|
|
855
1011
|
}
|
|
856
|
-
function
|
|
857
|
-
let t =
|
|
858
|
-
t || (t = /* @__PURE__ */ new Map(),
|
|
1012
|
+
function tt(e) {
|
|
1013
|
+
let t = $e.get(e.buffer);
|
|
1014
|
+
t || (t = /* @__PURE__ */ new Map(), $e.set(e.buffer, t));
|
|
859
1015
|
let n = `${e.byteOffset}:${e.byteLength}`, r = t.get(n);
|
|
860
1016
|
if (r) return r;
|
|
861
1017
|
let i = H(e), a = i.view.getUint8(i.offset++), o = a & 15;
|
|
@@ -865,7 +1021,7 @@ function Be(e) {
|
|
|
865
1021
|
let d = i.offset, f = Array(l);
|
|
866
1022
|
for (let t = 0; t < l; t++) {
|
|
867
1023
|
let n = u[t], r = u[t + 1], i = new Uint8Array(e.buffer, e.byteOffset + d + n, r - n);
|
|
868
|
-
f[t] =
|
|
1024
|
+
f[t] = B.decode(i);
|
|
869
1025
|
}
|
|
870
1026
|
let p = {
|
|
871
1027
|
dictionary: f,
|
|
@@ -880,13 +1036,13 @@ function U(e, t) {
|
|
|
880
1036
|
}
|
|
881
1037
|
function W(e, t, n) {
|
|
882
1038
|
let r = e.view.getUint8(e.offset++), i = r & 3, a = r >> 2;
|
|
883
|
-
if (i === 0) return
|
|
884
|
-
if (i === 2) return
|
|
885
|
-
if (i === 3) return
|
|
1039
|
+
if (i === 0) return nt(e, a, n);
|
|
1040
|
+
if (i === 2) return rt(e, a, t, n);
|
|
1041
|
+
if (i === 3) return it(e, a, t, n);
|
|
886
1042
|
let o = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, a);
|
|
887
|
-
return e.offset += a,
|
|
1043
|
+
return e.offset += a, B.decode(o);
|
|
888
1044
|
}
|
|
889
|
-
function
|
|
1045
|
+
function nt(e, t, n) {
|
|
890
1046
|
switch (t) {
|
|
891
1047
|
case 0: return null;
|
|
892
1048
|
case 1: return !0;
|
|
@@ -927,10 +1083,10 @@ function Ve(e, t, n) {
|
|
|
927
1083
|
let t = e.view.getFloat32(e.offset, !0);
|
|
928
1084
|
return e.offset += 4, t;
|
|
929
1085
|
}
|
|
930
|
-
case 15: return
|
|
1086
|
+
case 15: return at(e);
|
|
931
1087
|
case 16: {
|
|
932
|
-
let t =
|
|
933
|
-
return
|
|
1088
|
+
let t = at(e);
|
|
1089
|
+
return B.decode(t);
|
|
934
1090
|
}
|
|
935
1091
|
case 17: {
|
|
936
1092
|
let t = e.view.getBigInt64(e.offset, !0);
|
|
@@ -950,7 +1106,7 @@ function Ve(e, t, n) {
|
|
|
950
1106
|
default: throw Error(`parquet unsupported variant primitive type: ${t}`);
|
|
951
1107
|
}
|
|
952
1108
|
}
|
|
953
|
-
function
|
|
1109
|
+
function rt(e, t, n, r) {
|
|
954
1110
|
let i = (t & 3) + 1, a = (t >> 2 & 3) + 1, o = t >> 4 & 1 ? U(e, 4) : e.view.getUint8(e.offset++), s = Array(o);
|
|
955
1111
|
for (let t = 0; t < o; t++) s[t] = U(e, a);
|
|
956
1112
|
let c = Array(o + 1);
|
|
@@ -965,14 +1121,17 @@ function He(e, t, n, r) {
|
|
|
965
1121
|
}
|
|
966
1122
|
return e.offset += c[c.length - 1], l;
|
|
967
1123
|
}
|
|
968
|
-
function
|
|
1124
|
+
function it(e, t, n, r) {
|
|
969
1125
|
let i = t & 3, a = t >> 2 & 1, o = i + 1, s = U(e, a ? 4 : 1), c = Array(s + 1);
|
|
970
1126
|
for (let t = 0; t < c.length; t++) c[t] = U(e, o);
|
|
971
1127
|
let l = e.offset, u = Array(s);
|
|
972
|
-
for (let t = 0; t < s; t++)
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
1128
|
+
for (let t = 0; t < s; t++) {
|
|
1129
|
+
let i = {
|
|
1130
|
+
view: e.view,
|
|
1131
|
+
offset: l + c[t]
|
|
1132
|
+
};
|
|
1133
|
+
u[t] = W(i, n, r);
|
|
1134
|
+
}
|
|
976
1135
|
return e.offset = l + c[c.length - 1], u;
|
|
977
1136
|
}
|
|
978
1137
|
function G(e, t) {
|
|
@@ -987,7 +1146,7 @@ function G(e, t) {
|
|
|
987
1146
|
}
|
|
988
1147
|
return Number(r) * 10 ** -n;
|
|
989
1148
|
}
|
|
990
|
-
function
|
|
1149
|
+
function at(e) {
|
|
991
1150
|
let t = e.view.getUint32(e.offset, !0);
|
|
992
1151
|
e.offset += 4;
|
|
993
1152
|
let n = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
@@ -995,7 +1154,7 @@ function We(e) {
|
|
|
995
1154
|
}
|
|
996
1155
|
//#endregion
|
|
997
1156
|
//#region ../../node_modules/hyparquet/src/assemble.js
|
|
998
|
-
function
|
|
1157
|
+
function ot(e, t, n, r, i) {
|
|
999
1158
|
let a = x(i);
|
|
1000
1159
|
if (!t?.length && !n.length) {
|
|
1001
1160
|
if (!a || !r.length) return r;
|
|
@@ -1038,7 +1197,7 @@ function K(e, t, n, r = 0) {
|
|
|
1038
1197
|
if (!c) throw Error("parquet map column missing keys");
|
|
1039
1198
|
if (!l) throw Error("parquet map column missing values");
|
|
1040
1199
|
if (c.length !== l.length) throw Error("parquet map column key/value length mismatch");
|
|
1041
|
-
let u =
|
|
1200
|
+
let u = st(c, l, o);
|
|
1042
1201
|
a && q(u, r), e.delete(`${i}.${s}.key`), e.delete(`${i}.${s}.value`), e.set(i, u);
|
|
1043
1202
|
return;
|
|
1044
1203
|
}
|
|
@@ -1051,16 +1210,16 @@ function K(e, t, n, r = 0) {
|
|
|
1051
1210
|
s[r.element.name] = t;
|
|
1052
1211
|
}
|
|
1053
1212
|
for (let n of t.children) e.delete(n.path.join("."));
|
|
1054
|
-
let c =
|
|
1055
|
-
t.element.logical_type?.type === "VARIANT" && (c =
|
|
1213
|
+
let c = ct(s, o);
|
|
1214
|
+
t.element.logical_type?.type === "VARIANT" && (c = et(c, n)), a && q(c, r), e.set(i, c);
|
|
1056
1215
|
}
|
|
1057
1216
|
}
|
|
1058
1217
|
function q(e, t) {
|
|
1059
1218
|
for (let n = 0; n < e.length; n++) t ? q(e[n], t - 1) : e[n] = e[n][0];
|
|
1060
1219
|
}
|
|
1061
|
-
function
|
|
1220
|
+
function st(e, t, n) {
|
|
1062
1221
|
let r = [];
|
|
1063
|
-
for (let i = 0; i < e.length; i++) if (n) r.push(
|
|
1222
|
+
for (let i = 0; i < e.length; i++) if (n) r.push(st(e[i], t[i], n - 1));
|
|
1064
1223
|
else if (e[i]) {
|
|
1065
1224
|
let n = {};
|
|
1066
1225
|
for (let r = 0; r < e[i].length; r++) {
|
|
@@ -1071,7 +1230,7 @@ function Ke(e, t, n) {
|
|
|
1071
1230
|
} else r.push(void 0);
|
|
1072
1231
|
return r;
|
|
1073
1232
|
}
|
|
1074
|
-
function
|
|
1233
|
+
function ct(e, t) {
|
|
1075
1234
|
let n = Object.keys(e), r = e[n[0]]?.length, i = [];
|
|
1076
1235
|
for (let a = 0; a < r; a++) {
|
|
1077
1236
|
let o = {};
|
|
@@ -1079,42 +1238,69 @@ function qe(e, t) {
|
|
|
1079
1238
|
if (e[t].length !== r) throw Error("parquet struct parsing error");
|
|
1080
1239
|
o[t] = e[t][a];
|
|
1081
1240
|
}
|
|
1082
|
-
t ? i.push(
|
|
1241
|
+
t ? i.push(ct(o, t - 1)) : i.push(o);
|
|
1083
1242
|
}
|
|
1084
1243
|
return i;
|
|
1085
1244
|
}
|
|
1086
1245
|
//#endregion
|
|
1087
1246
|
//#region ../../node_modules/hyparquet/src/delta.js
|
|
1088
1247
|
function J(e, t, n) {
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
let
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1248
|
+
if (n instanceof Int32Array) {
|
|
1249
|
+
lt(e, t, n);
|
|
1250
|
+
return;
|
|
1251
|
+
}
|
|
1252
|
+
let r = w(e), i = w(e);
|
|
1253
|
+
w(e);
|
|
1254
|
+
let a = E(e), o = 0;
|
|
1255
|
+
n[o++] = a;
|
|
1256
|
+
let s = r / i;
|
|
1257
|
+
for (; o < t;) {
|
|
1258
|
+
let r = E(e), c = new Uint8Array(i);
|
|
1259
|
+
for (let t = 0; t < i; t++) c[t] = e.view.getUint8(e.offset++);
|
|
1260
|
+
for (let l = 0; l < i && o < t; l++) {
|
|
1261
|
+
let i = c[l];
|
|
1262
|
+
if (i) {
|
|
1263
|
+
let c = 0, l = s, u = (1n << BigInt(i)) - 1n;
|
|
1264
|
+
for (; l && o < t;) {
|
|
1265
|
+
let t = BigInt(e.view.getUint32(e.offset, !0) >>> c) & u;
|
|
1266
|
+
for (c += i; c >= 32;) c -= 32, e.offset += 4, c && (t |= BigInt(e.view.getUint32(e.offset, !0)) << BigInt(i - c) & u);
|
|
1267
|
+
let s = r + t;
|
|
1268
|
+
a += s, n[o++] = a, l--;
|
|
1106
1269
|
}
|
|
1107
|
-
|
|
1108
|
-
} else for (let e = 0; e <
|
|
1270
|
+
l && (e.offset += Math.ceil((l * i + c) / 8));
|
|
1271
|
+
} else for (let e = 0; e < s && o < t; e++) a += r, n[o++] = a;
|
|
1109
1272
|
}
|
|
1110
1273
|
}
|
|
1111
1274
|
}
|
|
1112
|
-
function
|
|
1275
|
+
function lt(e, t, n) {
|
|
1276
|
+
let r = w(e), i = w(e);
|
|
1277
|
+
w(e);
|
|
1278
|
+
let a = T(e), o = 0;
|
|
1279
|
+
n[o++] = a;
|
|
1280
|
+
let s = r / i;
|
|
1281
|
+
for (; o < t;) {
|
|
1282
|
+
let r = T(e), c = e.offset;
|
|
1283
|
+
e.offset += i;
|
|
1284
|
+
for (let l = 0; l < i && o < t; l++) {
|
|
1285
|
+
let i = e.view.getUint8(c + l), u = e.offset + s * i / 8, d = 0;
|
|
1286
|
+
for (let c = 0; c < s && o < t; c++) {
|
|
1287
|
+
let t = 0, s = 0;
|
|
1288
|
+
for (; s < i;) {
|
|
1289
|
+
let n = Math.min(8 - d, i - s);
|
|
1290
|
+
t |= (e.view.getUint8(e.offset) >>> d & (1 << n) - 1) << s, s += n, d += n, d === 8 && (d = 0, e.offset++);
|
|
1291
|
+
}
|
|
1292
|
+
a = a + r + t | 0, n[o++] = a;
|
|
1293
|
+
}
|
|
1294
|
+
e.offset = u;
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
function ut(e, t, n) {
|
|
1113
1299
|
let r = new Int32Array(t);
|
|
1114
1300
|
J(e, t, r);
|
|
1115
1301
|
for (let i = 0; i < t; i++) n[i] = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, r[i]), e.offset += r[i];
|
|
1116
1302
|
}
|
|
1117
|
-
function
|
|
1303
|
+
function dt(e, t, n) {
|
|
1118
1304
|
let r = new Int32Array(t);
|
|
1119
1305
|
J(e, t, r);
|
|
1120
1306
|
let i = new Int32Array(t);
|
|
@@ -1130,21 +1316,21 @@ function Y(e, t, n, r) {
|
|
|
1130
1316
|
r === void 0 && (r = e.view.getUint32(e.offset, !0), e.offset += 4);
|
|
1131
1317
|
let i = e.offset, a = 0;
|
|
1132
1318
|
for (; a < n.length;) {
|
|
1133
|
-
let r =
|
|
1134
|
-
if (r & 1) a =
|
|
1319
|
+
let r = w(e);
|
|
1320
|
+
if (r & 1) a = pt(e, r, t, n, a);
|
|
1135
1321
|
else {
|
|
1136
1322
|
let i = r >>> 1;
|
|
1137
|
-
|
|
1323
|
+
ft(e, i, t, n, a), a += i;
|
|
1138
1324
|
}
|
|
1139
1325
|
}
|
|
1140
1326
|
e.offset = i + r;
|
|
1141
1327
|
}
|
|
1142
|
-
function
|
|
1328
|
+
function ft(e, t, n, r, i) {
|
|
1143
1329
|
let a = n + 7 >> 3, o = 0;
|
|
1144
1330
|
for (let t = 0; t < a; t++) o |= e.view.getUint8(e.offset++) << (t << 3);
|
|
1145
1331
|
for (let e = 0; e < t; e++) r[i + e] = o;
|
|
1146
1332
|
}
|
|
1147
|
-
function
|
|
1333
|
+
function pt(e, t, n, r, i) {
|
|
1148
1334
|
let a = t >> 1 << 3, o = (1 << n) - 1, s = 0;
|
|
1149
1335
|
if (e.offset < e.view.byteLength) s = e.view.getUint8(e.offset++);
|
|
1150
1336
|
else if (o) throw Error(`parquet bitpack offset ${e.offset} out of range`);
|
|
@@ -1152,8 +1338,8 @@ function Ze(e, t, n, r, i) {
|
|
|
1152
1338
|
for (; a;) l > 8 ? (l -= 8, c -= 8, s >>>= 8) : c - l < n ? (s |= e.view.getUint8(e.offset) << c, e.offset++, c += 8) : (i < r.length && (r[i++] = s >> l & o), a--, l += n);
|
|
1153
1339
|
return i;
|
|
1154
1340
|
}
|
|
1155
|
-
function
|
|
1156
|
-
let i =
|
|
1341
|
+
function mt(e, t, n, r) {
|
|
1342
|
+
let i = ht(n, r), a = new Uint8Array(t * i);
|
|
1157
1343
|
for (let n = 0; n < i; n++) for (let r = 0; r < t; r++) a[r * i + n] = e.view.getUint8(e.offset++);
|
|
1158
1344
|
if (n === "FLOAT") return new Float32Array(a.buffer);
|
|
1159
1345
|
if (n === "DOUBLE") return new Float64Array(a.buffer);
|
|
@@ -1166,7 +1352,7 @@ function Qe(e, t, n, r) {
|
|
|
1166
1352
|
}
|
|
1167
1353
|
throw Error(`parquet byte_stream_split unsupported type: ${n}`);
|
|
1168
1354
|
}
|
|
1169
|
-
function
|
|
1355
|
+
function ht(e, t) {
|
|
1170
1356
|
switch (e) {
|
|
1171
1357
|
case "INT32":
|
|
1172
1358
|
case "FLOAT": return 4;
|
|
@@ -1182,35 +1368,36 @@ function $e(e, t) {
|
|
|
1182
1368
|
//#region ../../node_modules/hyparquet/src/plain.js
|
|
1183
1369
|
function X(e, t, n, r) {
|
|
1184
1370
|
if (n === 0) return [];
|
|
1185
|
-
if (t === "BOOLEAN") return
|
|
1186
|
-
if (t === "INT32") return
|
|
1187
|
-
if (t === "INT64") return
|
|
1188
|
-
if (t === "INT96") return
|
|
1189
|
-
if (t === "FLOAT") return
|
|
1190
|
-
if (t === "DOUBLE") return
|
|
1191
|
-
if (t === "BYTE_ARRAY") return
|
|
1371
|
+
if (t === "BOOLEAN") return gt(e, n);
|
|
1372
|
+
if (t === "INT32") return _t(e, n);
|
|
1373
|
+
if (t === "INT64") return vt(e, n);
|
|
1374
|
+
if (t === "INT96") return yt(e, n);
|
|
1375
|
+
if (t === "FLOAT") return bt(e, n);
|
|
1376
|
+
if (t === "DOUBLE") return xt(e, n);
|
|
1377
|
+
if (t === "BYTE_ARRAY") return St(e, n);
|
|
1192
1378
|
if (t === "FIXED_LEN_BYTE_ARRAY") {
|
|
1193
1379
|
if (!r) throw Error("parquet missing fixed length");
|
|
1194
|
-
return
|
|
1195
|
-
}
|
|
1380
|
+
return Ct(e, n, r);
|
|
1381
|
+
}
|
|
1382
|
+
throw Error(`parquet unhandled type: ${t}`);
|
|
1196
1383
|
}
|
|
1197
|
-
function
|
|
1384
|
+
function gt(e, t) {
|
|
1198
1385
|
let n = Array(t);
|
|
1199
1386
|
for (let r = 0; r < t; r++) {
|
|
1200
|
-
let t = e.offset + (r / 8 | 0), i = r % 8;
|
|
1201
|
-
n[r] = (
|
|
1387
|
+
let t = e.offset + (r / 8 | 0), i = r % 8, a = e.view.getUint8(t);
|
|
1388
|
+
n[r] = !!(a & 1 << i);
|
|
1202
1389
|
}
|
|
1203
1390
|
return e.offset += Math.ceil(t / 8), n;
|
|
1204
1391
|
}
|
|
1205
|
-
function
|
|
1392
|
+
function _t(e, t) {
|
|
1206
1393
|
let n = (e.view.byteOffset + e.offset) % 4 ? new Int32Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 4)) : new Int32Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1207
1394
|
return e.offset += t * 4, n;
|
|
1208
1395
|
}
|
|
1209
|
-
function
|
|
1396
|
+
function vt(e, t) {
|
|
1210
1397
|
let n = (e.view.byteOffset + e.offset) % 8 ? new BigInt64Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 8)) : new BigInt64Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1211
1398
|
return e.offset += t * 8, n;
|
|
1212
1399
|
}
|
|
1213
|
-
function
|
|
1400
|
+
function yt(e, t) {
|
|
1214
1401
|
let n = Array(t);
|
|
1215
1402
|
for (let r = 0; r < t; r++) {
|
|
1216
1403
|
let t = e.view.getBigInt64(e.offset + r * 12, !0), i = e.view.getInt32(e.offset + r * 12 + 8, !0);
|
|
@@ -1218,15 +1405,15 @@ function rt(e, t) {
|
|
|
1218
1405
|
}
|
|
1219
1406
|
return e.offset += t * 12, n;
|
|
1220
1407
|
}
|
|
1221
|
-
function
|
|
1408
|
+
function bt(e, t) {
|
|
1222
1409
|
let n = (e.view.byteOffset + e.offset) % 4 ? new Float32Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 4)) : new Float32Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1223
1410
|
return e.offset += t * 4, n;
|
|
1224
1411
|
}
|
|
1225
|
-
function
|
|
1412
|
+
function xt(e, t) {
|
|
1226
1413
|
let n = (e.view.byteOffset + e.offset) % 8 ? new Float64Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 8)) : new Float64Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1227
1414
|
return e.offset += t * 8, n;
|
|
1228
1415
|
}
|
|
1229
|
-
function
|
|
1416
|
+
function St(e, t) {
|
|
1230
1417
|
let n = Array(t);
|
|
1231
1418
|
for (let r = 0; r < t; r++) {
|
|
1232
1419
|
let t = e.view.getUint32(e.offset, !0);
|
|
@@ -1234,7 +1421,7 @@ function ot(e, t) {
|
|
|
1234
1421
|
}
|
|
1235
1422
|
return n;
|
|
1236
1423
|
}
|
|
1237
|
-
function
|
|
1424
|
+
function Ct(e, t, n) {
|
|
1238
1425
|
let r = Array(t);
|
|
1239
1426
|
for (let i = 0; i < t; i++) r[i] = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, n), e.offset += n;
|
|
1240
1427
|
return r;
|
|
@@ -1245,17 +1432,17 @@ function Z(e, t, n) {
|
|
|
1245
1432
|
}
|
|
1246
1433
|
//#endregion
|
|
1247
1434
|
//#region ../../node_modules/hyparquet/src/snappy.js
|
|
1248
|
-
var
|
|
1435
|
+
var wt = [
|
|
1249
1436
|
0,
|
|
1250
1437
|
255,
|
|
1251
1438
|
65535,
|
|
1252
1439
|
16777215,
|
|
1253
1440
|
4294967295
|
|
1254
1441
|
];
|
|
1255
|
-
function
|
|
1442
|
+
function Tt(e, t, n, r, i) {
|
|
1256
1443
|
for (let a = 0; a < i; a++) n[r + a] = e[t + a];
|
|
1257
1444
|
}
|
|
1258
|
-
function
|
|
1445
|
+
function Et(e, t) {
|
|
1259
1446
|
let n = e.byteLength, r = t.byteLength, i = 0, a = 0;
|
|
1260
1447
|
for (; i < n;) {
|
|
1261
1448
|
let t = e[i];
|
|
@@ -1278,39 +1465,37 @@ function ut(e, t) {
|
|
|
1278
1465
|
case 3:
|
|
1279
1466
|
if (n <= i + 3) throw Error("snappy error end of input");
|
|
1280
1467
|
o = (r >>> 2) + 1, s = e[i] + (e[i + 1] << 8) + (e[i + 2] << 16) + (e[i + 3] << 24), i += 4;
|
|
1281
|
-
break;
|
|
1282
|
-
default: break;
|
|
1283
1468
|
}
|
|
1284
1469
|
if (s === 0 || isNaN(s)) throw Error(`invalid offset ${s} pos ${i} inputLength ${n}`);
|
|
1285
1470
|
if (s > a) throw Error("cannot copy from before start of buffer");
|
|
1286
|
-
|
|
1471
|
+
Tt(t, a - s, t, a, o), a += o;
|
|
1287
1472
|
} else {
|
|
1288
1473
|
let o = (r >>> 2) + 1;
|
|
1289
1474
|
if (o > 60) {
|
|
1290
1475
|
if (i + 3 >= n) throw Error("snappy error literal pos + 3 >= inputLength");
|
|
1291
1476
|
let t = o - 60;
|
|
1292
|
-
o = e[i] + (e[i + 1] << 8) + (e[i + 2] << 16) + (e[i + 3] << 24), o = (o &
|
|
1477
|
+
o = e[i] + (e[i + 1] << 8) + (e[i + 2] << 16) + (e[i + 3] << 24), o = (o & wt[t]) + 1, i += t;
|
|
1293
1478
|
}
|
|
1294
1479
|
if (i + o > n) throw Error("snappy error literal exceeds input length");
|
|
1295
|
-
|
|
1480
|
+
Tt(e, i, t, a, o), i += o, a += o;
|
|
1296
1481
|
}
|
|
1297
1482
|
}
|
|
1298
1483
|
if (a !== r) throw Error("premature end of input");
|
|
1299
1484
|
}
|
|
1300
1485
|
//#endregion
|
|
1301
1486
|
//#region ../../node_modules/hyparquet/src/datapage.js
|
|
1302
|
-
function
|
|
1487
|
+
function Dt(e, t, { type: n, element: r, schemaPath: i }) {
|
|
1303
1488
|
let a = new DataView(e.buffer, e.byteOffset, e.byteLength), o = {
|
|
1304
1489
|
view: a,
|
|
1305
1490
|
offset: 0
|
|
1306
|
-
}, s, c =
|
|
1491
|
+
}, s, c = Ot(o, t, i), { definitionLevels: l, numNulls: u } = kt(o, t, i), d = t.num_values - u;
|
|
1307
1492
|
if (t.encoding === "PLAIN") s = X(o, n, d, r.type_length);
|
|
1308
1493
|
else if (t.encoding === "PLAIN_DICTIONARY" || t.encoding === "RLE_DICTIONARY" || t.encoding === "RLE") {
|
|
1309
1494
|
let e = n === "BOOLEAN" ? 1 : a.getUint8(o.offset++);
|
|
1310
1495
|
e ? (s = Array(d), n === "BOOLEAN" ? (Y(o, e, s), s = s.map((e) => !!e)) : Y(o, e, s, a.byteLength - o.offset)) : s = new Uint8Array(d);
|
|
1311
|
-
} else if (t.encoding === "BYTE_STREAM_SPLIT") s =
|
|
1496
|
+
} else if (t.encoding === "BYTE_STREAM_SPLIT") s = mt(o, d, n, r.type_length);
|
|
1312
1497
|
else if (t.encoding === "DELTA_BINARY_PACKED") s = n === "INT32" ? new Int32Array(d) : new BigInt64Array(d), J(o, d, s);
|
|
1313
|
-
else if (t.encoding === "DELTA_LENGTH_BYTE_ARRAY") s = Array(d),
|
|
1498
|
+
else if (t.encoding === "DELTA_LENGTH_BYTE_ARRAY") s = Array(d), ut(o, d, s);
|
|
1314
1499
|
else throw Error(`parquet unsupported encoding: ${t.encoding}`);
|
|
1315
1500
|
return {
|
|
1316
1501
|
definitionLevels: l,
|
|
@@ -1318,7 +1503,7 @@ function dt(e, t, { type: n, element: r, schemaPath: i }) {
|
|
|
1318
1503
|
dataPage: s
|
|
1319
1504
|
};
|
|
1320
1505
|
}
|
|
1321
|
-
function
|
|
1506
|
+
function Ot(e, t, n) {
|
|
1322
1507
|
if (n.length > 1) {
|
|
1323
1508
|
let r = te(n);
|
|
1324
1509
|
if (r) {
|
|
@@ -1328,7 +1513,7 @@ function ft(e, t, n) {
|
|
|
1328
1513
|
}
|
|
1329
1514
|
return [];
|
|
1330
1515
|
}
|
|
1331
|
-
function
|
|
1516
|
+
function kt(e, t, n) {
|
|
1332
1517
|
let r = x(n);
|
|
1333
1518
|
if (!r) return {
|
|
1334
1519
|
definitionLevels: [],
|
|
@@ -1347,20 +1532,20 @@ function Q(e, t, n, r) {
|
|
|
1347
1532
|
let i, a = r?.[n];
|
|
1348
1533
|
if (n === "UNCOMPRESSED") i = e;
|
|
1349
1534
|
else if (a) i = a(e, t);
|
|
1350
|
-
else if (n === "SNAPPY") i = new Uint8Array(t),
|
|
1535
|
+
else if (n === "SNAPPY") i = new Uint8Array(t), Et(e, i);
|
|
1351
1536
|
else throw Error(`parquet unsupported compression codec: ${n}`);
|
|
1352
1537
|
if (i?.length !== t) throw Error(`parquet decompressed page length ${i?.length} does not match header ${t}`);
|
|
1353
1538
|
return i;
|
|
1354
1539
|
}
|
|
1355
|
-
function
|
|
1540
|
+
function At(e, t, n) {
|
|
1356
1541
|
let r = {
|
|
1357
1542
|
view: new DataView(e.buffer, e.byteOffset, e.byteLength),
|
|
1358
1543
|
offset: 0
|
|
1359
1544
|
}, { type: i, element: a, schemaPath: o, codec: s, compressors: c } = n, l = t.data_page_header_v2;
|
|
1360
1545
|
if (!l) throw Error("parquet data page header v2 is undefined");
|
|
1361
|
-
let u =
|
|
1546
|
+
let u = jt(r, l, o);
|
|
1362
1547
|
r.offset = l.repetition_levels_byte_length;
|
|
1363
|
-
let d =
|
|
1548
|
+
let d = Mt(r, l, o), f = t.uncompressed_page_size - l.definition_levels_byte_length - l.repetition_levels_byte_length, p = e.subarray(r.offset);
|
|
1364
1549
|
l.is_compressed !== !1 && (p = Q(p, f, s, c));
|
|
1365
1550
|
let m = new DataView(p.buffer, p.byteOffset, p.byteLength), h = {
|
|
1366
1551
|
view: m,
|
|
@@ -1372,9 +1557,9 @@ function mt(e, t, n) {
|
|
|
1372
1557
|
let e = m.getUint8(h.offset++);
|
|
1373
1558
|
g = Array(_), Y(h, e, g, f - 1);
|
|
1374
1559
|
} else if (l.encoding === "DELTA_BINARY_PACKED") g = i === "INT32" ? new Int32Array(_) : new BigInt64Array(_), J(h, _, g);
|
|
1375
|
-
else if (l.encoding === "DELTA_LENGTH_BYTE_ARRAY") g = Array(_),
|
|
1376
|
-
else if (l.encoding === "DELTA_BYTE_ARRAY") g = Array(_),
|
|
1377
|
-
else if (l.encoding === "BYTE_STREAM_SPLIT") g =
|
|
1560
|
+
else if (l.encoding === "DELTA_LENGTH_BYTE_ARRAY") g = Array(_), ut(h, _, g);
|
|
1561
|
+
else if (l.encoding === "DELTA_BYTE_ARRAY") g = Array(_), dt(h, _, g);
|
|
1562
|
+
else if (l.encoding === "BYTE_STREAM_SPLIT") g = mt(h, _, i, a.type_length);
|
|
1378
1563
|
else throw Error(`parquet unsupported encoding: ${l.encoding}`);
|
|
1379
1564
|
return {
|
|
1380
1565
|
definitionLevels: d,
|
|
@@ -1382,13 +1567,13 @@ function mt(e, t, n) {
|
|
|
1382
1567
|
dataPage: g
|
|
1383
1568
|
};
|
|
1384
1569
|
}
|
|
1385
|
-
function
|
|
1570
|
+
function jt(e, t, n) {
|
|
1386
1571
|
let r = te(n);
|
|
1387
1572
|
if (!r) return [];
|
|
1388
1573
|
let i = Array(t.num_values);
|
|
1389
1574
|
return Y(e, $(r), i, t.repetition_levels_byte_length), i;
|
|
1390
1575
|
}
|
|
1391
|
-
function
|
|
1576
|
+
function Mt(e, t, n) {
|
|
1392
1577
|
let r = x(n);
|
|
1393
1578
|
if (r) {
|
|
1394
1579
|
let n = Array(t.num_values);
|
|
@@ -1400,8 +1585,8 @@ function $(e) {
|
|
|
1400
1585
|
}
|
|
1401
1586
|
//#endregion
|
|
1402
1587
|
//#region ../../node_modules/hyparquet/src/column.js
|
|
1403
|
-
function
|
|
1404
|
-
let { pathInSchema: o, schemaPath: s } = i, c =
|
|
1588
|
+
function Nt(e, { groupStart: t, selectStart: n, selectEnd: r }, i, a) {
|
|
1589
|
+
let { pathInSchema: o, schemaPath: s } = i, c = ie(s), l = [], u, d, f = 0, p = 0, m = a && (() => {
|
|
1405
1590
|
d && a({
|
|
1406
1591
|
pathInSchema: o,
|
|
1407
1592
|
columnData: d,
|
|
@@ -1410,12 +1595,12 @@ function _t(e, { groupStart: t, selectStart: n, selectEnd: r }, i, a) {
|
|
|
1410
1595
|
});
|
|
1411
1596
|
});
|
|
1412
1597
|
for (; (c ? f < r : e.offset < e.view.byteLength - 1) && !(e.offset >= e.view.byteLength - 1);) {
|
|
1413
|
-
let t =
|
|
1598
|
+
let t = Ft(e);
|
|
1414
1599
|
if (t.type === "DICTIONARY_PAGE") {
|
|
1415
|
-
let { data: n } =
|
|
1600
|
+
let { data: n } = Pt(e, t, i, u, void 0, 0);
|
|
1416
1601
|
n && (u = h(n, i));
|
|
1417
1602
|
} else {
|
|
1418
|
-
let r = d?.length || 0, a =
|
|
1603
|
+
let r = d?.length || 0, a = Pt(e, t, i, u, d, n - f);
|
|
1419
1604
|
a.skipped ? (l.length || (p += a.skipped), f += a.skipped) : a.data && d === a.data ? f += a.data.length - r : a.data && a.data.length && (m?.(), l.push(a.data), f += a.data.length, d = a.data);
|
|
1420
1605
|
}
|
|
1421
1606
|
}
|
|
@@ -1424,27 +1609,29 @@ function _t(e, { groupStart: t, selectStart: n, selectEnd: r }, i, a) {
|
|
|
1424
1609
|
skipped: p
|
|
1425
1610
|
};
|
|
1426
1611
|
}
|
|
1427
|
-
function
|
|
1612
|
+
function Pt(e, t, n, r, i, a) {
|
|
1428
1613
|
let { type: o, element: s, schemaPath: c, codec: l, compressors: u } = n, d = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, t.compressed_page_size);
|
|
1429
1614
|
if (e.offset += t.compressed_page_size, t.type === "DATA_PAGE") {
|
|
1430
1615
|
let e = t.data_page_header;
|
|
1431
1616
|
if (!e) throw Error("parquet data page header is undefined");
|
|
1432
|
-
if (a > e.num_values &&
|
|
1433
|
-
let { definitionLevels: o, repetitionLevels: s, dataPage: f } =
|
|
1617
|
+
if (a > e.num_values && ie(c)) return { skipped: e.num_values };
|
|
1618
|
+
let { definitionLevels: o, repetitionLevels: s, dataPage: f } = Dt(Q(d, Number(t.uncompressed_page_size), l, u), e, n), p = m(f, r, e.encoding, n);
|
|
1434
1619
|
return {
|
|
1435
1620
|
skipped: 0,
|
|
1436
|
-
data:
|
|
1621
|
+
data: ot(Array.isArray(i) ? i : [], o, s, p, c)
|
|
1437
1622
|
};
|
|
1438
|
-
}
|
|
1623
|
+
}
|
|
1624
|
+
if (t.type === "DATA_PAGE_V2") {
|
|
1439
1625
|
let e = t.data_page_header_v2;
|
|
1440
1626
|
if (!e) throw Error("parquet data page header v2 is undefined");
|
|
1441
1627
|
if (a > e.num_rows) return { skipped: e.num_values };
|
|
1442
|
-
let { definitionLevels: o, repetitionLevels: s, dataPage: l } =
|
|
1628
|
+
let { definitionLevels: o, repetitionLevels: s, dataPage: l } = At(d, t, n), u = m(l, r, e.encoding, n);
|
|
1443
1629
|
return {
|
|
1444
1630
|
skipped: 0,
|
|
1445
|
-
data:
|
|
1631
|
+
data: ot(Array.isArray(i) ? i : [], o, s, u, c)
|
|
1446
1632
|
};
|
|
1447
|
-
}
|
|
1633
|
+
}
|
|
1634
|
+
if (t.type === "DICTIONARY_PAGE") {
|
|
1448
1635
|
let e = t.dictionary_page_header;
|
|
1449
1636
|
if (!e) throw Error("parquet dictionary page header is undefined");
|
|
1450
1637
|
let n = Q(d, Number(t.uncompressed_page_size), l, u);
|
|
@@ -1455,10 +1642,11 @@ function vt(e, t, n, r, i, a) {
|
|
|
1455
1642
|
offset: 0
|
|
1456
1643
|
}, o, e.num_values, s.type_length)
|
|
1457
1644
|
};
|
|
1458
|
-
}
|
|
1645
|
+
}
|
|
1646
|
+
throw Error(`parquet unsupported page type: ${t.type}`);
|
|
1459
1647
|
}
|
|
1460
|
-
function
|
|
1461
|
-
let n =
|
|
1648
|
+
function Ft(e) {
|
|
1649
|
+
let n = S(e);
|
|
1462
1650
|
return {
|
|
1463
1651
|
type: a[n.field_1],
|
|
1464
1652
|
uncompressed_page_size: n.field_2,
|
|
@@ -1497,124 +1685,118 @@ function yt(e) {
|
|
|
1497
1685
|
};
|
|
1498
1686
|
}
|
|
1499
1687
|
//#endregion
|
|
1500
|
-
//#region ../../node_modules/hyparquet/src/indexes.js
|
|
1501
|
-
function bt(e) {
|
|
1502
|
-
let t = C(e);
|
|
1503
|
-
return {
|
|
1504
|
-
page_locations: t.field_1.map((e) => ({
|
|
1505
|
-
offset: e.field_1,
|
|
1506
|
-
compressed_page_size: e.field_2,
|
|
1507
|
-
first_row_index: e.field_3
|
|
1508
|
-
})),
|
|
1509
|
-
unencoded_byte_array_data_bytes: t.field_2
|
|
1510
|
-
};
|
|
1511
|
-
}
|
|
1512
|
-
//#endregion
|
|
1513
1688
|
//#region ../../node_modules/hyparquet/src/rowgroup.js
|
|
1514
|
-
function
|
|
1689
|
+
function It(e, { metadata: t }, n) {
|
|
1515
1690
|
let r = [];
|
|
1516
1691
|
for (let i of n.chunks) {
|
|
1517
|
-
let {
|
|
1518
|
-
pathInSchema:
|
|
1519
|
-
element:
|
|
1520
|
-
schemaPath:
|
|
1692
|
+
let { path_in_schema: a } = i.columnMetadata, o = b(t.schema, a), s = {
|
|
1693
|
+
pathInSchema: a,
|
|
1694
|
+
element: o[o.length - 1].element,
|
|
1695
|
+
schemaPath: o,
|
|
1696
|
+
...e,
|
|
1697
|
+
...i.columnMetadata,
|
|
1521
1698
|
parsers: {
|
|
1522
1699
|
...p,
|
|
1523
1700
|
...e.parsers
|
|
1524
|
-
}
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
offset: 0
|
|
1534
|
-
}, n, l, e.onPage))
|
|
1535
|
-
});
|
|
1536
|
-
continue;
|
|
1537
|
-
}
|
|
1538
|
-
r.push({
|
|
1539
|
-
pathInSchema: s,
|
|
1540
|
-
data: Promise.resolve(e.file.slice(i.offsetIndex.startByte, i.offsetIndex.endByte)).then(async (t) => {
|
|
1541
|
-
let { selectStart: r, selectEnd: i } = n, s = bt({
|
|
1701
|
+
}
|
|
1702
|
+
}, { startByte: c, endByte: l } = i.range;
|
|
1703
|
+
"pageLocations" in i ? r.push({
|
|
1704
|
+
pathInSchema: a,
|
|
1705
|
+
data: Lt(e, n, i, i.pageLocations, s)
|
|
1706
|
+
}) : "offsetIndex" in i ? r.push({
|
|
1707
|
+
pathInSchema: a,
|
|
1708
|
+
data: Promise.resolve(e.file.slice(i.offsetIndex.startByte, i.offsetIndex.endByte)).then((t) => {
|
|
1709
|
+
let r = He({
|
|
1542
1710
|
view: new DataView(t),
|
|
1543
1711
|
offset: 0
|
|
1544
|
-
}).page_locations
|
|
1545
|
-
|
|
1546
|
-
let t = s[e], a = Number(t.first_row_index), o = e + 1 < s.length ? Number(s[e + 1].first_row_index) : n.groupRows;
|
|
1547
|
-
c < 0 && !f && o > r && (u = Number(t.offset), c = a), a < i && (d = Number(t.offset) + t.compressed_page_size);
|
|
1548
|
-
}
|
|
1549
|
-
c < 0 && (c = 0);
|
|
1550
|
-
let p = await e.file.slice(u, d), { data: m, skipped: h } = _t({
|
|
1551
|
-
view: new DataView(p),
|
|
1552
|
-
offset: 0
|
|
1553
|
-
}, c ? {
|
|
1554
|
-
...n,
|
|
1555
|
-
groupStart: n.groupStart + c,
|
|
1556
|
-
selectStart: n.selectStart - c,
|
|
1557
|
-
selectEnd: n.selectEnd - c
|
|
1558
|
-
} : n, l, e.onPage);
|
|
1559
|
-
return {
|
|
1560
|
-
data: m,
|
|
1561
|
-
skipped: c + h
|
|
1562
|
-
};
|
|
1712
|
+
}).page_locations;
|
|
1713
|
+
return Lt(e, n, i, r, s);
|
|
1563
1714
|
})
|
|
1715
|
+
}) : r.push({
|
|
1716
|
+
pathInSchema: a,
|
|
1717
|
+
data: Promise.resolve(e.file.slice(c, l)).then((t) => Nt({
|
|
1718
|
+
view: new DataView(t),
|
|
1719
|
+
offset: 0
|
|
1720
|
+
}, n, s, e.onPage))
|
|
1564
1721
|
});
|
|
1565
1722
|
}
|
|
1566
1723
|
return {
|
|
1567
1724
|
groupStart: n.groupStart,
|
|
1568
1725
|
groupRows: n.groupRows,
|
|
1726
|
+
selectStart: n.selectStart,
|
|
1727
|
+
selectEnd: n.selectEnd,
|
|
1569
1728
|
asyncColumns: r
|
|
1570
1729
|
};
|
|
1571
1730
|
}
|
|
1572
|
-
function
|
|
1573
|
-
let {
|
|
1574
|
-
|
|
1731
|
+
async function Lt(e, t, n, r, i) {
|
|
1732
|
+
let { data_page_offset: a, dictionary_page_offset: o } = n.columnMetadata, { selectStart: s, selectEnd: c } = t, { startByte: l, endByte: u } = n.range, d = -1, f = o || a < r[0].offset;
|
|
1733
|
+
for (let e = 0; e < r.length; e++) {
|
|
1734
|
+
let n = r[e], i = Number(n.first_row_index), a = e + 1 < r.length ? Number(r[e + 1].first_row_index) : t.groupRows;
|
|
1735
|
+
d < 0 && a > s && (l = Number(n.offset), d = i), i < c && (u = Number(n.offset) + n.compressed_page_size);
|
|
1736
|
+
}
|
|
1737
|
+
d < 0 && (d = 0);
|
|
1738
|
+
let p;
|
|
1739
|
+
if (f && d) {
|
|
1740
|
+
let t = Number(r[0].offset) - n.range.startByte, [i, a] = await Promise.all([e.file.slice(n.range.startByte, Number(r[0].offset)), e.file.slice(l, u)]), o = new Uint8Array(t + a.byteLength);
|
|
1741
|
+
o.set(new Uint8Array(i, 0, t)), o.set(new Uint8Array(a), t), p = new DataView(o.buffer);
|
|
1742
|
+
} else p = f ? new DataView(await e.file.slice(n.range.startByte, u)) : new DataView(await e.file.slice(l, u));
|
|
1743
|
+
let { data: m, skipped: h } = Nt({
|
|
1744
|
+
view: p,
|
|
1745
|
+
offset: 0
|
|
1746
|
+
}, d ? {
|
|
1747
|
+
...t,
|
|
1748
|
+
groupStart: t.groupStart + d,
|
|
1749
|
+
selectStart: t.selectStart - d,
|
|
1750
|
+
selectEnd: t.selectEnd - d
|
|
1751
|
+
} : t, i, e.onPage);
|
|
1752
|
+
return {
|
|
1753
|
+
data: m,
|
|
1754
|
+
skipped: d + h
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
function Rt(e, t, n) {
|
|
1758
|
+
let { asyncColumns: r } = e, i = {
|
|
1575
1759
|
...p,
|
|
1576
1760
|
...n
|
|
1577
|
-
};
|
|
1578
|
-
let
|
|
1579
|
-
|
|
1580
|
-
let t = r.filter((t) => t.pathInSchema[0] === e.element.name);
|
|
1761
|
+
}, a = [];
|
|
1762
|
+
for (let n of t.children) if (n.children.length) {
|
|
1763
|
+
let t = r.filter((e) => e.pathInSchema[0] === n.element.name);
|
|
1581
1764
|
if (!t.length) continue;
|
|
1582
|
-
|
|
1583
|
-
pathInSchema:
|
|
1765
|
+
a.push({
|
|
1766
|
+
pathInSchema: n.path,
|
|
1584
1767
|
data: (async () => {
|
|
1585
|
-
let r = await Promise.all(t.map((e) => e.data)),
|
|
1768
|
+
let r = await Promise.all(t.map((e) => e.data)), a = /* @__PURE__ */ new Map(), o = r.map(({ data: e }) => Ie(e)), s = Math.max(e.selectStart ?? 0, ...r.map((e) => e.skipped)), c = Math.min(e.selectEnd ?? Infinity, ...r.map((e, t) => e.skipped + o[t].length));
|
|
1586
1769
|
for (let e = 0; e < t.length; e++) {
|
|
1587
|
-
let n =
|
|
1588
|
-
|
|
1770
|
+
let n = s - r[e].skipped, i = Math.max(0, c - s);
|
|
1771
|
+
a.set(t[e].pathInSchema.join("."), o[e].slice(n, n + i));
|
|
1589
1772
|
}
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
if (!o) throw Error("parquet column data not assembled");
|
|
1773
|
+
K(a, n, i);
|
|
1774
|
+
let l = a.get(n.element.name);
|
|
1775
|
+
if (!l) throw Error("parquet column data not assembled");
|
|
1594
1776
|
return {
|
|
1595
|
-
data: [
|
|
1596
|
-
skipped:
|
|
1777
|
+
data: [l],
|
|
1778
|
+
skipped: s
|
|
1597
1779
|
};
|
|
1598
1780
|
})()
|
|
1599
1781
|
});
|
|
1600
1782
|
} else {
|
|
1601
|
-
let
|
|
1602
|
-
|
|
1783
|
+
let e = r.find((e) => e.pathInSchema[0] === n.element.name);
|
|
1784
|
+
e && a.push(e);
|
|
1603
1785
|
}
|
|
1604
1786
|
return {
|
|
1605
1787
|
...e,
|
|
1606
|
-
asyncColumns:
|
|
1788
|
+
asyncColumns: a
|
|
1607
1789
|
};
|
|
1608
1790
|
}
|
|
1609
1791
|
//#endregion
|
|
1610
1792
|
//#region ../core/src/data/formats/parquetRead.js
|
|
1611
|
-
var
|
|
1612
|
-
function
|
|
1793
|
+
var zt = /* @__PURE__ */ new Map(), Bt = 200;
|
|
1794
|
+
function Vt(e) {
|
|
1613
1795
|
if (!e.metadata) throw Error("parquet requires metadata");
|
|
1614
|
-
let t =
|
|
1615
|
-
return e.file =
|
|
1796
|
+
let t = Ge(e);
|
|
1797
|
+
return e.file = Qe(e.file, t), t.groups.map((n) => It(e, t, n));
|
|
1616
1798
|
}
|
|
1617
|
-
function
|
|
1799
|
+
function Ht(e) {
|
|
1618
1800
|
if (!e) return [];
|
|
1619
1801
|
if (e.length === 1) return e[0];
|
|
1620
1802
|
let t = 0;
|
|
@@ -1626,13 +1808,13 @@ function Et(e) {
|
|
|
1626
1808
|
}
|
|
1627
1809
|
return n;
|
|
1628
1810
|
}
|
|
1629
|
-
function
|
|
1630
|
-
let t = e.join(""), n =
|
|
1811
|
+
function Ut(e) {
|
|
1812
|
+
let t = e.join(""), n = zt.get(t);
|
|
1631
1813
|
if (n) return n;
|
|
1632
1814
|
let r = e.map((e, t) => JSON.stringify(e) + ": columnData[" + t + "][row - columnSkipped[" + t + "]]").join(",\n"), i = Function("groupData", "selectStart", "selectCount", "columnData", "columnSkipped", "for (let selectRow = 0; selectRow < selectCount; selectRow++) {\n const row = selectStart + selectRow;\n groupData[selectRow] = {\n" + r + "\n };\n}\nreturn groupData;");
|
|
1633
|
-
return
|
|
1815
|
+
return zt.set(t, i), i;
|
|
1634
1816
|
}
|
|
1635
|
-
function
|
|
1817
|
+
function Wt(e, t, n, r, i, a) {
|
|
1636
1818
|
for (let o = 0; o < n; o++) {
|
|
1637
1819
|
let n = t + o, s = {};
|
|
1638
1820
|
for (let e = 0; e < r.length; e++) s[r[e]] = i[e][n - a[e]];
|
|
@@ -1640,22 +1822,22 @@ function Ot(e, t, n, r, i, a) {
|
|
|
1640
1822
|
}
|
|
1641
1823
|
return e;
|
|
1642
1824
|
}
|
|
1643
|
-
async function
|
|
1825
|
+
async function Gt({ asyncColumns: e }, t, n) {
|
|
1644
1826
|
let r = await Promise.all(e.map((e) => e.data)), i = e.length, a = Array(i), o = Array(i), s = Array(i);
|
|
1645
|
-
for (let t = 0; t < i; t++) a[t] = e[t].pathInSchema[0], o[t] =
|
|
1827
|
+
for (let t = 0; t < i; t++) a[t] = e[t].pathInSchema[0], o[t] = Ht(r[t].data), s[t] = r[t].skipped;
|
|
1646
1828
|
let c = n - t, l = Array(c);
|
|
1647
|
-
return i >
|
|
1829
|
+
return i > Bt ? Wt(l, t, c, a, o, s) : Ut(a)(l, t, c, o, s);
|
|
1648
1830
|
}
|
|
1649
|
-
async function
|
|
1831
|
+
async function Kt(e) {
|
|
1650
1832
|
if ("rowFormat" in e) throw Error("parquetRead supports only object rows; use rowFormat: \"object\" implicitly");
|
|
1651
1833
|
if ("filter" in e || "filterStrict" in e) throw Error("parquetRead does not support filtering");
|
|
1652
1834
|
e.metadata ??= await be(e.file, e);
|
|
1653
|
-
let { rowStart: t = 0, rowEnd: n, onChunk: r, onComplete: i } = e, a =
|
|
1835
|
+
let { rowStart: t = 0, rowEnd: n, onChunk: r, onComplete: i } = e, a = Vt(e);
|
|
1654
1836
|
if (!i && !r) {
|
|
1655
1837
|
for (let { asyncColumns: e } of a) for (let { data: t } of e) await t;
|
|
1656
1838
|
return;
|
|
1657
1839
|
}
|
|
1658
|
-
let o = Se(e.metadata), s = a.map((t) =>
|
|
1840
|
+
let o = Se(e.metadata), s = a.map((t) => Rt(t, o, e.parsers));
|
|
1659
1841
|
if (r) for (let e of s) for (let t of e.asyncColumns) t.data.then((n) => {
|
|
1660
1842
|
let i = e.groupStart + n.skipped;
|
|
1661
1843
|
for (let e of n.data) r({
|
|
@@ -1667,17 +1849,17 @@ async function At(e) {
|
|
|
1667
1849
|
});
|
|
1668
1850
|
if (i) {
|
|
1669
1851
|
let e = [];
|
|
1670
|
-
for (let r of s) Fe(e, await
|
|
1852
|
+
for (let r of s) Fe(e, await Gt(r, Math.max(t - r.groupStart, 0), Math.min((n ?? Infinity) - r.groupStart, r.groupRows)));
|
|
1671
1853
|
i(e);
|
|
1672
1854
|
} else for (let { asyncColumns: e } of s) for (let { data: t } of e) await t;
|
|
1673
1855
|
}
|
|
1674
|
-
function
|
|
1856
|
+
function qt(e) {
|
|
1675
1857
|
return new Promise((t, n) => {
|
|
1676
|
-
|
|
1858
|
+
Kt({
|
|
1677
1859
|
...e,
|
|
1678
1860
|
onComplete: t
|
|
1679
1861
|
}).catch(n);
|
|
1680
1862
|
});
|
|
1681
1863
|
}
|
|
1682
1864
|
//#endregion
|
|
1683
|
-
export {
|
|
1865
|
+
export { qt as parquetReadObjects };
|