@nika-js/onlymap 0.5.2 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -5
- package/THIRD-PARTY-LICENSES.md +93 -0
- package/dist/{Arrow.dom-BNjbQ9jb.js → Arrow.dom-BFGShwSA.js} +854 -1427
- package/dist/{LercDecode.es-ReXBgmrK.js → LercDecode.es-5mr_B7pP.js} +1 -1
- package/dist/{basemap-DF_rK3Aa.js → basemap-DJjFEgA9.js} +1 -1
- package/dist/builder-CqJZRFaq.js +580 -0
- package/dist/date-format.d.ts +18 -0
- package/dist/draw-controller.d.ts +10 -1
- package/dist/flatgeobuf-Cx0IwxhV.js +6332 -0
- package/dist/geodesy.d.ts +47 -0
- package/dist/geoparquet-AL10HAyd.js +35 -0
- package/dist/geoparquet.d.ts +33 -0
- package/dist/html-data.d.ts +2 -2
- package/dist/{index-DWXl9Ykk.js → index-BQaxwlV3.js} +1 -1
- package/dist/index-BYbMtNVH.js +2301 -0
- package/dist/{index-B87QAKKr.js → index-BtPO4p2v.js} +2 -2
- package/dist/index-CgPV7QlM.js +4019 -0
- package/dist/{index-BYW4X7IK.js → index-DqE9di9E.js} +1 -1
- package/dist/{index-ytnYzWRW.js → index-G3hmIs2P.js} +47016 -46502
- package/dist/{index-DcFZsjQm.js → index-iwo3R3LT.js} +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/{lerc-C_VBg2kh.js → lerc-Dx3uB2j-.js} +2 -2
- package/dist/measure-controller.d.ts +41 -0
- package/dist/onlymap.standalone.js +108100 -94902
- package/dist/onlymapjs.js +82 -67
- package/dist/{raster-Unx9530f.js → raster-BPn_dJpK.js} +2 -2
- package/dist/units.d.ts +40 -0
- package/dist/validation.d.ts +10 -0
- package/dist/version.d.ts +1 -1
- package/llms.txt +8 -5
- package/onlymapjs.html-data.json +141 -62
- package/package.json +4 -1
- package/skills/onlymapjs/SKILL.md +4 -1
- package/skills/onlymapjs/references/syntax.md +103 -19
- package/skills/onlymapjs/references/testing.md +11 -2
|
@@ -0,0 +1,2301 @@
|
|
|
1
|
+
const Ue = [
|
|
2
|
+
"BOOLEAN",
|
|
3
|
+
"INT32",
|
|
4
|
+
"INT64",
|
|
5
|
+
"INT96",
|
|
6
|
+
// deprecated
|
|
7
|
+
"FLOAT",
|
|
8
|
+
"DOUBLE",
|
|
9
|
+
"BYTE_ARRAY",
|
|
10
|
+
"FIXED_LEN_BYTE_ARRAY"
|
|
11
|
+
], G = [
|
|
12
|
+
"PLAIN",
|
|
13
|
+
"GROUP_VAR_INT",
|
|
14
|
+
// deprecated
|
|
15
|
+
"PLAIN_DICTIONARY",
|
|
16
|
+
"RLE",
|
|
17
|
+
"BIT_PACKED",
|
|
18
|
+
// deprecated
|
|
19
|
+
"DELTA_BINARY_PACKED",
|
|
20
|
+
"DELTA_LENGTH_BYTE_ARRAY",
|
|
21
|
+
"DELTA_BYTE_ARRAY",
|
|
22
|
+
"RLE_DICTIONARY",
|
|
23
|
+
"BYTE_STREAM_SPLIT"
|
|
24
|
+
], Ut = [
|
|
25
|
+
"REQUIRED",
|
|
26
|
+
"OPTIONAL",
|
|
27
|
+
"REPEATED"
|
|
28
|
+
], Pt = [
|
|
29
|
+
"UTF8",
|
|
30
|
+
"MAP",
|
|
31
|
+
"MAP_KEY_VALUE",
|
|
32
|
+
"LIST",
|
|
33
|
+
"ENUM",
|
|
34
|
+
"DECIMAL",
|
|
35
|
+
"DATE",
|
|
36
|
+
"TIME_MILLIS",
|
|
37
|
+
"TIME_MICROS",
|
|
38
|
+
"TIMESTAMP_MILLIS",
|
|
39
|
+
"TIMESTAMP_MICROS",
|
|
40
|
+
"UINT_8",
|
|
41
|
+
"UINT_16",
|
|
42
|
+
"UINT_32",
|
|
43
|
+
"UINT_64",
|
|
44
|
+
"INT_8",
|
|
45
|
+
"INT_16",
|
|
46
|
+
"INT_32",
|
|
47
|
+
"INT_64",
|
|
48
|
+
"JSON",
|
|
49
|
+
"BSON",
|
|
50
|
+
"INTERVAL"
|
|
51
|
+
], $t = [
|
|
52
|
+
"UNCOMPRESSED",
|
|
53
|
+
"SNAPPY",
|
|
54
|
+
"GZIP",
|
|
55
|
+
"LZO",
|
|
56
|
+
"BROTLI",
|
|
57
|
+
"LZ4",
|
|
58
|
+
"ZSTD",
|
|
59
|
+
"LZ4_RAW"
|
|
60
|
+
], lt = [
|
|
61
|
+
"DATA_PAGE",
|
|
62
|
+
"INDEX_PAGE",
|
|
63
|
+
"DICTIONARY_PAGE",
|
|
64
|
+
"DATA_PAGE_V2"
|
|
65
|
+
], Ct = [
|
|
66
|
+
"UNORDERED",
|
|
67
|
+
"ASCENDING",
|
|
68
|
+
"DESCENDING"
|
|
69
|
+
], Ft = [
|
|
70
|
+
"SPHERICAL",
|
|
71
|
+
"VINCENTY",
|
|
72
|
+
"THOMAS",
|
|
73
|
+
"ANDOYER",
|
|
74
|
+
"KARNEY"
|
|
75
|
+
];
|
|
76
|
+
function pe(e) {
|
|
77
|
+
const t = ce(e);
|
|
78
|
+
if (t.type === 1)
|
|
79
|
+
return { type: "Point", coordinates: Ae(e, t) };
|
|
80
|
+
if (t.type === 2)
|
|
81
|
+
return { type: "LineString", coordinates: Ee(e, t) };
|
|
82
|
+
if (t.type === 3)
|
|
83
|
+
return { type: "Polygon", coordinates: Pe(e, t) };
|
|
84
|
+
if (t.type === 4) {
|
|
85
|
+
const i = [];
|
|
86
|
+
for (let n = 0; n < t.count; n++)
|
|
87
|
+
i.push(Ae(e, ce(e)));
|
|
88
|
+
return { type: "MultiPoint", coordinates: i };
|
|
89
|
+
} else if (t.type === 5) {
|
|
90
|
+
const i = [];
|
|
91
|
+
for (let n = 0; n < t.count; n++)
|
|
92
|
+
i.push(Ee(e, ce(e)));
|
|
93
|
+
return { type: "MultiLineString", coordinates: i };
|
|
94
|
+
} else if (t.type === 6) {
|
|
95
|
+
const i = [];
|
|
96
|
+
for (let n = 0; n < t.count; n++)
|
|
97
|
+
i.push(Pe(e, ce(e)));
|
|
98
|
+
return { type: "MultiPolygon", coordinates: i };
|
|
99
|
+
} else if (t.type === 7) {
|
|
100
|
+
const i = [];
|
|
101
|
+
for (let n = 0; n < t.count; n++)
|
|
102
|
+
i.push(pe(e));
|
|
103
|
+
return { type: "GeometryCollection", geometries: i };
|
|
104
|
+
} else
|
|
105
|
+
throw new Error(`Unsupported geometry type: ${t.type}`);
|
|
106
|
+
}
|
|
107
|
+
function ce(e) {
|
|
108
|
+
const { view: t } = e, i = t.getUint8(e.offset++) === 1, n = t.getUint32(e.offset, i);
|
|
109
|
+
e.offset += 4;
|
|
110
|
+
const r = n % 1e3, f = Math.floor(n / 1e3);
|
|
111
|
+
let s = 0;
|
|
112
|
+
r > 1 && r <= 7 && (s = t.getUint32(e.offset, i), e.offset += 4);
|
|
113
|
+
let o = 2;
|
|
114
|
+
return f && o++, f === 3 && o++, { littleEndian: i, type: r, dim: o, count: s };
|
|
115
|
+
}
|
|
116
|
+
function Ae(e, t) {
|
|
117
|
+
const i = [];
|
|
118
|
+
for (let n = 0; n < t.dim; n++) {
|
|
119
|
+
const r = e.view.getFloat64(e.offset, t.littleEndian);
|
|
120
|
+
e.offset += 8, i.push(r);
|
|
121
|
+
}
|
|
122
|
+
return i;
|
|
123
|
+
}
|
|
124
|
+
function Ee(e, t) {
|
|
125
|
+
const i = [];
|
|
126
|
+
for (let n = 0; n < t.count; n++)
|
|
127
|
+
i.push(Ae(e, t));
|
|
128
|
+
return i;
|
|
129
|
+
}
|
|
130
|
+
function Pe(e, t) {
|
|
131
|
+
const { view: i } = e, n = [];
|
|
132
|
+
for (let r = 0; r < t.count; r++) {
|
|
133
|
+
const f = i.getUint32(e.offset, t.littleEndian);
|
|
134
|
+
e.offset += 4, n.push(Ee(e, { ...t, count: f }));
|
|
135
|
+
}
|
|
136
|
+
return n;
|
|
137
|
+
}
|
|
138
|
+
const $e = new TextDecoder(), oe = {
|
|
139
|
+
timestampFromMilliseconds(e) {
|
|
140
|
+
return new Date(Number(e));
|
|
141
|
+
},
|
|
142
|
+
timestampFromMicroseconds(e) {
|
|
143
|
+
return new Date(Number(e / 1000n));
|
|
144
|
+
},
|
|
145
|
+
timestampFromNanoseconds(e) {
|
|
146
|
+
return new Date(Number(e / 1000000n));
|
|
147
|
+
},
|
|
148
|
+
dateFromDays(e) {
|
|
149
|
+
return new Date(e * 864e5);
|
|
150
|
+
},
|
|
151
|
+
stringFromBytes(e) {
|
|
152
|
+
return e && $e.decode(e);
|
|
153
|
+
},
|
|
154
|
+
jsonFromBytes(e) {
|
|
155
|
+
return e && JSON.parse($e.decode(e));
|
|
156
|
+
},
|
|
157
|
+
geometryFromBytes(e) {
|
|
158
|
+
return e && pe({ view: new DataView(e.buffer, e.byteOffset, e.byteLength), offset: 0 });
|
|
159
|
+
},
|
|
160
|
+
geographyFromBytes(e) {
|
|
161
|
+
return e && pe({ view: new DataView(e.buffer, e.byteOffset, e.byteLength), offset: 0 });
|
|
162
|
+
},
|
|
163
|
+
uuidFromBytes(e) {
|
|
164
|
+
if (!e) return;
|
|
165
|
+
const t = Array.from(e, (i) => i.toString(16).padStart(2, "0")).join("");
|
|
166
|
+
return t.slice(0, 8) + "-" + t.slice(8, 12) + "-" + t.slice(12, 16) + "-" + t.slice(16, 20) + "-" + t.slice(20, 32);
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
function Ce(e, t, i, n) {
|
|
170
|
+
if (t && i.endsWith("_DICTIONARY")) {
|
|
171
|
+
let r = e;
|
|
172
|
+
e instanceof Uint8Array && !(t instanceof Uint8Array) && (r = new t.constructor(e.length));
|
|
173
|
+
for (let f = 0; f < e.length; f++)
|
|
174
|
+
r[f] = t[e[f]];
|
|
175
|
+
return r;
|
|
176
|
+
} else
|
|
177
|
+
return ct(e, n);
|
|
178
|
+
}
|
|
179
|
+
function ct(e, t) {
|
|
180
|
+
const { element: i, parsers: n, utf8: r = !0, schemaPath: f } = t, { type: s, converted_type: o, logical_type: c } = i, d = i.repetition_type !== "REQUIRED";
|
|
181
|
+
if (f?.some((l) => l.element.logical_type?.type === "VARIANT") && s === "BYTE_ARRAY" && o !== "UTF8" && c?.type !== "STRING")
|
|
182
|
+
return e;
|
|
183
|
+
if (o === "DECIMAL") {
|
|
184
|
+
const u = 10 ** -(i.scale || 0), g = new Array(e.length);
|
|
185
|
+
for (let h = 0; h < g.length; h++)
|
|
186
|
+
e[h] instanceof Uint8Array ? g[h] = at(e[h]) * u : g[h] = Number(e[h]) * u;
|
|
187
|
+
return g;
|
|
188
|
+
}
|
|
189
|
+
if (!o && s === "INT96")
|
|
190
|
+
return Array.from(e).map((l) => n.timestampFromNanoseconds(kt(l)));
|
|
191
|
+
if (o === "DATE")
|
|
192
|
+
return Array.from(e).map((l) => n.dateFromDays(l));
|
|
193
|
+
if (o === "TIMESTAMP_MILLIS")
|
|
194
|
+
return Array.from(e).map((l) => n.timestampFromMilliseconds(l));
|
|
195
|
+
if (o === "TIMESTAMP_MICROS")
|
|
196
|
+
return Array.from(e).map((l) => n.timestampFromMicroseconds(l));
|
|
197
|
+
if (o === "JSON")
|
|
198
|
+
return e.map((l) => n.jsonFromBytes(l));
|
|
199
|
+
if (o === "BSON")
|
|
200
|
+
throw new Error("parquet bson not supported");
|
|
201
|
+
if (o === "INTERVAL")
|
|
202
|
+
throw new Error("parquet interval not supported");
|
|
203
|
+
if (c?.type === "GEOMETRY")
|
|
204
|
+
return e.map((l) => n.geometryFromBytes(l));
|
|
205
|
+
if (c?.type === "GEOGRAPHY")
|
|
206
|
+
return e.map((l) => n.geographyFromBytes(l));
|
|
207
|
+
if (c?.type === "UUID")
|
|
208
|
+
return e.map((l) => n.uuidFromBytes(l));
|
|
209
|
+
if (o === "UTF8" || c?.type === "STRING" || r && s === "BYTE_ARRAY")
|
|
210
|
+
return e.map((l) => n.stringFromBytes(l));
|
|
211
|
+
if (o === "UINT_64" || c?.type === "INTEGER" && c.bitWidth === 64 && !c.isSigned) {
|
|
212
|
+
if (e instanceof BigInt64Array) return new BigUint64Array(e.buffer, e.byteOffset, e.length);
|
|
213
|
+
const l = d ? new Array(e.length) : new BigUint64Array(e.length);
|
|
214
|
+
for (let u = 0; u < l.length; u++) l[u] = e[u];
|
|
215
|
+
return l;
|
|
216
|
+
}
|
|
217
|
+
if (o === "UINT_32" || c?.type === "INTEGER" && c.bitWidth === 32 && !c.isSigned) {
|
|
218
|
+
if (e instanceof Int32Array) return new Uint32Array(e.buffer, e.byteOffset, e.length);
|
|
219
|
+
const l = d ? new Array(e.length) : new Uint32Array(e.length);
|
|
220
|
+
for (let u = 0; u < l.length; u++)
|
|
221
|
+
l[u] = e[u] < 0 ? 4294967296 + e[u] : e[u];
|
|
222
|
+
return l;
|
|
223
|
+
}
|
|
224
|
+
if (c?.type === "FLOAT16")
|
|
225
|
+
return Array.from(e).map(ut);
|
|
226
|
+
if (c?.type === "TIMESTAMP") {
|
|
227
|
+
const { unit: l } = c;
|
|
228
|
+
let u = n.timestampFromMilliseconds;
|
|
229
|
+
l === "MICROS" && (u = n.timestampFromMicroseconds), l === "NANOS" && (u = n.timestampFromNanoseconds);
|
|
230
|
+
const g = new Array(e.length);
|
|
231
|
+
for (let h = 0; h < g.length; h++)
|
|
232
|
+
g[h] = u(e[h]);
|
|
233
|
+
return g;
|
|
234
|
+
}
|
|
235
|
+
return e;
|
|
236
|
+
}
|
|
237
|
+
function at(e) {
|
|
238
|
+
if (!e.length) return 0;
|
|
239
|
+
let t = 0n;
|
|
240
|
+
for (const n of e)
|
|
241
|
+
t = t * 256n + BigInt(n);
|
|
242
|
+
const i = e.length * 8;
|
|
243
|
+
return t >= 2n ** BigInt(i - 1) && (t -= 2n ** BigInt(i)), Number(t);
|
|
244
|
+
}
|
|
245
|
+
function kt(e) {
|
|
246
|
+
const t = (e >> 64n) - 2440588n, i = e & 0xffffffffffffffffn;
|
|
247
|
+
return t * 86400000000000n + i;
|
|
248
|
+
}
|
|
249
|
+
function ut(e) {
|
|
250
|
+
if (!e) return;
|
|
251
|
+
const t = e[1] << 8 | e[0], i = t >> 15 ? -1 : 1, n = t >> 10 & 31, r = t & 1023;
|
|
252
|
+
return n === 0 ? i * 2 ** -14 * (r / 1024) : n === 31 ? r ? NaN : i * (1 / 0) : i * 2 ** (n - 15) * (1 + r / 1024);
|
|
253
|
+
}
|
|
254
|
+
function dt(e, t, i) {
|
|
255
|
+
const n = e[t], r = [];
|
|
256
|
+
let f = 1;
|
|
257
|
+
if (n.num_children)
|
|
258
|
+
for (; r.length < n.num_children; ) {
|
|
259
|
+
const s = e[t + f], o = dt(e, t + f, [...i, s.name]);
|
|
260
|
+
f += o.count, r.push(o);
|
|
261
|
+
}
|
|
262
|
+
return { count: f, element: n, children: r, path: i };
|
|
263
|
+
}
|
|
264
|
+
function _t(e, t) {
|
|
265
|
+
let i = dt(e, 0, []);
|
|
266
|
+
const n = [i];
|
|
267
|
+
for (const r of t) {
|
|
268
|
+
const f = i.children.find((s) => s.element.name === r);
|
|
269
|
+
if (!f) throw new Error(`parquet schema element not found: ${t}`);
|
|
270
|
+
n.push(f), i = f;
|
|
271
|
+
}
|
|
272
|
+
return n;
|
|
273
|
+
}
|
|
274
|
+
function Re(e) {
|
|
275
|
+
const t = [];
|
|
276
|
+
function i(n) {
|
|
277
|
+
if (n.children.length)
|
|
278
|
+
for (const r of n.children)
|
|
279
|
+
i(r);
|
|
280
|
+
else
|
|
281
|
+
t.push(n.path.join("."));
|
|
282
|
+
}
|
|
283
|
+
return i(e), t;
|
|
284
|
+
}
|
|
285
|
+
function gt(e) {
|
|
286
|
+
let t = 0;
|
|
287
|
+
for (const { element: i } of e)
|
|
288
|
+
i.repetition_type === "REPEATED" && t++;
|
|
289
|
+
return t;
|
|
290
|
+
}
|
|
291
|
+
function Be(e) {
|
|
292
|
+
let t = 0;
|
|
293
|
+
for (const { element: i } of e.slice(1))
|
|
294
|
+
i.repetition_type !== "REQUIRED" && t++;
|
|
295
|
+
return t;
|
|
296
|
+
}
|
|
297
|
+
function Yt(e) {
|
|
298
|
+
if (!e || e.element.converted_type !== "LIST" || e.children.length > 1) return !1;
|
|
299
|
+
const t = e.children[0];
|
|
300
|
+
return !(t.children.length > 1 || t.element.repetition_type !== "REPEATED");
|
|
301
|
+
}
|
|
302
|
+
function qt(e) {
|
|
303
|
+
if (!e || e.element.converted_type !== "MAP" || e.children.length > 1) return !1;
|
|
304
|
+
const t = e.children[0];
|
|
305
|
+
return !(t.children.length !== 2 || t.element.repetition_type !== "REPEATED" || t.children.find((r) => r.element.name === "key")?.element.repetition_type === "REPEATED" || t.children.find((r) => r.element.name === "value")?.element.repetition_type === "REPEATED");
|
|
306
|
+
}
|
|
307
|
+
function ht(e) {
|
|
308
|
+
if (e.length !== 2) return !1;
|
|
309
|
+
const [, t] = e;
|
|
310
|
+
return !(t.element.repetition_type === "REPEATED" || t.children.length);
|
|
311
|
+
}
|
|
312
|
+
const jt = 0, Fe = 1, ke = 2, Ye = 3, Gt = 4, Vt = 5, zt = 6, Wt = 7, Ht = 8, Kt = 9, Qt = 12;
|
|
313
|
+
function X(e) {
|
|
314
|
+
const t = {};
|
|
315
|
+
let i = 0;
|
|
316
|
+
for (; e.offset < e.view.byteLength; ) {
|
|
317
|
+
const n = e.view.getUint8(e.offset++), r = n & 15;
|
|
318
|
+
if (r === jt) break;
|
|
319
|
+
const f = n >> 4;
|
|
320
|
+
i = f ? i + f : yt(e), t[`field_${i}`] = Ie(e, r);
|
|
321
|
+
}
|
|
322
|
+
return t;
|
|
323
|
+
}
|
|
324
|
+
function Ie(e, t) {
|
|
325
|
+
switch (t) {
|
|
326
|
+
case Fe:
|
|
327
|
+
return !0;
|
|
328
|
+
case ke:
|
|
329
|
+
return !1;
|
|
330
|
+
case Ye:
|
|
331
|
+
return e.view.getInt8(e.offset++);
|
|
332
|
+
case Gt:
|
|
333
|
+
case Vt:
|
|
334
|
+
return yt(e);
|
|
335
|
+
case zt:
|
|
336
|
+
return be(e);
|
|
337
|
+
case Wt: {
|
|
338
|
+
const i = e.view.getFloat64(e.offset, !0);
|
|
339
|
+
return e.offset += 8, i;
|
|
340
|
+
}
|
|
341
|
+
case Ht: {
|
|
342
|
+
const i = W(e), n = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, i);
|
|
343
|
+
return e.offset += i, n;
|
|
344
|
+
}
|
|
345
|
+
case Kt: {
|
|
346
|
+
const i = e.view.getUint8(e.offset++), n = i & 15;
|
|
347
|
+
let r = i >> 4;
|
|
348
|
+
r === 15 && (r = W(e));
|
|
349
|
+
const f = n === Fe || n === ke, s = new Array(r);
|
|
350
|
+
for (let o = 0; o < r; o++)
|
|
351
|
+
s[o] = f ? Ie(e, Ye) === 1 : Ie(e, n);
|
|
352
|
+
return s;
|
|
353
|
+
}
|
|
354
|
+
case Qt:
|
|
355
|
+
return X(e);
|
|
356
|
+
default:
|
|
357
|
+
throw new Error(`thrift unhandled type: ${t}`);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
function W(e) {
|
|
361
|
+
let t = 0, i = 0;
|
|
362
|
+
for (; ; ) {
|
|
363
|
+
const n = e.view.getUint8(e.offset++);
|
|
364
|
+
if (t |= (n & 127) << i, !(n & 128))
|
|
365
|
+
return t;
|
|
366
|
+
i += 7;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
function Zt(e) {
|
|
370
|
+
let t = 0n, i = 0n;
|
|
371
|
+
for (; ; ) {
|
|
372
|
+
const n = e.view.getUint8(e.offset++);
|
|
373
|
+
if (t |= BigInt(n & 127) << i, !(n & 128))
|
|
374
|
+
return t;
|
|
375
|
+
i += 7n;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
function yt(e) {
|
|
379
|
+
const t = W(e);
|
|
380
|
+
return t >>> 1 ^ -(t & 1);
|
|
381
|
+
}
|
|
382
|
+
function be(e) {
|
|
383
|
+
const t = Zt(e);
|
|
384
|
+
return t >> 1n ^ -(t & 1n);
|
|
385
|
+
}
|
|
386
|
+
function Jt(e, t) {
|
|
387
|
+
const i = /* @__PURE__ */ new Map(), n = t?.find(({ key: f }) => f === "geo")?.value, r = (n && JSON.parse(n)?.columns) ?? {};
|
|
388
|
+
for (const [f, s] of Object.entries(r)) {
|
|
389
|
+
if (s.encoding !== "WKB") continue;
|
|
390
|
+
const o = s.edges === "spherical" ? "GEOGRAPHY" : "GEOMETRY", c = s.crs?.id ?? s.crs?.ids?.[0], d = c ? `${c.authority}:${c.code.toString()}` : void 0;
|
|
391
|
+
i.set(f, { type: o, crs: d });
|
|
392
|
+
}
|
|
393
|
+
for (let f = 1; f < e.length; f++) {
|
|
394
|
+
const { logical_type: s, name: o, num_children: c, type: d } = e[f];
|
|
395
|
+
if (c) {
|
|
396
|
+
f += c;
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
d === "BYTE_ARRAY" && !s && (e[f].logical_type = i.get(o));
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
const mt = 1 << 19, Xt = new TextDecoder();
|
|
403
|
+
function D(e) {
|
|
404
|
+
return e && Xt.decode(e);
|
|
405
|
+
}
|
|
406
|
+
async function he(e, { parsers: t, initialFetchSize: i = mt, geoparquet: n = !0 } = {}) {
|
|
407
|
+
if (!e || !(e.byteLength >= 0)) throw new Error("parquet expected AsyncBuffer");
|
|
408
|
+
const r = Math.max(0, e.byteLength - i), f = await e.slice(r, e.byteLength), s = new DataView(f);
|
|
409
|
+
if (s.getUint32(f.byteLength - 4, !0) !== 827474256)
|
|
410
|
+
throw new Error("parquet file invalid (footer != PAR1)");
|
|
411
|
+
const o = s.getUint32(f.byteLength - 8, !0);
|
|
412
|
+
if (o > e.byteLength - 8)
|
|
413
|
+
throw new Error(`parquet metadata length ${o} exceeds available buffer ${e.byteLength - 8}`);
|
|
414
|
+
if (o + 8 > i) {
|
|
415
|
+
const c = e.byteLength - o - 8, d = await e.slice(c, r), a = new ArrayBuffer(o + 8), l = new Uint8Array(a);
|
|
416
|
+
return l.set(new Uint8Array(d)), l.set(new Uint8Array(f), r - c), qe(a, { parsers: t, geoparquet: n });
|
|
417
|
+
} else
|
|
418
|
+
return qe(f, { parsers: t, geoparquet: n });
|
|
419
|
+
}
|
|
420
|
+
function qe(e, { parsers: t, geoparquet: i = !0 } = {}) {
|
|
421
|
+
if (!(e instanceof ArrayBuffer)) throw new Error("parquet expected ArrayBuffer");
|
|
422
|
+
const n = new DataView(e);
|
|
423
|
+
if (t = { ...oe, ...t }, n.byteLength < 8)
|
|
424
|
+
throw new Error("parquet file is too short");
|
|
425
|
+
if (n.getUint32(n.byteLength - 4, !0) !== 827474256)
|
|
426
|
+
throw new Error("parquet file invalid (footer != PAR1)");
|
|
427
|
+
const r = n.byteLength - 8, f = n.getUint32(r, !0);
|
|
428
|
+
if (f > n.byteLength - 8)
|
|
429
|
+
throw new Error(`parquet metadata length ${f} exceeds available buffer ${n.byteLength - 8}`);
|
|
430
|
+
const s = r - f, c = X({ view: n, offset: s }), d = c.field_1, a = c.field_2.map((y) => ({
|
|
431
|
+
type: Ue[y.field_1],
|
|
432
|
+
type_length: y.field_2,
|
|
433
|
+
repetition_type: Ut[y.field_3],
|
|
434
|
+
name: D(y.field_4),
|
|
435
|
+
num_children: y.field_5,
|
|
436
|
+
converted_type: Pt[y.field_6],
|
|
437
|
+
scale: y.field_7,
|
|
438
|
+
precision: y.field_8,
|
|
439
|
+
field_id: y.field_9,
|
|
440
|
+
logical_type: en(y.field_10)
|
|
441
|
+
})), l = a.filter((y) => y.type), u = c.field_3, g = c.field_4.map((y) => ({
|
|
442
|
+
columns: y.field_1.map((_, w) => ({
|
|
443
|
+
file_path: D(_.field_1),
|
|
444
|
+
file_offset: _.field_2,
|
|
445
|
+
meta_data: _.field_3 && {
|
|
446
|
+
type: Ue[_.field_3.field_1],
|
|
447
|
+
encodings: _.field_3.field_2?.map((p) => G[p]),
|
|
448
|
+
path_in_schema: _.field_3.field_3.map(D),
|
|
449
|
+
codec: $t[_.field_3.field_4],
|
|
450
|
+
num_values: _.field_3.field_5,
|
|
451
|
+
total_uncompressed_size: _.field_3.field_6,
|
|
452
|
+
total_compressed_size: _.field_3.field_7,
|
|
453
|
+
key_value_metadata: _.field_3.field_8?.map((p) => ({
|
|
454
|
+
key: D(p.field_1),
|
|
455
|
+
value: D(p.field_2)
|
|
456
|
+
})),
|
|
457
|
+
data_page_offset: _.field_3.field_9,
|
|
458
|
+
index_page_offset: _.field_3.field_10,
|
|
459
|
+
dictionary_page_offset: _.field_3.field_11,
|
|
460
|
+
statistics: tn(_.field_3.field_12, l[w], t),
|
|
461
|
+
encoding_stats: _.field_3.field_13?.map((p) => ({
|
|
462
|
+
page_type: lt[p.field_1],
|
|
463
|
+
encoding: G[p.field_2],
|
|
464
|
+
count: p.field_3
|
|
465
|
+
})),
|
|
466
|
+
bloom_filter_offset: _.field_3.field_14,
|
|
467
|
+
bloom_filter_length: _.field_3.field_15,
|
|
468
|
+
size_statistics: _.field_3.field_16 && {
|
|
469
|
+
unencoded_byte_array_data_bytes: _.field_3.field_16.field_1,
|
|
470
|
+
repetition_level_histogram: _.field_3.field_16.field_2,
|
|
471
|
+
definition_level_histogram: _.field_3.field_16.field_3
|
|
472
|
+
},
|
|
473
|
+
geospatial_statistics: _.field_3.field_17 && {
|
|
474
|
+
bbox: _.field_3.field_17.field_1 && {
|
|
475
|
+
xmin: _.field_3.field_17.field_1.field_1,
|
|
476
|
+
xmax: _.field_3.field_17.field_1.field_2,
|
|
477
|
+
ymin: _.field_3.field_17.field_1.field_3,
|
|
478
|
+
ymax: _.field_3.field_17.field_1.field_4,
|
|
479
|
+
zmin: _.field_3.field_17.field_1.field_5,
|
|
480
|
+
zmax: _.field_3.field_17.field_1.field_6,
|
|
481
|
+
mmin: _.field_3.field_17.field_1.field_7,
|
|
482
|
+
mmax: _.field_3.field_17.field_1.field_8
|
|
483
|
+
},
|
|
484
|
+
geospatial_types: _.field_3.field_17.field_2
|
|
485
|
+
}
|
|
486
|
+
},
|
|
487
|
+
offset_index_offset: _.field_4,
|
|
488
|
+
offset_index_length: _.field_5,
|
|
489
|
+
column_index_offset: _.field_6,
|
|
490
|
+
column_index_length: _.field_7,
|
|
491
|
+
crypto_metadata: _.field_8,
|
|
492
|
+
encrypted_column_metadata: _.field_9
|
|
493
|
+
})),
|
|
494
|
+
total_byte_size: y.field_2,
|
|
495
|
+
num_rows: y.field_3,
|
|
496
|
+
sorting_columns: y.field_4?.map((_) => ({
|
|
497
|
+
column_idx: _.field_1,
|
|
498
|
+
descending: _.field_2,
|
|
499
|
+
nulls_first: _.field_3
|
|
500
|
+
})),
|
|
501
|
+
file_offset: y.field_5,
|
|
502
|
+
total_compressed_size: y.field_6,
|
|
503
|
+
ordinal: y.field_7
|
|
504
|
+
})), h = c.field_5?.map((y) => ({
|
|
505
|
+
key: D(y.field_1),
|
|
506
|
+
value: D(y.field_2)
|
|
507
|
+
})), m = D(c.field_6);
|
|
508
|
+
return i && Jt(a, h), {
|
|
509
|
+
version: d,
|
|
510
|
+
schema: a,
|
|
511
|
+
num_rows: u,
|
|
512
|
+
row_groups: g,
|
|
513
|
+
key_value_metadata: h,
|
|
514
|
+
created_by: m,
|
|
515
|
+
metadata_length: f
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
function F({ schema: e }) {
|
|
519
|
+
return _t(e, [])[0];
|
|
520
|
+
}
|
|
521
|
+
function en(e) {
|
|
522
|
+
return e?.field_1 ? { type: "STRING" } : e?.field_2 ? { type: "MAP" } : e?.field_3 ? { type: "LIST" } : e?.field_4 ? { type: "ENUM" } : e?.field_5 ? {
|
|
523
|
+
type: "DECIMAL",
|
|
524
|
+
scale: e.field_5.field_1,
|
|
525
|
+
precision: e.field_5.field_2
|
|
526
|
+
} : e?.field_6 ? { type: "DATE" } : e?.field_7 ? {
|
|
527
|
+
type: "TIME",
|
|
528
|
+
isAdjustedToUTC: e.field_7.field_1,
|
|
529
|
+
unit: je(e.field_7.field_2)
|
|
530
|
+
} : e?.field_8 ? {
|
|
531
|
+
type: "TIMESTAMP",
|
|
532
|
+
isAdjustedToUTC: e.field_8.field_1,
|
|
533
|
+
unit: je(e.field_8.field_2)
|
|
534
|
+
} : e?.field_10 ? {
|
|
535
|
+
type: "INTEGER",
|
|
536
|
+
bitWidth: e.field_10.field_1,
|
|
537
|
+
isSigned: e.field_10.field_2
|
|
538
|
+
} : e?.field_11 ? { type: "NULL" } : e?.field_12 ? { type: "JSON" } : e?.field_13 ? { type: "BSON" } : e?.field_14 ? { type: "UUID" } : e?.field_15 ? { type: "FLOAT16" } : e?.field_16 ? {
|
|
539
|
+
type: "VARIANT",
|
|
540
|
+
specification_version: e.field_16.field_1
|
|
541
|
+
} : e?.field_17 ? {
|
|
542
|
+
type: "GEOMETRY",
|
|
543
|
+
crs: D(e.field_17.field_1)
|
|
544
|
+
} : e?.field_18 ? {
|
|
545
|
+
type: "GEOGRAPHY",
|
|
546
|
+
crs: D(e.field_18.field_1),
|
|
547
|
+
algorithm: Ft[e.field_18.field_2]
|
|
548
|
+
} : e;
|
|
549
|
+
}
|
|
550
|
+
function je(e) {
|
|
551
|
+
if (e.field_1) return "MILLIS";
|
|
552
|
+
if (e.field_2) return "MICROS";
|
|
553
|
+
if (e.field_3) return "NANOS";
|
|
554
|
+
throw new Error("parquet time unit required");
|
|
555
|
+
}
|
|
556
|
+
function tn(e, t, i) {
|
|
557
|
+
return e && {
|
|
558
|
+
max: J(e.field_1, t, i),
|
|
559
|
+
min: J(e.field_2, t, i),
|
|
560
|
+
null_count: e.field_3,
|
|
561
|
+
distinct_count: e.field_4,
|
|
562
|
+
max_value: J(e.field_5, t, i),
|
|
563
|
+
min_value: J(e.field_6, t, i),
|
|
564
|
+
is_max_value_exact: e.field_7,
|
|
565
|
+
is_min_value_exact: e.field_8
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
function J(e, t, i) {
|
|
569
|
+
const { type: n, converted_type: r, logical_type: f } = t;
|
|
570
|
+
if (e === void 0) return e;
|
|
571
|
+
if (n === "BOOLEAN") return e[0] === 1;
|
|
572
|
+
if (n === "BYTE_ARRAY") return i.stringFromBytes(e);
|
|
573
|
+
const s = new DataView(e.buffer, e.byteOffset, e.byteLength);
|
|
574
|
+
if (n === "FLOAT" && s.byteLength === 4) return s.getFloat32(0, !0);
|
|
575
|
+
if (n === "DOUBLE" && s.byteLength === 8) return s.getFloat64(0, !0);
|
|
576
|
+
if (n === "INT32" && r === "DECIMAL" && s.byteLength === 4)
|
|
577
|
+
return s.getInt32(0, !0) * 10 ** -(t.scale || 0);
|
|
578
|
+
if (n === "INT64" && r === "DECIMAL" && s.byteLength === 8)
|
|
579
|
+
return Number(s.getBigInt64(0, !0)) * 10 ** -(t.scale || 0);
|
|
580
|
+
if (n === "INT32" && r === "DATE") return i.dateFromDays(s.getInt32(0, !0));
|
|
581
|
+
if (n === "INT64" && r === "TIMESTAMP_MILLIS") return i.timestampFromMilliseconds(s.getBigInt64(0, !0));
|
|
582
|
+
if (n === "INT64" && r === "TIMESTAMP_MICROS") return i.timestampFromMicroseconds(s.getBigInt64(0, !0));
|
|
583
|
+
if (n === "INT64" && f?.type === "TIMESTAMP" && f?.unit === "NANOS") return i.timestampFromNanoseconds(s.getBigInt64(0, !0));
|
|
584
|
+
if (n === "INT64" && f?.type === "TIMESTAMP" && f?.unit === "MICROS") return i.timestampFromMicroseconds(s.getBigInt64(0, !0));
|
|
585
|
+
if (n === "INT64" && f?.type === "TIMESTAMP") return i.timestampFromMilliseconds(s.getBigInt64(0, !0));
|
|
586
|
+
const o = r?.startsWith("UINT_") || f?.type === "INTEGER" && !f.isSigned;
|
|
587
|
+
return n === "INT32" && o && s.byteLength === 4 ? s.getUint32(0, !0) : n === "INT64" && o && s.byteLength === 8 ? s.getBigUint64(0, !0) : n === "INT32" && s.byteLength === 4 ? s.getInt32(0, !0) : n === "INT64" && s.byteLength === 8 ? s.getBigInt64(0, !0) : r === "DECIMAL" ? at(e) * 10 ** -(t.scale || 0) : f?.type === "FLOAT16" ? ut(e) : f?.type === "UUID" ? i.uuidFromBytes(e) : e;
|
|
588
|
+
}
|
|
589
|
+
function nn(e, t, i = void 0) {
|
|
590
|
+
i = { ...oe, ...i };
|
|
591
|
+
const n = X(e);
|
|
592
|
+
return {
|
|
593
|
+
null_pages: n.field_1,
|
|
594
|
+
min_values: n.field_2.map((r) => J(r, t, i)),
|
|
595
|
+
max_values: n.field_3.map((r) => J(r, t, i)),
|
|
596
|
+
boundary_order: Ct[n.field_4],
|
|
597
|
+
null_counts: n.field_5,
|
|
598
|
+
repetition_level_histograms: n.field_6,
|
|
599
|
+
definition_level_histograms: n.field_7
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
function ve(e) {
|
|
603
|
+
const t = X(e);
|
|
604
|
+
return {
|
|
605
|
+
// @ts-ignore
|
|
606
|
+
page_locations: t.field_1.map((i) => ({
|
|
607
|
+
offset: i.field_1,
|
|
608
|
+
compressed_page_size: i.field_2,
|
|
609
|
+
first_row_index: i.field_3
|
|
610
|
+
})),
|
|
611
|
+
unencoded_byte_array_data_bytes: t.field_2
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
const L = 0xffffffffffffffffn, V = 0x9e3779b185ebca87n, ee = 0xc2b2ae3d27d4eb4fn, Ge = 0x165667b19e3779f9n, wt = 0x85ebca77c2b2ae63n, Ve = 0x27d4eb2f165667c5n;
|
|
615
|
+
function C(e, t) {
|
|
616
|
+
return (e << t | e >> 64n - t) & L;
|
|
617
|
+
}
|
|
618
|
+
function Z(e, t) {
|
|
619
|
+
return e = e + t * ee & L, e = C(e, 31n), e * V & L;
|
|
620
|
+
}
|
|
621
|
+
function ae(e, t) {
|
|
622
|
+
return e ^= Z(0n, t), e * V + wt & L;
|
|
623
|
+
}
|
|
624
|
+
function $(e, t = 0n) {
|
|
625
|
+
const i = new DataView(e.buffer, e.byteOffset, e.byteLength), n = e.byteLength;
|
|
626
|
+
let r = 0, f;
|
|
627
|
+
if (n >= 32) {
|
|
628
|
+
let s = t + V + ee & L, o = t + ee & L, c = t, d = t - V & L;
|
|
629
|
+
for (; r + 32 <= n; )
|
|
630
|
+
s = Z(s, i.getBigUint64(r, !0)), r += 8, o = Z(o, i.getBigUint64(r, !0)), r += 8, c = Z(c, i.getBigUint64(r, !0)), r += 8, d = Z(d, i.getBigUint64(r, !0)), r += 8;
|
|
631
|
+
f = C(s, 1n) + C(o, 7n) + C(c, 12n) + C(d, 18n) & L, f = ae(f, s), f = ae(f, o), f = ae(f, c), f = ae(f, d);
|
|
632
|
+
} else
|
|
633
|
+
f = t + Ve & L;
|
|
634
|
+
for (f = f + BigInt(n) & L; r + 8 <= n; )
|
|
635
|
+
f ^= Z(0n, i.getBigUint64(r, !0)), f = C(f, 27n) * V + wt & L, r += 8;
|
|
636
|
+
for (r + 4 <= n && (f ^= BigInt(i.getUint32(r, !0)) * V & L, f = C(f, 23n) * ee + Ge & L, r += 4); r < n; )
|
|
637
|
+
f ^= BigInt(i.getUint8(r)) * Ve & L, f = C(f, 11n) * V & L, r += 1;
|
|
638
|
+
return f ^= f >> 33n, f = f * ee & L, f ^= f >> 29n, f = f * Ge & L, f ^= f >> 32n, f;
|
|
639
|
+
}
|
|
640
|
+
const rn = new TextEncoder(), fn = new Uint32Array([
|
|
641
|
+
1203114875,
|
|
642
|
+
1150766481,
|
|
643
|
+
2284105051,
|
|
644
|
+
2729912477,
|
|
645
|
+
1884591559,
|
|
646
|
+
770785867,
|
|
647
|
+
2667333959,
|
|
648
|
+
1550580529
|
|
649
|
+
]);
|
|
650
|
+
function sn(e, t) {
|
|
651
|
+
return Number((e >> 32n) * BigInt(t) >> 32n);
|
|
652
|
+
}
|
|
653
|
+
function on(e) {
|
|
654
|
+
const t = new Uint32Array(8), i = Number(e & 0xffffffffn) | 0;
|
|
655
|
+
for (let n = 0; n < 8; n++)
|
|
656
|
+
t[n] = 1 << (Math.imul(i, fn[n]) >>> 27);
|
|
657
|
+
return t;
|
|
658
|
+
}
|
|
659
|
+
function ze(e, t) {
|
|
660
|
+
const i = sn(t, e.length >> 3) << 3, n = on(t);
|
|
661
|
+
for (let r = 0; r < 8; r++)
|
|
662
|
+
if ((e[i + r] & n[r]) === 0) return !1;
|
|
663
|
+
return !0;
|
|
664
|
+
}
|
|
665
|
+
function ln(e) {
|
|
666
|
+
const t = X(e), i = t.field_1;
|
|
667
|
+
if (typeof i != "number" || i <= 0 || i % 32 !== 0 || !t.field_2?.field_1 || !t.field_3?.field_1 || !t.field_4?.field_1) return;
|
|
668
|
+
const { view: n, offset: r } = e;
|
|
669
|
+
if (r + i > n.byteLength)
|
|
670
|
+
throw new Error(`parquet bloom filter truncated: need ${i} bytes, have ${n.byteLength - r}`);
|
|
671
|
+
const f = new Uint32Array(i >> 2);
|
|
672
|
+
for (let s = 0; s < f.length; s++)
|
|
673
|
+
f[s] = n.getUint32(r + s * 4, !0);
|
|
674
|
+
return e.offset = r + i, { numBytes: i, blocks: f };
|
|
675
|
+
}
|
|
676
|
+
function We(e, t) {
|
|
677
|
+
if (e == null) return;
|
|
678
|
+
const { type: i, converted_type: n, logical_type: r } = t;
|
|
679
|
+
if (i === "BOOLEAN")
|
|
680
|
+
return typeof e != "boolean" ? void 0 : $(new Uint8Array([e ? 1 : 0]));
|
|
681
|
+
if (i === "FLOAT") {
|
|
682
|
+
if (typeof e != "number") return;
|
|
683
|
+
const f = new ArrayBuffer(4);
|
|
684
|
+
return new DataView(f).setFloat32(0, e, !0), $(new Uint8Array(f));
|
|
685
|
+
}
|
|
686
|
+
if (i === "DOUBLE") {
|
|
687
|
+
if (typeof e != "number") return;
|
|
688
|
+
const f = new ArrayBuffer(8);
|
|
689
|
+
return new DataView(f).setFloat64(0, e, !0), $(new Uint8Array(f));
|
|
690
|
+
}
|
|
691
|
+
if (i === "INT32") {
|
|
692
|
+
if (n === "DATE" || n === "DECIMAL" || n === "TIME_MILLIS" || r?.type === "DATE" || r?.type === "TIME" || r?.type === "DECIMAL" || typeof e != "number" || !Number.isInteger(e)) return;
|
|
693
|
+
const f = new ArrayBuffer(4);
|
|
694
|
+
return new DataView(f).setInt32(0, e | 0, !0), $(new Uint8Array(f));
|
|
695
|
+
}
|
|
696
|
+
if (i === "INT64") {
|
|
697
|
+
if (n === "TIMESTAMP_MILLIS" || n === "TIMESTAMP_MICROS" || n === "TIME_MICROS" || n === "DECIMAL" || r?.type === "TIMESTAMP" || r?.type === "TIME" || r?.type === "DECIMAL") return;
|
|
698
|
+
let f;
|
|
699
|
+
if (typeof e == "bigint") f = e;
|
|
700
|
+
else if (typeof e == "number" && Number.isSafeInteger(e)) f = BigInt(e);
|
|
701
|
+
else return;
|
|
702
|
+
const s = new ArrayBuffer(8);
|
|
703
|
+
return new DataView(s).setBigUint64(0, BigInt.asUintN(64, f), !0), $(new Uint8Array(s));
|
|
704
|
+
}
|
|
705
|
+
if (i === "BYTE_ARRAY")
|
|
706
|
+
return n === "JSON" || n === "BSON" || n === "DECIMAL" || r?.type === "JSON" || r?.type === "BSON" || r?.type === "VARIANT" || r?.type === "GEOMETRY" || r?.type === "GEOGRAPHY" ? void 0 : typeof e == "string" ? $(rn.encode(e)) : e instanceof Uint8Array ? $(e) : void 0;
|
|
707
|
+
if (i === "FIXED_LEN_BYTE_ARRAY")
|
|
708
|
+
return n === "DECIMAL" || n === "INTERVAL" || r?.type === "DECIMAL" || r?.type === "UUID" || r?.type === "FLOAT16" || r?.type === "GEOMETRY" || r?.type === "GEOGRAPHY" ? void 0 : e instanceof Uint8Array ? $(e) : void 0;
|
|
709
|
+
}
|
|
710
|
+
function cn(e) {
|
|
711
|
+
const t = /* @__PURE__ */ new Set();
|
|
712
|
+
return Te(e, t), t;
|
|
713
|
+
}
|
|
714
|
+
function Te(e, t) {
|
|
715
|
+
if (e) {
|
|
716
|
+
if ("$and" in e && Array.isArray(e.$and)) {
|
|
717
|
+
for (const i of e.$and) Te(i, t);
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
if ("$or" in e && Array.isArray(e.$or)) {
|
|
721
|
+
for (const i of e.$or) Te(i, t);
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
724
|
+
if (!("$nor" in e))
|
|
725
|
+
for (const [i, n] of Object.entries(e))
|
|
726
|
+
i.startsWith("$") || (typeof n == "object" && n !== null && !Array.isArray(n) ? ("$eq" in n || "$in" in n) && t.add(i) : t.add(i));
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
function He(e) {
|
|
730
|
+
if (e === void 0) return null;
|
|
731
|
+
if (typeof e == "bigint") return Number(e);
|
|
732
|
+
if (Object.is(e, -0)) return 0;
|
|
733
|
+
if (Array.isArray(e)) return e.map(He);
|
|
734
|
+
if (e instanceof Uint8Array) return Array.from(e);
|
|
735
|
+
if (e instanceof Date) return e.toISOString();
|
|
736
|
+
if (e instanceof Object) {
|
|
737
|
+
const t = {};
|
|
738
|
+
for (const i of Object.keys(e))
|
|
739
|
+
e[i] !== void 0 && (t[i] = He(e[i]));
|
|
740
|
+
return t;
|
|
741
|
+
}
|
|
742
|
+
return e;
|
|
743
|
+
}
|
|
744
|
+
function xe(e, t) {
|
|
745
|
+
for (let n = 0; n < t.length; n += 1e4)
|
|
746
|
+
e.push(...t.slice(n, n + 1e4));
|
|
747
|
+
}
|
|
748
|
+
function P(e, t, i = !0) {
|
|
749
|
+
if (i ? e === t : e == t) return !0;
|
|
750
|
+
if (!e || !t || typeof e != "object" || typeof t != "object") return !1;
|
|
751
|
+
if (e instanceof Uint8Array && t instanceof Uint8Array) {
|
|
752
|
+
if (e.length !== t.length) return !1;
|
|
753
|
+
for (let r = 0; r < e.length; r++)
|
|
754
|
+
if (e[r] !== t[r]) return !1;
|
|
755
|
+
return !0;
|
|
756
|
+
}
|
|
757
|
+
if (e instanceof Date || t instanceof Date)
|
|
758
|
+
return e instanceof Date && t instanceof Date && e.getTime() === t.getTime();
|
|
759
|
+
if (Array.isArray(e) && Array.isArray(t)) {
|
|
760
|
+
if (e.length !== t.length) return !1;
|
|
761
|
+
for (let r = 0; r < e.length; r++)
|
|
762
|
+
if (!P(e[r], t[r], i)) return !1;
|
|
763
|
+
return !0;
|
|
764
|
+
}
|
|
765
|
+
const n = Object.keys(e);
|
|
766
|
+
if (n.length !== Object.keys(t).length) return !1;
|
|
767
|
+
for (const r of n)
|
|
768
|
+
if (!P(e[r], t[r], i)) return !1;
|
|
769
|
+
return !0;
|
|
770
|
+
}
|
|
771
|
+
async function Ke(e, t = {}, i = globalThis.fetch) {
|
|
772
|
+
const n = new AbortController(), r = new Headers(t.headers);
|
|
773
|
+
r.set("Range", "bytes=0-0");
|
|
774
|
+
const f = await i(e, {
|
|
775
|
+
...t,
|
|
776
|
+
headers: r,
|
|
777
|
+
signal: n.signal
|
|
778
|
+
});
|
|
779
|
+
if (!f.ok) throw new Error(`fetch with range failed ${f.status}`);
|
|
780
|
+
if (f.status === 206) {
|
|
781
|
+
const s = f.headers.get("Content-Range");
|
|
782
|
+
if (!s) throw new Error("missing content-range header");
|
|
783
|
+
const o = s.match(/bytes \d+-\d+\/(\d+)/);
|
|
784
|
+
if (!o) throw new Error(`invalid content-range header: ${s}`);
|
|
785
|
+
return parseInt(o[1]);
|
|
786
|
+
}
|
|
787
|
+
if (f.status === 200) {
|
|
788
|
+
const s = f.headers.get("Content-Length");
|
|
789
|
+
if (n.abort(), s) return parseInt(s);
|
|
790
|
+
}
|
|
791
|
+
throw new Error("server does not support range requests and missing content-length");
|
|
792
|
+
}
|
|
793
|
+
async function an(e, t, i) {
|
|
794
|
+
const n = i ?? globalThis.fetch, r = await n(e, { ...t, method: "HEAD" });
|
|
795
|
+
if (r.status === 403)
|
|
796
|
+
return Ke(e, t, n);
|
|
797
|
+
if (!r.ok) throw new Error(`fetch head failed ${r.status}`);
|
|
798
|
+
const f = r.headers.get("Content-Length");
|
|
799
|
+
return f ? parseInt(f) : Ke(e, t, n);
|
|
800
|
+
}
|
|
801
|
+
async function Jn({ url: e, byteLength: t, requestInit: i, fetch: n }) {
|
|
802
|
+
if (!e) throw new Error("missing url");
|
|
803
|
+
const r = n ?? globalThis.fetch;
|
|
804
|
+
t ??= await an(e, i, r);
|
|
805
|
+
let f;
|
|
806
|
+
const s = i || {};
|
|
807
|
+
return {
|
|
808
|
+
byteLength: t,
|
|
809
|
+
async slice(o, c) {
|
|
810
|
+
if (f)
|
|
811
|
+
return f.then((u) => u.slice(o, c));
|
|
812
|
+
const d = new Headers(s.headers), a = c === void 0 ? "" : c - 1;
|
|
813
|
+
d.set("Range", `bytes=${o}-${a}`);
|
|
814
|
+
const l = await r(e, { ...s, headers: d });
|
|
815
|
+
if (!l.ok || !l.body) throw new Error(`fetch failed ${l.status}`);
|
|
816
|
+
if (l.status === 200)
|
|
817
|
+
return f = l.arrayBuffer(), f.then((u) => u.slice(o, c));
|
|
818
|
+
if (l.status === 206)
|
|
819
|
+
return l.arrayBuffer();
|
|
820
|
+
throw new Error(`fetch received unexpected status code ${l.status}`);
|
|
821
|
+
}
|
|
822
|
+
};
|
|
823
|
+
}
|
|
824
|
+
function Xn({ byteLength: e, slice: t }, { minSize: i = mt } = {}) {
|
|
825
|
+
if (e < i) {
|
|
826
|
+
const r = t(0, e);
|
|
827
|
+
return {
|
|
828
|
+
byteLength: e,
|
|
829
|
+
async slice(f, s) {
|
|
830
|
+
return (await r).slice(f, s);
|
|
831
|
+
}
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
const n = /* @__PURE__ */ new Map();
|
|
835
|
+
return {
|
|
836
|
+
byteLength: e,
|
|
837
|
+
/**
|
|
838
|
+
* @param {number} start
|
|
839
|
+
* @param {number} [end]
|
|
840
|
+
* @returns {Awaitable<ArrayBuffer>}
|
|
841
|
+
*/
|
|
842
|
+
slice(r, f) {
|
|
843
|
+
const s = un(r, f, e), o = n.get(s);
|
|
844
|
+
if (o) return o;
|
|
845
|
+
const c = t(r, f);
|
|
846
|
+
return n.set(s, c), c;
|
|
847
|
+
}
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
function un(e, t, i) {
|
|
851
|
+
if (e < 0) {
|
|
852
|
+
if (t !== void 0) throw new Error(`invalid suffix range [${e}, ${t}]`);
|
|
853
|
+
return i === void 0 ? `${e},` : `${i + e},${i}`;
|
|
854
|
+
} else if (t !== void 0) {
|
|
855
|
+
if (e > t) throw new Error(`invalid empty range [${e}, ${t}]`);
|
|
856
|
+
return `${e},${t}`;
|
|
857
|
+
} else return i === void 0 ? `${e},` : `${e},${i}`;
|
|
858
|
+
}
|
|
859
|
+
function pt(e) {
|
|
860
|
+
if (!e) return [];
|
|
861
|
+
if (e.length === 1) return e[0];
|
|
862
|
+
const t = [];
|
|
863
|
+
for (const i of e)
|
|
864
|
+
xe(t, i);
|
|
865
|
+
return t;
|
|
866
|
+
}
|
|
867
|
+
const Qe = new TextEncoder();
|
|
868
|
+
function ie(e) {
|
|
869
|
+
if (!e) return [];
|
|
870
|
+
const t = [];
|
|
871
|
+
return "$and" in e && Array.isArray(e.$and) ? t.push(...e.$and.flatMap(ie)) : "$or" in e && Array.isArray(e.$or) ? t.push(...e.$or.flatMap(ie)) : "$nor" in e && Array.isArray(e.$nor) ? t.push(...e.$nor.flatMap(ie)) : t.push(...Object.keys(e)), [...new Set(t)];
|
|
872
|
+
}
|
|
873
|
+
function dn(e) {
|
|
874
|
+
return [...new Set(ie(e).map((t) => t.split(".")[0]))];
|
|
875
|
+
}
|
|
876
|
+
function z(e, t, i = !0) {
|
|
877
|
+
return "$and" in t && Array.isArray(t.$and) ? t.$and.every((n) => z(e, n, i)) : "$or" in t && Array.isArray(t.$or) ? t.$or.some((n) => z(e, n, i)) : "$nor" in t && Array.isArray(t.$nor) ? !t.$nor.some((n) => z(e, n, i)) : Object.entries(t).every(([n, r]) => {
|
|
878
|
+
const f = hn(e, n);
|
|
879
|
+
return typeof r != "object" || r === null || Array.isArray(r) ? P(f, r, i) : Object.entries(r || {}).every(([s, o]) => s === "$gt" ? f > o : s === "$gte" ? f >= o : s === "$lt" ? f < o : s === "$lte" ? f <= o : s === "$eq" ? P(f, o, i) : s === "$ne" ? !P(f, o, i) : s === "$in" ? Array.isArray(o) && o.includes(f) : s === "$nin" ? Array.isArray(o) && !o.includes(f) : s === "$not" ? !z({ [n]: f }, { [n]: o }, i) : !0);
|
|
880
|
+
});
|
|
881
|
+
}
|
|
882
|
+
function re({ rowGroup: e, physicalColumns: t, filter: i, strict: n = !0, bloomFilters: r, schemaElements: f }) {
|
|
883
|
+
if (!i) return !1;
|
|
884
|
+
if ("$and" in i && Array.isArray(i.$and))
|
|
885
|
+
return i.$and.some((s) => re({ rowGroup: e, physicalColumns: t, filter: s, strict: n, bloomFilters: r, schemaElements: f }));
|
|
886
|
+
if ("$or" in i && Array.isArray(i.$or))
|
|
887
|
+
return i.$or.every((s) => re({ rowGroup: e, physicalColumns: t, filter: s, strict: n, bloomFilters: r, schemaElements: f }));
|
|
888
|
+
if ("$nor" in i && Array.isArray(i.$nor))
|
|
889
|
+
return !1;
|
|
890
|
+
for (const [s, o] of Object.entries(i)) {
|
|
891
|
+
const c = t.indexOf(s);
|
|
892
|
+
if (c === -1) continue;
|
|
893
|
+
const d = e.columns[c].meta_data?.statistics, { min: a, max: l, min_value: u, max_value: g, null_count: h } = d || {}, m = u !== void 0 ? u : a, y = g !== void 0 ? g : l, _ = m !== void 0 && y !== void 0, w = r?.[s], p = f?.[s], I = z({ value: null }, { value: o }, n) && (h === void 0 || h > 0);
|
|
894
|
+
if (_ && !I && At(o, m, y, n, p))
|
|
895
|
+
return !0;
|
|
896
|
+
for (const [A, R] of Object.entries(o || {}))
|
|
897
|
+
if (w && p) {
|
|
898
|
+
if (A === "$eq") {
|
|
899
|
+
const B = We(R, p);
|
|
900
|
+
if (B !== void 0 && !ze(w.blocks, B)) return !0;
|
|
901
|
+
}
|
|
902
|
+
if (A === "$in" && Array.isArray(R) && R.length > 0) {
|
|
903
|
+
let B = !0;
|
|
904
|
+
for (const K of R) {
|
|
905
|
+
const M = We(K, p);
|
|
906
|
+
if (M === void 0 || ze(w.blocks, M)) {
|
|
907
|
+
B = !1;
|
|
908
|
+
break;
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
if (B) return !0;
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
return !1;
|
|
916
|
+
}
|
|
917
|
+
function At(e, t, i, n, r) {
|
|
918
|
+
if (t === void 0 || i === void 0) return !1;
|
|
919
|
+
const f = r?.type === "FLOAT" || r?.type === "DOUBLE" || r?.logical_type?.type === "FLOAT16";
|
|
920
|
+
for (const [s, o] of Object.entries(e || {})) {
|
|
921
|
+
const c = Q(t, o, n, r), d = Q(i, o, n, r), l = !(t instanceof Uint8Array || i instanceof Uint8Array) && (r?.type !== "BYTE_ARRAY" || typeof o == "string" && [...o].every((u) => u.charCodeAt(0) <= 127));
|
|
922
|
+
if (s === "$gt" && l && d !== void 0 && d <= 0 || s === "$gte" && l && d !== void 0 && d < 0 || s === "$lt" && l && c !== void 0 && c >= 0 || s === "$lte" && l && c !== void 0 && c > 0) return !0;
|
|
923
|
+
if (s === "$eq") {
|
|
924
|
+
const u = Q(o, t, n, r), g = Q(o, i, n, r);
|
|
925
|
+
if (u !== void 0 && u < 0 || g !== void 0 && g > 0) return !0;
|
|
926
|
+
}
|
|
927
|
+
if (s === "$ne" && !f && P(t, i, n) && P(t, o, n) || s === "$in" && Array.isArray(o) && o.every((u) => {
|
|
928
|
+
const g = Q(u, t, n, r), h = Q(u, i, n, r);
|
|
929
|
+
return g !== void 0 && g < 0 || h !== void 0 && h > 0;
|
|
930
|
+
}) || s === "$nin" && !f && Array.isArray(o) && P(t, i, n) && o.includes(t)) return !0;
|
|
931
|
+
}
|
|
932
|
+
return !1;
|
|
933
|
+
}
|
|
934
|
+
function Q(e, t, i, n) {
|
|
935
|
+
if (n?.type === "BYTE_ARRAY")
|
|
936
|
+
return typeof e != "string" || typeof t != "string" ? void 0 : Ze(Qe.encode(e), Qe.encode(t));
|
|
937
|
+
if (e instanceof Uint8Array || t instanceof Uint8Array)
|
|
938
|
+
return !(e instanceof Uint8Array) || !(t instanceof Uint8Array) ? void 0 : Ze(e, t);
|
|
939
|
+
if (e < t) return -1;
|
|
940
|
+
if (e > t) return 1;
|
|
941
|
+
if (P(e, t, i)) return 0;
|
|
942
|
+
}
|
|
943
|
+
function Ze(e, t) {
|
|
944
|
+
const i = Math.min(e.length, t.length);
|
|
945
|
+
for (let n = 0; n < i; n++) {
|
|
946
|
+
if (e[n] < t[n]) return -1;
|
|
947
|
+
if (e[n] > t[n]) return 1;
|
|
948
|
+
}
|
|
949
|
+
return e.length < t.length ? -1 : e.length > t.length ? 1 : 0;
|
|
950
|
+
}
|
|
951
|
+
function _n(e, t) {
|
|
952
|
+
return z({ value: null }, { value: e }, t);
|
|
953
|
+
}
|
|
954
|
+
function Le(e, t, i, n = !0) {
|
|
955
|
+
if (!e) return;
|
|
956
|
+
if ("$and" in e && Array.isArray(e.$and)) {
|
|
957
|
+
let f;
|
|
958
|
+
for (const s of e.$and)
|
|
959
|
+
f = Je(f, Le(s, t, i, n));
|
|
960
|
+
return f;
|
|
961
|
+
}
|
|
962
|
+
if ("$or" in e && Array.isArray(e.$or)) {
|
|
963
|
+
let f = [];
|
|
964
|
+
for (const s of e.$or) {
|
|
965
|
+
const o = Le(s, t, i, n);
|
|
966
|
+
if (!o) return;
|
|
967
|
+
f = gn(f, o);
|
|
968
|
+
}
|
|
969
|
+
return f;
|
|
970
|
+
}
|
|
971
|
+
if ("$nor" in e && Array.isArray(e.$nor))
|
|
972
|
+
return;
|
|
973
|
+
let r;
|
|
974
|
+
for (const [f, s] of Object.entries(e)) {
|
|
975
|
+
const o = t[f];
|
|
976
|
+
if (!o) continue;
|
|
977
|
+
const c = _n(s, n), d = [];
|
|
978
|
+
for (let a = 0; a < o.pageStarts.length; a++) {
|
|
979
|
+
const l = o.pageStarts[a], u = a + 1 < o.pageStarts.length ? o.pageStarts[a + 1] : i, g = o.nullCounts?.[a], h = c && (g === void 0 || g > 0);
|
|
980
|
+
if (!(!o.nullPages[a] && !h && At(s, o.minValues[a], o.maxValues[a], n, o.element))) {
|
|
981
|
+
const y = d[d.length - 1];
|
|
982
|
+
y && y[1] === l ? y[1] = u : d.push([l, u]);
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
r = Je(r, d);
|
|
986
|
+
}
|
|
987
|
+
return r;
|
|
988
|
+
}
|
|
989
|
+
function Je(e, t) {
|
|
990
|
+
if (!e) return t;
|
|
991
|
+
if (!t) return e;
|
|
992
|
+
const i = [];
|
|
993
|
+
let n = 0, r = 0;
|
|
994
|
+
for (; n < e.length && r < t.length; ) {
|
|
995
|
+
const f = Math.max(e[n][0], t[r][0]), s = Math.min(e[n][1], t[r][1]);
|
|
996
|
+
f < s && i.push([f, s]), e[n][1] < t[r][1] ? n++ : r++;
|
|
997
|
+
}
|
|
998
|
+
return i;
|
|
999
|
+
}
|
|
1000
|
+
function gn(e, t) {
|
|
1001
|
+
const i = [];
|
|
1002
|
+
let n = 0, r = 0;
|
|
1003
|
+
for (; n < e.length || r < t.length; ) {
|
|
1004
|
+
const f = r >= t.length || n < e.length && e[n][0] <= t[r][0] ? e[n++] : t[r++], s = i[i.length - 1];
|
|
1005
|
+
s && f[0] <= s[1] ? s[1] = Math.max(s[1], f[1]) : i.push([f[0], f[1]]);
|
|
1006
|
+
}
|
|
1007
|
+
return i;
|
|
1008
|
+
}
|
|
1009
|
+
function hn(e, t) {
|
|
1010
|
+
let i = e;
|
|
1011
|
+
for (const n of t.split("."))
|
|
1012
|
+
i = i?.[n];
|
|
1013
|
+
return i;
|
|
1014
|
+
}
|
|
1015
|
+
const yn = 1 << 21;
|
|
1016
|
+
function mn({ metadata: e, rowStart: t = 0, rowEnd: i = 1 / 0, columns: n, filter: r, filterStrict: f = !0, useOffsetIndex: s = !1, bloomFiltersByGroup: o, schemaElements: c, pageRangesByGroup: d, pageLocationsByGroup: a }) {
|
|
1017
|
+
if (!e) throw new Error("parquetPlan requires metadata");
|
|
1018
|
+
const l = [], u = [], g = [], h = F(e), m = Re(h), y = r ? {
|
|
1019
|
+
...Et(h),
|
|
1020
|
+
...c
|
|
1021
|
+
} : c;
|
|
1022
|
+
let _ = 0, w = 0;
|
|
1023
|
+
for (const p of e.row_groups) {
|
|
1024
|
+
const I = Number(p.num_rows), A = _ + I, R = o?.[w];
|
|
1025
|
+
if (I > 0 && A > t && _ < i && !re({ rowGroup: p, physicalColumns: m, filter: r, strict: f, bloomFilters: R, schemaElements: y })) {
|
|
1026
|
+
const B = Math.max(t - _, 0), K = Math.min(i - _, I), M = d?.[w], Y = a?.[w];
|
|
1027
|
+
let N = M ? M.map(([x, b]) => [Math.max(x, B), Math.min(b, K)]).filter(([x, b]) => x < b) : [[B, K]];
|
|
1028
|
+
if (N.length > 1 && (p.columns.every((b) => {
|
|
1029
|
+
const T = b.meta_data?.path_in_schema[0], E = b.meta_data?.path_in_schema.join(".");
|
|
1030
|
+
return n && T && !n.includes(T) ? !0 : !!(b.offset_index_offset && b.offset_index_length) || !!(E && Y?.[E]);
|
|
1031
|
+
}) ? N = wn(N, p, n, Y) : N = [[N[0][0], N[N.length - 1][1]]]), N.length) {
|
|
1032
|
+
const x = [], b = N.length > 1 || N[0][0] > 0 || N[0][1] < I;
|
|
1033
|
+
for (const E of p.columns) {
|
|
1034
|
+
const v = E.meta_data;
|
|
1035
|
+
if (E.file_path) throw new Error("parquet file_path not supported");
|
|
1036
|
+
if (!v) throw new Error("parquet column metadata is undefined");
|
|
1037
|
+
if (!n || n.includes(v.path_in_schema[0])) {
|
|
1038
|
+
const S = v.dictionary_page_offset || v.data_page_offset, O = Number(S), U = Number(S + v.total_compressed_size), q = Y?.[v.path_in_schema.join(".")];
|
|
1039
|
+
if (q && b)
|
|
1040
|
+
x.push({
|
|
1041
|
+
columnMetadata: v,
|
|
1042
|
+
pageLocations: q,
|
|
1043
|
+
range: { startByte: O, endByte: U }
|
|
1044
|
+
});
|
|
1045
|
+
else if ((s || M) && E.offset_index_offset && E.offset_index_length && b) {
|
|
1046
|
+
const j = Number(E.offset_index_offset);
|
|
1047
|
+
x.push({
|
|
1048
|
+
columnMetadata: v,
|
|
1049
|
+
offsetIndex: {
|
|
1050
|
+
startByte: j,
|
|
1051
|
+
endByte: j + E.offset_index_length
|
|
1052
|
+
},
|
|
1053
|
+
range: { startByte: O, endByte: U }
|
|
1054
|
+
});
|
|
1055
|
+
} else
|
|
1056
|
+
x.push({
|
|
1057
|
+
columnMetadata: v,
|
|
1058
|
+
range: { startByte: O, endByte: U }
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
for (const [E, v] of N)
|
|
1063
|
+
l.push({ chunks: x, rowGroup: p, groupStart: _, groupRows: I, selectStart: E, selectEnd: v });
|
|
1064
|
+
let T;
|
|
1065
|
+
for (const E of x)
|
|
1066
|
+
if (!("pageLocations" in E)) if ("offsetIndex" in E)
|
|
1067
|
+
g.push(E.offsetIndex);
|
|
1068
|
+
else {
|
|
1069
|
+
const { range: v } = E;
|
|
1070
|
+
n ? u.push(v) : T && v.endByte - T.startByte <= yn ? T.endByte = v.endByte : (T && u.push(T), T = { ...v });
|
|
1071
|
+
}
|
|
1072
|
+
T && u.push(T);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
_ = A, w++;
|
|
1076
|
+
}
|
|
1077
|
+
return isFinite(i) || (i = _), u.push(...g), { metadata: e, rowStart: t, rowEnd: i, columns: n, fetches: u, groups: l };
|
|
1078
|
+
}
|
|
1079
|
+
function wn(e, t, i, n) {
|
|
1080
|
+
const r = t.columns.filter((s) => !i || i.includes(s.meta_data?.path_in_schema[0] || "")).map((s) => n?.[s.meta_data?.path_in_schema.join(".") || ""]), f = [];
|
|
1081
|
+
for (const s of e) {
|
|
1082
|
+
const o = f[f.length - 1], c = o && r.some((d) => {
|
|
1083
|
+
if (!d) return !0;
|
|
1084
|
+
const a = Xe(o, d, Number(t.num_rows)), l = Xe(s, d, Number(t.num_rows));
|
|
1085
|
+
return a[0] <= l[1] && l[0] <= a[1];
|
|
1086
|
+
});
|
|
1087
|
+
o && c ? o[1] = s[1] : f.push([...s]);
|
|
1088
|
+
}
|
|
1089
|
+
return f;
|
|
1090
|
+
}
|
|
1091
|
+
function Xe([e, t], i, n) {
|
|
1092
|
+
let r = 1 / 0, f = -1 / 0;
|
|
1093
|
+
for (let s = 0; s < i.length; s++) {
|
|
1094
|
+
const o = Number(i[s].first_row_index);
|
|
1095
|
+
(s + 1 < i.length ? Number(i[s + 1].first_row_index) : n) > e && o < t && (r = Math.min(r, s), f = s);
|
|
1096
|
+
}
|
|
1097
|
+
return [r, f];
|
|
1098
|
+
}
|
|
1099
|
+
async function pn({ file: e, metadata: t, filter: i, filterStrict: n = !0 }) {
|
|
1100
|
+
const r = t.row_groups.map(() => (
|
|
1101
|
+
/** @type {Record<string, BloomFilter>} */
|
|
1102
|
+
{}
|
|
1103
|
+
)), f = cn(i);
|
|
1104
|
+
if (f.size === 0) return r;
|
|
1105
|
+
const s = Re(F(t)), o = [];
|
|
1106
|
+
return t.row_groups.forEach((c, d) => {
|
|
1107
|
+
if (!re({ rowGroup: c, physicalColumns: s, filter: i, strict: n }))
|
|
1108
|
+
for (const a of f) {
|
|
1109
|
+
const l = s.indexOf(a);
|
|
1110
|
+
if (l === -1) continue;
|
|
1111
|
+
const u = c.columns[l]?.meta_data;
|
|
1112
|
+
if (!u?.bloom_filter_offset || !u.bloom_filter_length) continue;
|
|
1113
|
+
const g = Number(u.bloom_filter_offset), h = g + u.bloom_filter_length;
|
|
1114
|
+
o.push((async () => {
|
|
1115
|
+
const m = await e.slice(g, h), y = ln({ view: new DataView(m), offset: 0 });
|
|
1116
|
+
y && (r[d][a] = y);
|
|
1117
|
+
})());
|
|
1118
|
+
}
|
|
1119
|
+
}), o.length && await Promise.all(o), r;
|
|
1120
|
+
}
|
|
1121
|
+
async function An({ file: e, metadata: t, filter: i, filterStrict: n = !0, rowStart: r = 0, rowEnd: f = 1 / 0, columns: s, bloomFiltersByGroup: o, schemaElements: c, parsers: d }) {
|
|
1122
|
+
const a = t.row_groups.map(() => {
|
|
1123
|
+
}), l = t.row_groups.map(() => (
|
|
1124
|
+
/** @type {Record<string, PageLocation[]>} */
|
|
1125
|
+
{}
|
|
1126
|
+
));
|
|
1127
|
+
if (i && "$nor" in i && Array.isArray(i.$nor))
|
|
1128
|
+
return { pageRangesByGroup: a, pageLocationsByGroup: l };
|
|
1129
|
+
const u = ie(i);
|
|
1130
|
+
if (!u.length) return { pageRangesByGroup: a, pageLocationsByGroup: l };
|
|
1131
|
+
const g = F(t), h = Re(g), m = {
|
|
1132
|
+
...Et(g),
|
|
1133
|
+
...c
|
|
1134
|
+
}, y = [], _ = [], w = [];
|
|
1135
|
+
let p = 0;
|
|
1136
|
+
if (t.row_groups.forEach((I, A) => {
|
|
1137
|
+
const R = Number(I.num_rows), B = p + R, K = R > 0 && B > r && p < f;
|
|
1138
|
+
if (p = B, !K || re({ rowGroup: I, physicalColumns: h, filter: i, strict: n, bloomFilters: o?.[A], schemaElements: m })) return;
|
|
1139
|
+
const M = {};
|
|
1140
|
+
let Y = 0;
|
|
1141
|
+
const N = /* @__PURE__ */ new Set();
|
|
1142
|
+
let x = !1;
|
|
1143
|
+
for (const b of u) {
|
|
1144
|
+
const T = h.indexOf(b);
|
|
1145
|
+
if (T === -1) continue;
|
|
1146
|
+
const E = I.columns[T];
|
|
1147
|
+
if (!E?.meta_data || !E.column_index_offset || !E.column_index_length || !E.offset_index_offset || !E.offset_index_length) continue;
|
|
1148
|
+
const S = m[b];
|
|
1149
|
+
if (!S) continue;
|
|
1150
|
+
x = !0, N.add(b);
|
|
1151
|
+
const O = Number(E.column_index_offset), U = Number(E.offset_index_offset), q = O + E.column_index_length, j = U + E.offset_index_length;
|
|
1152
|
+
y.push(
|
|
1153
|
+
{ startByte: O, endByte: q },
|
|
1154
|
+
{ startByte: U, endByte: j }
|
|
1155
|
+
), Y++, _.push(async (De) => {
|
|
1156
|
+
const [Ot, Dt] = await Promise.all([
|
|
1157
|
+
De.slice(O, q),
|
|
1158
|
+
De.slice(U, j)
|
|
1159
|
+
]), le = nn({ view: new DataView(Ot), offset: 0 }, S, d), Me = ve({ view: new DataView(Dt), offset: 0 });
|
|
1160
|
+
l[A][b] = Me.page_locations, M[b] = {
|
|
1161
|
+
minValues: le.min_values,
|
|
1162
|
+
maxValues: le.max_values,
|
|
1163
|
+
nullPages: le.null_pages,
|
|
1164
|
+
nullCounts: le.null_counts,
|
|
1165
|
+
pageStarts: Me.page_locations.map((Mt) => Number(Mt.first_row_index)),
|
|
1166
|
+
element: S
|
|
1167
|
+
};
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
if (x)
|
|
1171
|
+
for (const b of I.columns) {
|
|
1172
|
+
const T = b.meta_data;
|
|
1173
|
+
if (!T) continue;
|
|
1174
|
+
const E = T.path_in_schema[0], v = T.path_in_schema.join(".");
|
|
1175
|
+
if (s && !s.includes(E) || N.has(v) || !b.offset_index_offset || !b.offset_index_length) continue;
|
|
1176
|
+
N.add(v);
|
|
1177
|
+
const S = Number(b.offset_index_offset), O = S + b.offset_index_length;
|
|
1178
|
+
y.push({ startByte: S, endByte: O }), Y++, _.push(async (U) => {
|
|
1179
|
+
const q = await U.slice(S, O), j = ve({ view: new DataView(q), offset: 0 });
|
|
1180
|
+
l[A][v] = j.page_locations;
|
|
1181
|
+
});
|
|
1182
|
+
}
|
|
1183
|
+
Y && w.push({ rgIdx: A, groupRows: R, columnPages: M });
|
|
1184
|
+
}), _.length) {
|
|
1185
|
+
const I = It(e, { fetches: En(y) });
|
|
1186
|
+
await Promise.all(_.map((A) => A(I)));
|
|
1187
|
+
for (const { rgIdx: A, groupRows: R, columnPages: B } of w)
|
|
1188
|
+
a[A] = Le(i, B, R, n);
|
|
1189
|
+
}
|
|
1190
|
+
return { pageRangesByGroup: a, pageLocationsByGroup: l };
|
|
1191
|
+
}
|
|
1192
|
+
function En(e) {
|
|
1193
|
+
const t = e.map((n) => ({ ...n })).sort((n, r) => n.startByte - r.startByte || n.endByte - r.endByte), i = [];
|
|
1194
|
+
for (const n of t) {
|
|
1195
|
+
const r = i[i.length - 1];
|
|
1196
|
+
r && n.startByte <= r.endByte ? r.endByte = Math.max(r.endByte, n.endByte) : i.push(n);
|
|
1197
|
+
}
|
|
1198
|
+
return i;
|
|
1199
|
+
}
|
|
1200
|
+
function Et(e) {
|
|
1201
|
+
const t = {};
|
|
1202
|
+
function i(n) {
|
|
1203
|
+
if (n.children.length)
|
|
1204
|
+
for (const r of n.children) i(r);
|
|
1205
|
+
else
|
|
1206
|
+
t[n.path.join(".")] = n.element;
|
|
1207
|
+
}
|
|
1208
|
+
return i(e), t;
|
|
1209
|
+
}
|
|
1210
|
+
function It(e, { fetches: t }) {
|
|
1211
|
+
const i = t.map(({ startByte: n, endByte: r }) => e.slice(n, r));
|
|
1212
|
+
return {
|
|
1213
|
+
byteLength: e.byteLength,
|
|
1214
|
+
slice(n, r = e.byteLength) {
|
|
1215
|
+
const f = t.findIndex(({ startByte: s, endByte: o }) => s <= n && r <= o);
|
|
1216
|
+
if (f < 0)
|
|
1217
|
+
return e.slice(n, r);
|
|
1218
|
+
if (t[f].startByte !== n || t[f].endByte !== r) {
|
|
1219
|
+
const s = n - t[f].startByte, o = r - t[f].startByte;
|
|
1220
|
+
return i[f] instanceof Promise ? i[f].then((c) => c.slice(s, o)) : i[f].slice(s, o);
|
|
1221
|
+
} else
|
|
1222
|
+
return i[f];
|
|
1223
|
+
}
|
|
1224
|
+
};
|
|
1225
|
+
}
|
|
1226
|
+
const Se = new TextDecoder(), et = /* @__PURE__ */ new WeakMap();
|
|
1227
|
+
function bt(e, t = oe) {
|
|
1228
|
+
if (Array.isArray(e))
|
|
1229
|
+
return e.map((i) => bt(i, t));
|
|
1230
|
+
if (typeof e != "object") return e;
|
|
1231
|
+
if ("metadata" in e) {
|
|
1232
|
+
const i = In(e.metadata), n = e.typed_value && ue(e.typed_value, i, t), r = e.value && fe(ge(e.value), i, t);
|
|
1233
|
+
return n && r ? { ...r, ...n } : n ?? r;
|
|
1234
|
+
}
|
|
1235
|
+
return e;
|
|
1236
|
+
}
|
|
1237
|
+
function ue(e, t, i) {
|
|
1238
|
+
if (e instanceof Date) return e;
|
|
1239
|
+
if (e && typeof e == "object" && !Array.isArray(e) && !(e instanceof Uint8Array)) {
|
|
1240
|
+
if ("typed_value" in e && e.typed_value !== null && e.typed_value !== void 0)
|
|
1241
|
+
return ue(e.typed_value, t, i);
|
|
1242
|
+
if ("value" in e && e.value instanceof Uint8Array)
|
|
1243
|
+
return fe(ge(e.value), t, i);
|
|
1244
|
+
if ("typed_value" in e || "value" in e)
|
|
1245
|
+
return null;
|
|
1246
|
+
const n = {};
|
|
1247
|
+
for (const [r, f] of Object.entries(e))
|
|
1248
|
+
t.dictionary.includes(r) && (n[r] = ue(f, t, i));
|
|
1249
|
+
return n;
|
|
1250
|
+
}
|
|
1251
|
+
return e instanceof Uint8Array ? fe(ge(e), t, i) : Array.isArray(e) ? e.map((n) => ue(n, t, i)) : e;
|
|
1252
|
+
}
|
|
1253
|
+
function ge(e) {
|
|
1254
|
+
return { view: new DataView(e.buffer, e.byteOffset, e.byteLength), offset: 0 };
|
|
1255
|
+
}
|
|
1256
|
+
function In(e) {
|
|
1257
|
+
let t = et.get(e.buffer);
|
|
1258
|
+
t || (t = /* @__PURE__ */ new Map(), et.set(e.buffer, t));
|
|
1259
|
+
const i = `${e.byteOffset}:${e.byteLength}`, n = t.get(i);
|
|
1260
|
+
if (n) return n;
|
|
1261
|
+
const r = ge(e), f = r.view.getUint8(r.offset++), s = f & 15;
|
|
1262
|
+
if (s !== 1) throw new Error(`parquet unsupported variant metadata version: ${s}`);
|
|
1263
|
+
const o = (f >> 4 & 1) === 1, c = (f >> 6 & 3) + 1, d = H(r, c), a = new Array(d + 1);
|
|
1264
|
+
for (let h = 0; h < a.length; h++)
|
|
1265
|
+
a[h] = H(r, c);
|
|
1266
|
+
const l = r.offset, u = new Array(d);
|
|
1267
|
+
for (let h = 0; h < d; h++) {
|
|
1268
|
+
const m = a[h], y = a[h + 1], _ = new Uint8Array(e.buffer, e.byteOffset + l + m, y - m);
|
|
1269
|
+
u[h] = Se.decode(_);
|
|
1270
|
+
}
|
|
1271
|
+
const g = { dictionary: u, sorted: o };
|
|
1272
|
+
return t.set(i, g), g;
|
|
1273
|
+
}
|
|
1274
|
+
function H(e, t) {
|
|
1275
|
+
let i = 0;
|
|
1276
|
+
for (let n = 0; n < t; n++)
|
|
1277
|
+
i |= e.view.getUint8(e.offset + n) << n * 8;
|
|
1278
|
+
return e.offset += t, i;
|
|
1279
|
+
}
|
|
1280
|
+
function fe(e, t, i) {
|
|
1281
|
+
const n = e.view.getUint8(e.offset++), r = n & 3, f = n >> 2;
|
|
1282
|
+
if (r === 0) return bn(e, f, i);
|
|
1283
|
+
if (r === 2) return vn(e, f, t, i);
|
|
1284
|
+
if (r === 3) return Tn(e, f, t, i);
|
|
1285
|
+
const s = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, f);
|
|
1286
|
+
return e.offset += f, Se.decode(s);
|
|
1287
|
+
}
|
|
1288
|
+
function bn(e, t, i) {
|
|
1289
|
+
switch (t) {
|
|
1290
|
+
case 0:
|
|
1291
|
+
return null;
|
|
1292
|
+
case 1:
|
|
1293
|
+
return !0;
|
|
1294
|
+
case 2:
|
|
1295
|
+
return !1;
|
|
1296
|
+
case 3: {
|
|
1297
|
+
const n = e.view.getInt8(e.offset);
|
|
1298
|
+
return e.offset += 1, n;
|
|
1299
|
+
}
|
|
1300
|
+
case 4: {
|
|
1301
|
+
const n = e.view.getInt16(e.offset, !0);
|
|
1302
|
+
return e.offset += 2, n;
|
|
1303
|
+
}
|
|
1304
|
+
case 5: {
|
|
1305
|
+
const n = e.view.getInt32(e.offset, !0);
|
|
1306
|
+
return e.offset += 4, n;
|
|
1307
|
+
}
|
|
1308
|
+
case 6: {
|
|
1309
|
+
const n = e.view.getBigInt64(e.offset, !0);
|
|
1310
|
+
return e.offset += 8, n;
|
|
1311
|
+
}
|
|
1312
|
+
case 7: {
|
|
1313
|
+
const n = e.view.getFloat64(e.offset, !0);
|
|
1314
|
+
return e.offset += 8, n;
|
|
1315
|
+
}
|
|
1316
|
+
case 8:
|
|
1317
|
+
return we(e, 4);
|
|
1318
|
+
case 9:
|
|
1319
|
+
return we(e, 8);
|
|
1320
|
+
case 10:
|
|
1321
|
+
return we(e, 16);
|
|
1322
|
+
case 11: {
|
|
1323
|
+
const n = e.view.getInt32(e.offset, !0);
|
|
1324
|
+
return e.offset += 4, i.dateFromDays(n);
|
|
1325
|
+
}
|
|
1326
|
+
case 12:
|
|
1327
|
+
// timestamp_micros (utc)
|
|
1328
|
+
case 13: {
|
|
1329
|
+
const n = e.view.getBigInt64(e.offset, !0);
|
|
1330
|
+
return e.offset += 8, i.timestampFromMicroseconds(n);
|
|
1331
|
+
}
|
|
1332
|
+
case 14: {
|
|
1333
|
+
const n = e.view.getFloat32(e.offset, !0);
|
|
1334
|
+
return e.offset += 4, n;
|
|
1335
|
+
}
|
|
1336
|
+
case 15:
|
|
1337
|
+
return tt(e);
|
|
1338
|
+
case 16: {
|
|
1339
|
+
const n = tt(e);
|
|
1340
|
+
return Se.decode(n);
|
|
1341
|
+
}
|
|
1342
|
+
case 17: {
|
|
1343
|
+
const n = e.view.getBigInt64(e.offset, !0);
|
|
1344
|
+
return e.offset += 8, n;
|
|
1345
|
+
}
|
|
1346
|
+
case 18:
|
|
1347
|
+
// timestamp_nanos (utc)
|
|
1348
|
+
case 19: {
|
|
1349
|
+
const n = e.view.getBigInt64(e.offset, !0);
|
|
1350
|
+
return e.offset += 8, i.timestampFromNanoseconds(n);
|
|
1351
|
+
}
|
|
1352
|
+
case 20: {
|
|
1353
|
+
const n = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, 16);
|
|
1354
|
+
e.offset += 16;
|
|
1355
|
+
const r = Array.from(n, (f) => f.toString(16).padStart(2, "0")).join("");
|
|
1356
|
+
return `${r.slice(0, 8)}-${r.slice(8, 12)}-${r.slice(12, 16)}-${r.slice(16, 20)}-${r.slice(20)}`;
|
|
1357
|
+
}
|
|
1358
|
+
default:
|
|
1359
|
+
throw new Error(`parquet unsupported variant primitive type: ${t}`);
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
function vn(e, t, i, n) {
|
|
1363
|
+
const r = (t & 3) + 1, f = (t >> 2 & 3) + 1, o = t >> 4 & 1 ? H(e, 4) : e.view.getUint8(e.offset++), c = new Array(o);
|
|
1364
|
+
for (let l = 0; l < o; l++)
|
|
1365
|
+
c[l] = H(e, f);
|
|
1366
|
+
const d = new Array(o + 1);
|
|
1367
|
+
for (let l = 0; l < d.length; l++)
|
|
1368
|
+
d[l] = H(e, r);
|
|
1369
|
+
const a = {};
|
|
1370
|
+
for (let l = 0; l < o; l++) {
|
|
1371
|
+
const u = i.dictionary[c[l]], g = {
|
|
1372
|
+
view: e.view,
|
|
1373
|
+
offset: e.offset + d[l]
|
|
1374
|
+
};
|
|
1375
|
+
a[u] = fe(g, i, n);
|
|
1376
|
+
}
|
|
1377
|
+
return e.offset += d[d.length - 1], a;
|
|
1378
|
+
}
|
|
1379
|
+
function Tn(e, t, i, n) {
|
|
1380
|
+
const r = t & 3, f = t >> 2 & 1, s = r + 1, o = H(e, f ? 4 : 1), c = new Array(o + 1);
|
|
1381
|
+
for (let l = 0; l < c.length; l++)
|
|
1382
|
+
c[l] = H(e, s);
|
|
1383
|
+
const d = e.offset, a = new Array(o);
|
|
1384
|
+
for (let l = 0; l < o; l++) {
|
|
1385
|
+
const u = {
|
|
1386
|
+
view: e.view,
|
|
1387
|
+
offset: d + c[l]
|
|
1388
|
+
};
|
|
1389
|
+
a[l] = fe(u, i, n);
|
|
1390
|
+
}
|
|
1391
|
+
return e.offset = d + c[c.length - 1], a;
|
|
1392
|
+
}
|
|
1393
|
+
function we(e, t) {
|
|
1394
|
+
const i = e.view.getUint8(e.offset);
|
|
1395
|
+
e.offset += 1;
|
|
1396
|
+
let n;
|
|
1397
|
+
if (t === 4)
|
|
1398
|
+
n = BigInt(e.view.getInt32(e.offset, !0)), e.offset += 4;
|
|
1399
|
+
else if (t === 8)
|
|
1400
|
+
n = e.view.getBigInt64(e.offset, !0), e.offset += 8;
|
|
1401
|
+
else {
|
|
1402
|
+
const r = e.view.getBigUint64(e.offset, !0);
|
|
1403
|
+
n = e.view.getBigInt64(e.offset + 8, !0) << 64n | r, e.offset += 16;
|
|
1404
|
+
}
|
|
1405
|
+
return Number(n) * 10 ** -i;
|
|
1406
|
+
}
|
|
1407
|
+
function tt(e) {
|
|
1408
|
+
const t = e.view.getUint32(e.offset, !0);
|
|
1409
|
+
e.offset += 4;
|
|
1410
|
+
const i = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1411
|
+
return e.offset += t, i;
|
|
1412
|
+
}
|
|
1413
|
+
function nt(e, t, i, n, r) {
|
|
1414
|
+
const f = Be(r);
|
|
1415
|
+
if (!t?.length && !i.length) {
|
|
1416
|
+
if (!f || !n.length) return n;
|
|
1417
|
+
t = new Array(n.length).fill(f);
|
|
1418
|
+
}
|
|
1419
|
+
const s = t?.length || i.length, o = r.map(({ element: h }) => h.repetition_type);
|
|
1420
|
+
let c = 0;
|
|
1421
|
+
const d = [e];
|
|
1422
|
+
let a = e, l = 0, u = 0, g = 0;
|
|
1423
|
+
if (i[0])
|
|
1424
|
+
for (; l < o.length - 2 && g < i[0]; )
|
|
1425
|
+
l++, o[l] !== "REQUIRED" && (a = a.at(-1), d.push(a), u++), o[l] === "REPEATED" && g++;
|
|
1426
|
+
for (let h = 0; h < s; h++) {
|
|
1427
|
+
const m = t?.length ? t[h] : f, y = i[h];
|
|
1428
|
+
for (; l && (y < g || o[l] !== "REPEATED"); )
|
|
1429
|
+
o[l] !== "REQUIRED" && (d.pop(), u--), o[l] === "REPEATED" && g--, l--;
|
|
1430
|
+
for (a = d.at(-1); (l < o.length - 2 || o[l + 1] === "REPEATED") && (u < m || o[l + 1] === "REQUIRED"); ) {
|
|
1431
|
+
if (l++, o[l] !== "REQUIRED") {
|
|
1432
|
+
const _ = [];
|
|
1433
|
+
a.push(_), a = _, d.push(_), u++;
|
|
1434
|
+
}
|
|
1435
|
+
o[l] === "REPEATED" && g++;
|
|
1436
|
+
}
|
|
1437
|
+
m === f ? a.push(n[c++]) : l === o.length - 2 ? a.push(null) : a.push([]);
|
|
1438
|
+
}
|
|
1439
|
+
if (!e.length)
|
|
1440
|
+
for (let h = 0; h < f; h++) {
|
|
1441
|
+
const m = [];
|
|
1442
|
+
a.push(m), a = m;
|
|
1443
|
+
}
|
|
1444
|
+
return e;
|
|
1445
|
+
}
|
|
1446
|
+
function te(e, t, i, n = 0) {
|
|
1447
|
+
const r = t.path.join("."), f = t.element.repetition_type === "OPTIONAL", s = f ? n + 1 : n;
|
|
1448
|
+
if (Yt(t)) {
|
|
1449
|
+
let o = t.children[0], c = s;
|
|
1450
|
+
o.children.length === 1 && (o = o.children[0], c++), te(e, o, i, c);
|
|
1451
|
+
const d = o.path.join("."), a = e.get(d);
|
|
1452
|
+
if (!a) throw new Error("parquet list column missing values");
|
|
1453
|
+
f && de(a, n), e.set(r, a), e.delete(d);
|
|
1454
|
+
return;
|
|
1455
|
+
}
|
|
1456
|
+
if (qt(t)) {
|
|
1457
|
+
const o = t.children[0].element.name;
|
|
1458
|
+
te(e, t.children[0].children[0], i, s + 1), te(e, t.children[0].children[1], i, s + 1);
|
|
1459
|
+
const c = e.get(`${r}.${o}.key`), d = e.get(`${r}.${o}.value`);
|
|
1460
|
+
if (!c) throw new Error("parquet map column missing keys");
|
|
1461
|
+
if (!d) throw new Error("parquet map column missing values");
|
|
1462
|
+
if (c.length !== d.length)
|
|
1463
|
+
throw new Error("parquet map column key/value length mismatch");
|
|
1464
|
+
const a = vt(c, d, s);
|
|
1465
|
+
f && de(a, n), e.delete(`${r}.${o}.key`), e.delete(`${r}.${o}.value`), e.set(r, a);
|
|
1466
|
+
return;
|
|
1467
|
+
}
|
|
1468
|
+
if (t.children.length) {
|
|
1469
|
+
const o = t.element.repetition_type === "REQUIRED" ? n : n + 1, c = {};
|
|
1470
|
+
for (const a of t.children) {
|
|
1471
|
+
te(e, a, i, o);
|
|
1472
|
+
const l = e.get(a.path.join("."));
|
|
1473
|
+
if (!l) throw new Error("parquet struct missing child data");
|
|
1474
|
+
c[a.element.name] = l;
|
|
1475
|
+
}
|
|
1476
|
+
for (const a of t.children)
|
|
1477
|
+
e.delete(a.path.join("."));
|
|
1478
|
+
let d = Tt(c, o);
|
|
1479
|
+
t.element.logical_type?.type === "VARIANT" && (d = bt(d, i)), f && de(d, n), e.set(r, d);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
function de(e, t) {
|
|
1483
|
+
for (let i = 0; i < e.length; i++)
|
|
1484
|
+
t ? de(e[i], t - 1) : e[i] = e[i][0];
|
|
1485
|
+
}
|
|
1486
|
+
function vt(e, t, i) {
|
|
1487
|
+
const n = [];
|
|
1488
|
+
for (let r = 0; r < e.length; r++)
|
|
1489
|
+
if (i)
|
|
1490
|
+
n.push(vt(e[r], t[r], i - 1));
|
|
1491
|
+
else if (e[r]) {
|
|
1492
|
+
const f = {};
|
|
1493
|
+
for (let s = 0; s < e[r].length; s++) {
|
|
1494
|
+
const o = t[r][s];
|
|
1495
|
+
f[e[r][s]] = o === void 0 ? null : o;
|
|
1496
|
+
}
|
|
1497
|
+
n.push(f);
|
|
1498
|
+
} else
|
|
1499
|
+
n.push(void 0);
|
|
1500
|
+
return n;
|
|
1501
|
+
}
|
|
1502
|
+
function Tt(e, t) {
|
|
1503
|
+
const i = Object.keys(e), n = e[i[0]]?.length, r = [];
|
|
1504
|
+
for (let f = 0; f < n; f++) {
|
|
1505
|
+
const s = {};
|
|
1506
|
+
for (const o of i) {
|
|
1507
|
+
if (e[o].length !== n) throw new Error("parquet struct parsing error");
|
|
1508
|
+
s[o] = e[o][f];
|
|
1509
|
+
}
|
|
1510
|
+
t ? r.push(Tt(s, t - 1)) : r.push(s);
|
|
1511
|
+
}
|
|
1512
|
+
return r;
|
|
1513
|
+
}
|
|
1514
|
+
function se(e, t, i) {
|
|
1515
|
+
const n = i instanceof Int32Array, r = W(e), f = W(e);
|
|
1516
|
+
W(e);
|
|
1517
|
+
let s = be(e), o = 0;
|
|
1518
|
+
i[o++] = n ? Number(s) : s;
|
|
1519
|
+
const c = r / f;
|
|
1520
|
+
for (; o < t; ) {
|
|
1521
|
+
const d = be(e), a = new Uint8Array(f);
|
|
1522
|
+
for (let l = 0; l < f; l++)
|
|
1523
|
+
a[l] = e.view.getUint8(e.offset++);
|
|
1524
|
+
for (let l = 0; l < f && o < t; l++) {
|
|
1525
|
+
const u = BigInt(a[l]);
|
|
1526
|
+
if (u) {
|
|
1527
|
+
let g = 0n, h = c;
|
|
1528
|
+
const m = (1n << u) - 1n;
|
|
1529
|
+
for (; h && o < t; ) {
|
|
1530
|
+
let y = BigInt(e.view.getUint8(e.offset)) >> g & m;
|
|
1531
|
+
for (g += u; g >= 8; )
|
|
1532
|
+
g -= 8n, e.offset++, g && (y |= BigInt(e.view.getUint8(e.offset)) << u - g & m);
|
|
1533
|
+
const _ = d + y;
|
|
1534
|
+
s += _, i[o++] = n ? Number(s) : s, h--;
|
|
1535
|
+
}
|
|
1536
|
+
h && (e.offset += Math.ceil((h * Number(u) + Number(g)) / 8));
|
|
1537
|
+
} else
|
|
1538
|
+
for (let g = 0; g < c && o < t; g++)
|
|
1539
|
+
s += d, i[o++] = n ? Number(s) : s;
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
function Lt(e, t, i) {
|
|
1544
|
+
const n = new Int32Array(t);
|
|
1545
|
+
se(e, t, n);
|
|
1546
|
+
for (let r = 0; r < t; r++)
|
|
1547
|
+
i[r] = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, n[r]), e.offset += n[r];
|
|
1548
|
+
}
|
|
1549
|
+
function Ln(e, t, i) {
|
|
1550
|
+
const n = new Int32Array(t);
|
|
1551
|
+
se(e, t, n);
|
|
1552
|
+
const r = new Int32Array(t);
|
|
1553
|
+
se(e, t, r);
|
|
1554
|
+
for (let f = 0; f < t; f++) {
|
|
1555
|
+
const s = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, r[f]);
|
|
1556
|
+
n[f] ? (i[f] = new Uint8Array(n[f] + r[f]), i[f].set(i[f - 1].subarray(0, n[f])), i[f].set(s, n[f])) : i[f] = s, e.offset += r[f];
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
function k(e, t, i, n) {
|
|
1560
|
+
n === void 0 && (n = e.view.getUint32(e.offset, !0), e.offset += 4);
|
|
1561
|
+
const r = e.offset;
|
|
1562
|
+
let f = 0;
|
|
1563
|
+
for (; f < i.length; ) {
|
|
1564
|
+
const s = W(e);
|
|
1565
|
+
if (s & 1)
|
|
1566
|
+
f = Rn(e, s, t, i, f);
|
|
1567
|
+
else {
|
|
1568
|
+
const o = s >>> 1;
|
|
1569
|
+
Nn(e, o, t, i, f), f += o;
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
e.offset = r + n;
|
|
1573
|
+
}
|
|
1574
|
+
function Nn(e, t, i, n, r) {
|
|
1575
|
+
const f = i + 7 >> 3;
|
|
1576
|
+
let s = 0;
|
|
1577
|
+
for (let o = 0; o < f; o++)
|
|
1578
|
+
s |= e.view.getUint8(e.offset++) << (o << 3);
|
|
1579
|
+
for (let o = 0; o < t; o++)
|
|
1580
|
+
n[r + o] = s;
|
|
1581
|
+
}
|
|
1582
|
+
function Rn(e, t, i, n, r) {
|
|
1583
|
+
let f = t >> 1 << 3;
|
|
1584
|
+
const s = (1 << i) - 1;
|
|
1585
|
+
let o = 0;
|
|
1586
|
+
if (e.offset < e.view.byteLength)
|
|
1587
|
+
o = e.view.getUint8(e.offset++);
|
|
1588
|
+
else if (s)
|
|
1589
|
+
throw new Error(`parquet bitpack offset ${e.offset} out of range`);
|
|
1590
|
+
let c = 8, d = 0;
|
|
1591
|
+
for (; f; )
|
|
1592
|
+
d > 8 ? (d -= 8, c -= 8, o >>>= 8) : c - d < i ? (o |= e.view.getUint8(e.offset) << c, e.offset++, c += 8) : (r < n.length && (n[r++] = o >> d & s), f--, d += i);
|
|
1593
|
+
return r;
|
|
1594
|
+
}
|
|
1595
|
+
function Nt(e, t, i, n) {
|
|
1596
|
+
const r = Bn(i, n), f = new Uint8Array(t * r);
|
|
1597
|
+
for (let s = 0; s < r; s++)
|
|
1598
|
+
for (let o = 0; o < t; o++)
|
|
1599
|
+
f[o * r + s] = e.view.getUint8(e.offset++);
|
|
1600
|
+
if (i === "FLOAT") return new Float32Array(f.buffer);
|
|
1601
|
+
if (i === "DOUBLE") return new Float64Array(f.buffer);
|
|
1602
|
+
if (i === "INT32") return new Int32Array(f.buffer);
|
|
1603
|
+
if (i === "INT64") return new BigInt64Array(f.buffer);
|
|
1604
|
+
if (i === "FIXED_LEN_BYTE_ARRAY") {
|
|
1605
|
+
const s = new Array(t);
|
|
1606
|
+
for (let o = 0; o < t; o++)
|
|
1607
|
+
s[o] = f.subarray(o * r, (o + 1) * r);
|
|
1608
|
+
return s;
|
|
1609
|
+
}
|
|
1610
|
+
throw new Error(`parquet byte_stream_split unsupported type: ${i}`);
|
|
1611
|
+
}
|
|
1612
|
+
function Bn(e, t) {
|
|
1613
|
+
switch (e) {
|
|
1614
|
+
case "INT32":
|
|
1615
|
+
case "FLOAT":
|
|
1616
|
+
return 4;
|
|
1617
|
+
case "INT64":
|
|
1618
|
+
case "DOUBLE":
|
|
1619
|
+
return 8;
|
|
1620
|
+
case "FIXED_LEN_BYTE_ARRAY":
|
|
1621
|
+
if (!t) throw new Error("parquet byteWidth missing type_length");
|
|
1622
|
+
return t;
|
|
1623
|
+
default:
|
|
1624
|
+
throw new Error(`parquet unsupported type: ${e}`);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
function Oe(e, t, i, n) {
|
|
1628
|
+
if (i === 0) return [];
|
|
1629
|
+
if (t === "BOOLEAN")
|
|
1630
|
+
return xn(e, i);
|
|
1631
|
+
if (t === "INT32")
|
|
1632
|
+
return Sn(e, i);
|
|
1633
|
+
if (t === "INT64")
|
|
1634
|
+
return On(e, i);
|
|
1635
|
+
if (t === "INT96")
|
|
1636
|
+
return Dn(e, i);
|
|
1637
|
+
if (t === "FLOAT")
|
|
1638
|
+
return Mn(e, i);
|
|
1639
|
+
if (t === "DOUBLE")
|
|
1640
|
+
return Un(e, i);
|
|
1641
|
+
if (t === "BYTE_ARRAY")
|
|
1642
|
+
return Pn(e, i);
|
|
1643
|
+
if (t === "FIXED_LEN_BYTE_ARRAY") {
|
|
1644
|
+
if (!n) throw new Error("parquet missing fixed length");
|
|
1645
|
+
return $n(e, i, n);
|
|
1646
|
+
} else
|
|
1647
|
+
throw new Error(`parquet unhandled type: ${t}`);
|
|
1648
|
+
}
|
|
1649
|
+
function xn(e, t) {
|
|
1650
|
+
const i = new Array(t);
|
|
1651
|
+
for (let n = 0; n < t; n++) {
|
|
1652
|
+
const r = e.offset + (n / 8 | 0), f = n % 8, s = e.view.getUint8(r);
|
|
1653
|
+
i[n] = (s & 1 << f) !== 0;
|
|
1654
|
+
}
|
|
1655
|
+
return e.offset += Math.ceil(t / 8), i;
|
|
1656
|
+
}
|
|
1657
|
+
function Sn(e, t) {
|
|
1658
|
+
const i = (e.view.byteOffset + e.offset) % 4 ? new Int32Array(ye(e.view.buffer, e.view.byteOffset + e.offset, t * 4)) : new Int32Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1659
|
+
return e.offset += t * 4, i;
|
|
1660
|
+
}
|
|
1661
|
+
function On(e, t) {
|
|
1662
|
+
const i = (e.view.byteOffset + e.offset) % 8 ? new BigInt64Array(ye(e.view.buffer, e.view.byteOffset + e.offset, t * 8)) : new BigInt64Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1663
|
+
return e.offset += t * 8, i;
|
|
1664
|
+
}
|
|
1665
|
+
function Dn(e, t) {
|
|
1666
|
+
const i = new Array(t);
|
|
1667
|
+
for (let n = 0; n < t; n++) {
|
|
1668
|
+
const r = e.view.getBigInt64(e.offset + n * 12, !0), f = e.view.getInt32(e.offset + n * 12 + 8, !0);
|
|
1669
|
+
i[n] = BigInt(f) << 64n | r;
|
|
1670
|
+
}
|
|
1671
|
+
return e.offset += t * 12, i;
|
|
1672
|
+
}
|
|
1673
|
+
function Mn(e, t) {
|
|
1674
|
+
const i = (e.view.byteOffset + e.offset) % 4 ? new Float32Array(ye(e.view.buffer, e.view.byteOffset + e.offset, t * 4)) : new Float32Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1675
|
+
return e.offset += t * 4, i;
|
|
1676
|
+
}
|
|
1677
|
+
function Un(e, t) {
|
|
1678
|
+
const i = (e.view.byteOffset + e.offset) % 8 ? new Float64Array(ye(e.view.buffer, e.view.byteOffset + e.offset, t * 8)) : new Float64Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1679
|
+
return e.offset += t * 8, i;
|
|
1680
|
+
}
|
|
1681
|
+
function Pn(e, t) {
|
|
1682
|
+
const i = new Array(t);
|
|
1683
|
+
for (let n = 0; n < t; n++) {
|
|
1684
|
+
const r = e.view.getUint32(e.offset, !0);
|
|
1685
|
+
e.offset += 4, i[n] = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, r), e.offset += r;
|
|
1686
|
+
}
|
|
1687
|
+
return i;
|
|
1688
|
+
}
|
|
1689
|
+
function $n(e, t, i) {
|
|
1690
|
+
const n = new Array(t);
|
|
1691
|
+
for (let r = 0; r < t; r++)
|
|
1692
|
+
n[r] = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, i), e.offset += i;
|
|
1693
|
+
return n;
|
|
1694
|
+
}
|
|
1695
|
+
function ye(e, t, i) {
|
|
1696
|
+
const n = new ArrayBuffer(i);
|
|
1697
|
+
return new Uint8Array(n).set(new Uint8Array(e, t, i)), n;
|
|
1698
|
+
}
|
|
1699
|
+
const Cn = [0, 255, 65535, 16777215, 4294967295];
|
|
1700
|
+
function it(e, t, i, n, r) {
|
|
1701
|
+
for (let f = 0; f < r; f++)
|
|
1702
|
+
i[n + f] = e[t + f];
|
|
1703
|
+
}
|
|
1704
|
+
function Fn(e, t) {
|
|
1705
|
+
const i = e.byteLength, n = t.byteLength;
|
|
1706
|
+
let r = 0, f = 0;
|
|
1707
|
+
for (; r < i; ) {
|
|
1708
|
+
const s = e[r];
|
|
1709
|
+
if (r++, s < 128)
|
|
1710
|
+
break;
|
|
1711
|
+
}
|
|
1712
|
+
if (n && r >= i)
|
|
1713
|
+
throw new Error("invalid snappy length header");
|
|
1714
|
+
for (; r < i; ) {
|
|
1715
|
+
const s = e[r];
|
|
1716
|
+
let o = 0;
|
|
1717
|
+
if (r++, r >= i)
|
|
1718
|
+
throw new Error("missing eof marker");
|
|
1719
|
+
if ((s & 3) === 0) {
|
|
1720
|
+
let c = (s >>> 2) + 1;
|
|
1721
|
+
if (c > 60) {
|
|
1722
|
+
if (r + 3 >= i)
|
|
1723
|
+
throw new Error("snappy error literal pos + 3 >= inputLength");
|
|
1724
|
+
const d = c - 60;
|
|
1725
|
+
c = e[r] + (e[r + 1] << 8) + (e[r + 2] << 16) + (e[r + 3] << 24), c = (c & Cn[d]) + 1, r += d;
|
|
1726
|
+
}
|
|
1727
|
+
if (r + c > i)
|
|
1728
|
+
throw new Error("snappy error literal exceeds input length");
|
|
1729
|
+
it(e, r, t, f, c), r += c, f += c;
|
|
1730
|
+
} else {
|
|
1731
|
+
let c = 0;
|
|
1732
|
+
switch (s & 3) {
|
|
1733
|
+
case 1:
|
|
1734
|
+
o = (s >>> 2 & 7) + 4, c = e[r] + (s >>> 5 << 8), r++;
|
|
1735
|
+
break;
|
|
1736
|
+
case 2:
|
|
1737
|
+
if (i <= r + 1)
|
|
1738
|
+
throw new Error("snappy error end of input");
|
|
1739
|
+
o = (s >>> 2) + 1, c = e[r] + (e[r + 1] << 8), r += 2;
|
|
1740
|
+
break;
|
|
1741
|
+
case 3:
|
|
1742
|
+
if (i <= r + 3)
|
|
1743
|
+
throw new Error("snappy error end of input");
|
|
1744
|
+
o = (s >>> 2) + 1, c = e[r] + (e[r + 1] << 8) + (e[r + 2] << 16) + (e[r + 3] << 24), r += 4;
|
|
1745
|
+
break;
|
|
1746
|
+
}
|
|
1747
|
+
if (c === 0 || isNaN(c))
|
|
1748
|
+
throw new Error(`invalid offset ${c} pos ${r} inputLength ${i}`);
|
|
1749
|
+
if (c > f)
|
|
1750
|
+
throw new Error("cannot copy from before start of buffer");
|
|
1751
|
+
it(t, f - c, t, f, o), f += o;
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
if (f !== n) throw new Error("premature end of input");
|
|
1755
|
+
}
|
|
1756
|
+
function kn(e, t, { type: i, element: n, schemaPath: r }) {
|
|
1757
|
+
const f = new DataView(e.buffer, e.byteOffset, e.byteLength), s = { view: f, offset: 0 };
|
|
1758
|
+
let o;
|
|
1759
|
+
const c = Yn(s, t, r), { definitionLevels: d, numNulls: a } = qn(s, t, r), l = t.num_values - a;
|
|
1760
|
+
if (t.encoding === "PLAIN")
|
|
1761
|
+
o = Oe(s, i, l, n.type_length);
|
|
1762
|
+
else if (t.encoding === "PLAIN_DICTIONARY" || t.encoding === "RLE_DICTIONARY" || t.encoding === "RLE") {
|
|
1763
|
+
const u = i === "BOOLEAN" ? 1 : f.getUint8(s.offset++);
|
|
1764
|
+
u ? (o = new Array(l), i === "BOOLEAN" ? (k(s, u, o), o = o.map((g) => !!g)) : k(s, u, o, f.byteLength - s.offset)) : o = new Uint8Array(l);
|
|
1765
|
+
} else if (t.encoding === "BYTE_STREAM_SPLIT")
|
|
1766
|
+
o = Nt(s, l, i, n.type_length);
|
|
1767
|
+
else if (t.encoding === "DELTA_BINARY_PACKED")
|
|
1768
|
+
o = i === "INT32" ? new Int32Array(l) : new BigInt64Array(l), se(s, l, o);
|
|
1769
|
+
else if (t.encoding === "DELTA_LENGTH_BYTE_ARRAY")
|
|
1770
|
+
o = new Array(l), Lt(s, l, o);
|
|
1771
|
+
else
|
|
1772
|
+
throw new Error(`parquet unsupported encoding: ${t.encoding}`);
|
|
1773
|
+
return { definitionLevels: d, repetitionLevels: c, dataPage: o };
|
|
1774
|
+
}
|
|
1775
|
+
function Yn(e, t, i) {
|
|
1776
|
+
if (i.length > 1) {
|
|
1777
|
+
const n = gt(i);
|
|
1778
|
+
if (n) {
|
|
1779
|
+
const r = new Array(t.num_values);
|
|
1780
|
+
return k(e, me(n), r), r;
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
return [];
|
|
1784
|
+
}
|
|
1785
|
+
function qn(e, t, i) {
|
|
1786
|
+
const n = Be(i);
|
|
1787
|
+
if (!n) return { definitionLevels: [], numNulls: 0 };
|
|
1788
|
+
const r = new Array(t.num_values);
|
|
1789
|
+
k(e, me(n), r);
|
|
1790
|
+
let f = t.num_values;
|
|
1791
|
+
for (const s of r)
|
|
1792
|
+
s === n && f--;
|
|
1793
|
+
return f === 0 && (r.length = 0), { definitionLevels: r, numNulls: f };
|
|
1794
|
+
}
|
|
1795
|
+
function Ne(e, t, i, n) {
|
|
1796
|
+
let r;
|
|
1797
|
+
const f = n?.[i];
|
|
1798
|
+
if (i === "UNCOMPRESSED")
|
|
1799
|
+
r = e;
|
|
1800
|
+
else if (f)
|
|
1801
|
+
r = f(e, t);
|
|
1802
|
+
else if (i === "SNAPPY")
|
|
1803
|
+
r = new Uint8Array(t), Fn(e, r);
|
|
1804
|
+
else
|
|
1805
|
+
throw new Error(`parquet unsupported compression codec: ${i}`);
|
|
1806
|
+
if (r?.length !== t)
|
|
1807
|
+
throw new Error(`parquet decompressed page length ${r?.length} does not match header ${t}`);
|
|
1808
|
+
return r;
|
|
1809
|
+
}
|
|
1810
|
+
function jn(e, t, i) {
|
|
1811
|
+
const r = { view: new DataView(e.buffer, e.byteOffset, e.byteLength), offset: 0 }, { type: f, element: s, schemaPath: o, codec: c, compressors: d } = i, a = t.data_page_header_v2;
|
|
1812
|
+
if (!a) throw new Error("parquet data page header v2 is undefined");
|
|
1813
|
+
const l = Gn(r, a, o);
|
|
1814
|
+
r.offset = a.repetition_levels_byte_length;
|
|
1815
|
+
const u = Vn(r, a, o), g = t.uncompressed_page_size - a.definition_levels_byte_length - a.repetition_levels_byte_length;
|
|
1816
|
+
let h = e.subarray(r.offset);
|
|
1817
|
+
a.is_compressed !== !1 && (h = Ne(h, g, c, d));
|
|
1818
|
+
const m = new DataView(h.buffer, h.byteOffset, h.byteLength), y = { view: m, offset: 0 };
|
|
1819
|
+
let _;
|
|
1820
|
+
const w = a.num_values - a.num_nulls;
|
|
1821
|
+
if (a.encoding === "PLAIN")
|
|
1822
|
+
_ = Oe(y, f, w, s.type_length);
|
|
1823
|
+
else if (a.encoding === "RLE")
|
|
1824
|
+
_ = new Array(w), k(y, 1, _), _ = _.map((p) => !!p);
|
|
1825
|
+
else if (a.encoding === "PLAIN_DICTIONARY" || a.encoding === "RLE_DICTIONARY") {
|
|
1826
|
+
const p = m.getUint8(y.offset++);
|
|
1827
|
+
_ = new Array(w), k(y, p, _, g - 1);
|
|
1828
|
+
} else if (a.encoding === "DELTA_BINARY_PACKED")
|
|
1829
|
+
_ = f === "INT32" ? new Int32Array(w) : new BigInt64Array(w), se(y, w, _);
|
|
1830
|
+
else if (a.encoding === "DELTA_LENGTH_BYTE_ARRAY")
|
|
1831
|
+
_ = new Array(w), Lt(y, w, _);
|
|
1832
|
+
else if (a.encoding === "DELTA_BYTE_ARRAY")
|
|
1833
|
+
_ = new Array(w), Ln(y, w, _);
|
|
1834
|
+
else if (a.encoding === "BYTE_STREAM_SPLIT")
|
|
1835
|
+
_ = Nt(y, w, f, s.type_length);
|
|
1836
|
+
else
|
|
1837
|
+
throw new Error(`parquet unsupported encoding: ${a.encoding}`);
|
|
1838
|
+
return { definitionLevels: u, repetitionLevels: l, dataPage: _ };
|
|
1839
|
+
}
|
|
1840
|
+
function Gn(e, t, i) {
|
|
1841
|
+
const n = gt(i);
|
|
1842
|
+
if (!n) return [];
|
|
1843
|
+
const r = new Array(t.num_values);
|
|
1844
|
+
return k(e, me(n), r, t.repetition_levels_byte_length), r;
|
|
1845
|
+
}
|
|
1846
|
+
function Vn(e, t, i) {
|
|
1847
|
+
const n = Be(i);
|
|
1848
|
+
if (n) {
|
|
1849
|
+
const r = new Array(t.num_values);
|
|
1850
|
+
return k(e, me(n), r, t.definition_levels_byte_length), r;
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
function me(e) {
|
|
1854
|
+
return 32 - Math.clz32(e);
|
|
1855
|
+
}
|
|
1856
|
+
function Rt(e, { groupStart: t, selectStart: i, selectEnd: n }, r, f) {
|
|
1857
|
+
const { pathInSchema: s, schemaPath: o } = r, c = ht(o), d = [];
|
|
1858
|
+
let a, l, u = 0, g = 0;
|
|
1859
|
+
const h = f && (() => {
|
|
1860
|
+
l && f({
|
|
1861
|
+
pathInSchema: s,
|
|
1862
|
+
columnData: l,
|
|
1863
|
+
rowStart: t + u - l.length,
|
|
1864
|
+
rowEnd: t + u
|
|
1865
|
+
});
|
|
1866
|
+
});
|
|
1867
|
+
for (; (c ? u < n : e.offset < e.view.byteLength - 1) && !(e.offset >= e.view.byteLength - 1); ) {
|
|
1868
|
+
const m = zn(e);
|
|
1869
|
+
if (m.type === "DICTIONARY_PAGE") {
|
|
1870
|
+
const { data: y } = rt(e, m, r, a, void 0, 0);
|
|
1871
|
+
y && (a = ct(y, r));
|
|
1872
|
+
} else {
|
|
1873
|
+
const y = l?.length || 0, _ = rt(e, m, r, a, l, i - u);
|
|
1874
|
+
_.skipped ? (d.length || (g += _.skipped), u += _.skipped) : _.data && l === _.data ? u += _.data.length - y : _.data && _.data.length && (h?.(), d.push(_.data), u += _.data.length, l = _.data);
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
return h?.(), { data: d, skipped: g };
|
|
1878
|
+
}
|
|
1879
|
+
function rt(e, t, i, n, r, f) {
|
|
1880
|
+
const { type: s, element: o, schemaPath: c, codec: d, compressors: a } = i, l = new Uint8Array(
|
|
1881
|
+
e.view.buffer,
|
|
1882
|
+
e.view.byteOffset + e.offset,
|
|
1883
|
+
t.compressed_page_size
|
|
1884
|
+
);
|
|
1885
|
+
if (e.offset += t.compressed_page_size, t.type === "DATA_PAGE") {
|
|
1886
|
+
const u = t.data_page_header;
|
|
1887
|
+
if (!u) throw new Error("parquet data page header is undefined");
|
|
1888
|
+
if (f > u.num_values && ht(c))
|
|
1889
|
+
return { skipped: u.num_values };
|
|
1890
|
+
const g = Ne(l, Number(t.uncompressed_page_size), d, a), { definitionLevels: h, repetitionLevels: m, dataPage: y } = kn(g, u, i), _ = Ce(y, n, u.encoding, i), w = Array.isArray(r) ? r : [];
|
|
1891
|
+
return { skipped: 0, data: nt(w, h, m, _, c) };
|
|
1892
|
+
} else if (t.type === "DATA_PAGE_V2") {
|
|
1893
|
+
const u = t.data_page_header_v2;
|
|
1894
|
+
if (!u) throw new Error("parquet data page header v2 is undefined");
|
|
1895
|
+
if (f > u.num_rows)
|
|
1896
|
+
return { skipped: u.num_values };
|
|
1897
|
+
const { definitionLevels: g, repetitionLevels: h, dataPage: m } = jn(l, t, i), y = Ce(m, n, u.encoding, i), _ = Array.isArray(r) ? r : [];
|
|
1898
|
+
return { skipped: 0, data: nt(_, g, h, y, c) };
|
|
1899
|
+
} else if (t.type === "DICTIONARY_PAGE") {
|
|
1900
|
+
const u = t.dictionary_page_header;
|
|
1901
|
+
if (!u) throw new Error("parquet dictionary page header is undefined");
|
|
1902
|
+
const g = Ne(
|
|
1903
|
+
l,
|
|
1904
|
+
Number(t.uncompressed_page_size),
|
|
1905
|
+
d,
|
|
1906
|
+
a
|
|
1907
|
+
), h = { view: new DataView(g.buffer, g.byteOffset, g.byteLength), offset: 0 };
|
|
1908
|
+
return { skipped: 0, data: Oe(h, s, u.num_values, o.type_length) };
|
|
1909
|
+
} else
|
|
1910
|
+
throw new Error(`parquet unsupported page type: ${t.type}`);
|
|
1911
|
+
}
|
|
1912
|
+
function zn(e) {
|
|
1913
|
+
const t = X(e), i = lt[t.field_1], n = t.field_2, r = t.field_3, f = t.field_4, s = t.field_5 && {
|
|
1914
|
+
num_values: t.field_5.field_1,
|
|
1915
|
+
encoding: G[t.field_5.field_2],
|
|
1916
|
+
definition_level_encoding: G[t.field_5.field_3],
|
|
1917
|
+
repetition_level_encoding: G[t.field_5.field_4],
|
|
1918
|
+
statistics: t.field_5.field_5 && {
|
|
1919
|
+
max: t.field_5.field_5.field_1,
|
|
1920
|
+
min: t.field_5.field_5.field_2,
|
|
1921
|
+
null_count: t.field_5.field_5.field_3,
|
|
1922
|
+
distinct_count: t.field_5.field_5.field_4,
|
|
1923
|
+
max_value: t.field_5.field_5.field_5,
|
|
1924
|
+
min_value: t.field_5.field_5.field_6
|
|
1925
|
+
}
|
|
1926
|
+
}, o = t.field_6, c = t.field_7 && {
|
|
1927
|
+
num_values: t.field_7.field_1,
|
|
1928
|
+
encoding: G[t.field_7.field_2],
|
|
1929
|
+
is_sorted: t.field_7.field_3
|
|
1930
|
+
}, d = t.field_8 && {
|
|
1931
|
+
num_values: t.field_8.field_1,
|
|
1932
|
+
num_nulls: t.field_8.field_2,
|
|
1933
|
+
num_rows: t.field_8.field_3,
|
|
1934
|
+
encoding: G[t.field_8.field_4],
|
|
1935
|
+
definition_levels_byte_length: t.field_8.field_5,
|
|
1936
|
+
repetition_levels_byte_length: t.field_8.field_6,
|
|
1937
|
+
is_compressed: t.field_8.field_7 === void 0 ? !0 : t.field_8.field_7,
|
|
1938
|
+
// default true
|
|
1939
|
+
statistics: t.field_8.field_8
|
|
1940
|
+
};
|
|
1941
|
+
return {
|
|
1942
|
+
type: i,
|
|
1943
|
+
uncompressed_page_size: n,
|
|
1944
|
+
compressed_page_size: r,
|
|
1945
|
+
crc: f,
|
|
1946
|
+
data_page_header: s,
|
|
1947
|
+
index_page_header: o,
|
|
1948
|
+
dictionary_page_header: c,
|
|
1949
|
+
data_page_header_v2: d
|
|
1950
|
+
};
|
|
1951
|
+
}
|
|
1952
|
+
function Wn(e, { metadata: t }, i) {
|
|
1953
|
+
const n = [];
|
|
1954
|
+
for (const r of i.chunks) {
|
|
1955
|
+
const { path_in_schema: f } = r.columnMetadata, s = _t(t.schema, f), o = {
|
|
1956
|
+
pathInSchema: f,
|
|
1957
|
+
element: s[s.length - 1].element,
|
|
1958
|
+
schemaPath: s,
|
|
1959
|
+
parsers: { ...oe, ...e.parsers },
|
|
1960
|
+
...e,
|
|
1961
|
+
...r.columnMetadata
|
|
1962
|
+
}, { startByte: c, endByte: d } = r.range;
|
|
1963
|
+
"pageLocations" in r ? n.push({
|
|
1964
|
+
pathInSchema: f,
|
|
1965
|
+
data: ft(e, i, r, r.pageLocations, o)
|
|
1966
|
+
}) : "offsetIndex" in r ? n.push({
|
|
1967
|
+
pathInSchema: f,
|
|
1968
|
+
// fetch offset index
|
|
1969
|
+
data: Promise.resolve(e.file.slice(r.offsetIndex.startByte, r.offsetIndex.endByte)).then((a) => {
|
|
1970
|
+
const l = ve({ view: new DataView(a), offset: 0 }).page_locations;
|
|
1971
|
+
return ft(e, i, r, l, o);
|
|
1972
|
+
})
|
|
1973
|
+
}) : n.push({
|
|
1974
|
+
pathInSchema: f,
|
|
1975
|
+
data: Promise.resolve(e.file.slice(c, d)).then((a) => {
|
|
1976
|
+
const l = { view: new DataView(a), offset: 0 };
|
|
1977
|
+
return Rt(l, i, o, e.onPage);
|
|
1978
|
+
})
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
return {
|
|
1982
|
+
groupStart: i.groupStart,
|
|
1983
|
+
groupRows: i.groupRows,
|
|
1984
|
+
selectStart: i.selectStart,
|
|
1985
|
+
selectEnd: i.selectEnd,
|
|
1986
|
+
asyncColumns: n
|
|
1987
|
+
};
|
|
1988
|
+
}
|
|
1989
|
+
async function ft(e, t, i, n, r) {
|
|
1990
|
+
const { data_page_offset: f, dictionary_page_offset: s } = i.columnMetadata, { selectStart: o, selectEnd: c } = t;
|
|
1991
|
+
let { startByte: d, endByte: a } = i.range, l = -1;
|
|
1992
|
+
const u = s || f < n[0].offset;
|
|
1993
|
+
for (let w = 0; w < n.length; w++) {
|
|
1994
|
+
const p = n[w], I = Number(p.first_row_index), A = w + 1 < n.length ? Number(n[w + 1].first_row_index) : t.groupRows;
|
|
1995
|
+
l < 0 && A > o && (d = Number(p.offset), l = I), I < c && (a = Number(p.offset) + p.compressed_page_size);
|
|
1996
|
+
}
|
|
1997
|
+
l < 0 && (l = 0);
|
|
1998
|
+
let g;
|
|
1999
|
+
if (u && l) {
|
|
2000
|
+
const w = Number(n[0].offset) - i.range.startByte, [p, I] = await Promise.all([
|
|
2001
|
+
e.file.slice(i.range.startByte, Number(n[0].offset)),
|
|
2002
|
+
e.file.slice(d, a)
|
|
2003
|
+
]), A = new Uint8Array(w + I.byteLength);
|
|
2004
|
+
A.set(new Uint8Array(p, 0, w)), A.set(new Uint8Array(I), w), g = new DataView(A.buffer);
|
|
2005
|
+
} else u ? g = new DataView(await e.file.slice(i.range.startByte, a)) : g = new DataView(await e.file.slice(d, a));
|
|
2006
|
+
const h = { view: g, offset: 0 }, m = l ? {
|
|
2007
|
+
...t,
|
|
2008
|
+
groupStart: t.groupStart + l,
|
|
2009
|
+
selectStart: t.selectStart - l,
|
|
2010
|
+
selectEnd: t.selectEnd - l
|
|
2011
|
+
} : t, { data: y, skipped: _ } = Rt(h, m, r, e.onPage);
|
|
2012
|
+
return {
|
|
2013
|
+
data: y,
|
|
2014
|
+
skipped: l + _
|
|
2015
|
+
};
|
|
2016
|
+
}
|
|
2017
|
+
async function st({ asyncColumns: e }, t, i, n, r) {
|
|
2018
|
+
const f = await Promise.all(e.map(
|
|
2019
|
+
(l) => l.data.then(({ skipped: u, data: g }) => ({ skipped: u, data: pt(g) }))
|
|
2020
|
+
)), s = i - t;
|
|
2021
|
+
if (r === "object") {
|
|
2022
|
+
const l = Array(s);
|
|
2023
|
+
for (let u = 0; u < s; u++) {
|
|
2024
|
+
const g = {};
|
|
2025
|
+
for (let h = 0; h < e.length; h++) {
|
|
2026
|
+
const { data: m, skipped: y } = f[h];
|
|
2027
|
+
g[e[h].pathInSchema[0]] = m[t + u - y];
|
|
2028
|
+
}
|
|
2029
|
+
l[u] = g;
|
|
2030
|
+
}
|
|
2031
|
+
return l;
|
|
2032
|
+
}
|
|
2033
|
+
const o = e.map((l) => l.pathInSchema[0]).filter((l) => !n || n.includes(l)), c = n ?? o, d = c.map((l) => e.findIndex((u) => u.pathInSchema[0] === l)), a = Array(s);
|
|
2034
|
+
for (let l = 0; l < s; l++) {
|
|
2035
|
+
const u = Array(e.length);
|
|
2036
|
+
for (let g = 0; g < c.length; g++) {
|
|
2037
|
+
const h = d[g];
|
|
2038
|
+
if (h < 0) throw new Error(`parquet column not found: ${c[g]}`);
|
|
2039
|
+
const { data: m, skipped: y } = f[h];
|
|
2040
|
+
u[g] = m[t + l - y];
|
|
2041
|
+
}
|
|
2042
|
+
a[l] = u;
|
|
2043
|
+
}
|
|
2044
|
+
return a;
|
|
2045
|
+
}
|
|
2046
|
+
function Bt(e, t, i) {
|
|
2047
|
+
const { asyncColumns: n } = e;
|
|
2048
|
+
i = { ...oe, ...i };
|
|
2049
|
+
const r = [];
|
|
2050
|
+
for (const f of t.children)
|
|
2051
|
+
if (f.children.length) {
|
|
2052
|
+
const s = n.filter((o) => o.pathInSchema[0] === f.element.name);
|
|
2053
|
+
if (!s.length) continue;
|
|
2054
|
+
r.push({
|
|
2055
|
+
pathInSchema: f.path,
|
|
2056
|
+
data: (async () => {
|
|
2057
|
+
const o = await Promise.all(s.map((g) => g.data)), c = /* @__PURE__ */ new Map(), d = o.map(({ data: g }) => pt(g)), a = Math.max(...o.map((g) => g.skipped)), l = Math.min(...o.map((g, h) => g.skipped + d[h].length));
|
|
2058
|
+
for (let g = 0; g < s.length; g++) {
|
|
2059
|
+
const h = a - o[g].skipped, m = Math.max(0, l - a);
|
|
2060
|
+
c.set(
|
|
2061
|
+
s[g].pathInSchema.join("."),
|
|
2062
|
+
d[g].slice(h, h + m)
|
|
2063
|
+
);
|
|
2064
|
+
}
|
|
2065
|
+
te(c, f, i);
|
|
2066
|
+
const u = c.get(f.element.name);
|
|
2067
|
+
if (!u) throw new Error("parquet column data not assembled");
|
|
2068
|
+
return { data: [u], skipped: a };
|
|
2069
|
+
})()
|
|
2070
|
+
});
|
|
2071
|
+
} else {
|
|
2072
|
+
const s = n.find((o) => o.pathInSchema[0] === f.element.name);
|
|
2073
|
+
s && r.push(s);
|
|
2074
|
+
}
|
|
2075
|
+
return { ...e, asyncColumns: r };
|
|
2076
|
+
}
|
|
2077
|
+
async function Hn(e) {
|
|
2078
|
+
e.metadata ??= await he(e.file, e);
|
|
2079
|
+
const { rowStart: t = 0, rowEnd: i, columns: n, onChunk: r, onComplete: f, rowFormat: s, filter: o, filterStrict: c = !0 } = e;
|
|
2080
|
+
if (o && s !== "object")
|
|
2081
|
+
throw new Error('parquet filter requires rowFormat: "object"');
|
|
2082
|
+
const d = dn(o);
|
|
2083
|
+
if (d.length) {
|
|
2084
|
+
const y = F(e.metadata).children.map((w) => w.element.name), _ = d.filter((w) => !y.includes(w));
|
|
2085
|
+
if (_.length)
|
|
2086
|
+
throw new Error(`parquet filter columns not found: ${_.join(", ")}`);
|
|
2087
|
+
}
|
|
2088
|
+
let a = n, l = !1;
|
|
2089
|
+
if (n && o) {
|
|
2090
|
+
const y = d.filter((_) => !n.includes(_));
|
|
2091
|
+
y.length && (a = [...n, ...y], l = !0);
|
|
2092
|
+
}
|
|
2093
|
+
let u = a !== n ? { ...e, columns: a } : e;
|
|
2094
|
+
u = await St(u), u = await Qn(u);
|
|
2095
|
+
const g = xt(u);
|
|
2096
|
+
if (!f && !r) {
|
|
2097
|
+
await _e(g);
|
|
2098
|
+
return;
|
|
2099
|
+
}
|
|
2100
|
+
const h = F(e.metadata), m = g.map((y) => Bt(y, h, e.parsers));
|
|
2101
|
+
if (r)
|
|
2102
|
+
for (const y of m)
|
|
2103
|
+
for (const _ of y.asyncColumns)
|
|
2104
|
+
_.data.then(({ data: w, skipped: p }) => {
|
|
2105
|
+
let I = y.groupStart + p;
|
|
2106
|
+
for (const A of w)
|
|
2107
|
+
r({
|
|
2108
|
+
columnName: _.pathInSchema[0],
|
|
2109
|
+
columnData: A,
|
|
2110
|
+
rowStart: I,
|
|
2111
|
+
rowEnd: I + A.length
|
|
2112
|
+
}), I += A.length;
|
|
2113
|
+
}, () => {
|
|
2114
|
+
});
|
|
2115
|
+
if (f) {
|
|
2116
|
+
await _e(m);
|
|
2117
|
+
const y = [];
|
|
2118
|
+
for (const _ of m) {
|
|
2119
|
+
const w = _.selectStart ?? Math.max(t - _.groupStart, 0), p = _.selectEnd ?? Math.min((i ?? 1 / 0) - _.groupStart, _.groupRows), I = s === "object" ? await st(_, w, p, a, "object") : await st(_, w, p, n, "array");
|
|
2120
|
+
if (o) {
|
|
2121
|
+
for (
|
|
2122
|
+
const A of
|
|
2123
|
+
/** @type {Record<string, any>[]} */
|
|
2124
|
+
I
|
|
2125
|
+
)
|
|
2126
|
+
if (z(A, o, c)) {
|
|
2127
|
+
if (l && n)
|
|
2128
|
+
for (const R of d)
|
|
2129
|
+
n.includes(R) || delete A[R];
|
|
2130
|
+
y.push(A);
|
|
2131
|
+
}
|
|
2132
|
+
} else
|
|
2133
|
+
xe(y, I);
|
|
2134
|
+
}
|
|
2135
|
+
f(y);
|
|
2136
|
+
} else
|
|
2137
|
+
await _e(m);
|
|
2138
|
+
}
|
|
2139
|
+
async function _e(e) {
|
|
2140
|
+
const t = e.flatMap((r) => r.asyncColumns.map((f) => f.data)), n = (await Promise.allSettled(t)).find((r) => r.status === "rejected");
|
|
2141
|
+
if (n) throw n.reason;
|
|
2142
|
+
}
|
|
2143
|
+
function xt(e) {
|
|
2144
|
+
if (!e.metadata) throw new Error("parquet requires metadata");
|
|
2145
|
+
const t = mn(e);
|
|
2146
|
+
return e.file = It(e.file, t), t.groups.map((i) => Wn(e, t, i));
|
|
2147
|
+
}
|
|
2148
|
+
async function Kn(e) {
|
|
2149
|
+
if (e.columns?.length !== 1)
|
|
2150
|
+
throw new Error("parquetReadColumn expected columns: [columnName]");
|
|
2151
|
+
e.metadata ??= await he(e.file, e);
|
|
2152
|
+
const t = xt(await St(e)), i = F(e.metadata), n = t.map((f) => Bt(f, i, e.parsers));
|
|
2153
|
+
await _e(n);
|
|
2154
|
+
const r = [];
|
|
2155
|
+
for (const f of n) {
|
|
2156
|
+
const { data: s } = await f.asyncColumns[0].data;
|
|
2157
|
+
for (const o of s)
|
|
2158
|
+
xe(r, o);
|
|
2159
|
+
}
|
|
2160
|
+
return r;
|
|
2161
|
+
}
|
|
2162
|
+
async function St(e) {
|
|
2163
|
+
if (!e.useBloomFilters || !e.filter || !e.metadata) return e;
|
|
2164
|
+
const t = F(e.metadata), i = {};
|
|
2165
|
+
for (const r of t.children) i[r.element.name] = r.element;
|
|
2166
|
+
const n = await pn({
|
|
2167
|
+
file: e.file,
|
|
2168
|
+
metadata: e.metadata,
|
|
2169
|
+
filter: e.filter,
|
|
2170
|
+
filterStrict: e.filterStrict
|
|
2171
|
+
});
|
|
2172
|
+
return (
|
|
2173
|
+
/** @type {BaseParquetReadOptions} */
|
|
2174
|
+
{ ...e, bloomFiltersByGroup: n, schemaElements: i }
|
|
2175
|
+
);
|
|
2176
|
+
}
|
|
2177
|
+
async function Qn(e) {
|
|
2178
|
+
if (!e.usePageIndex || !e.filter || !e.metadata) return e;
|
|
2179
|
+
const { pageRangesByGroup: t, pageLocationsByGroup: i } = await An({
|
|
2180
|
+
file: e.file,
|
|
2181
|
+
metadata: e.metadata,
|
|
2182
|
+
filter: e.filter,
|
|
2183
|
+
filterStrict: e.filterStrict,
|
|
2184
|
+
rowStart: e.rowStart,
|
|
2185
|
+
rowEnd: e.rowEnd,
|
|
2186
|
+
columns: e.columns,
|
|
2187
|
+
// @ts-expect-error bloomFiltersByGroup/schemaElements are attached by withBloomFilters
|
|
2188
|
+
bloomFiltersByGroup: e.bloomFiltersByGroup,
|
|
2189
|
+
// @ts-expect-error bloomFiltersByGroup/schemaElements are attached by withBloomFilters
|
|
2190
|
+
schemaElements: e.schemaElements,
|
|
2191
|
+
parsers: e.parsers
|
|
2192
|
+
});
|
|
2193
|
+
return { ...e, pageRangesByGroup: t, pageLocationsByGroup: i };
|
|
2194
|
+
}
|
|
2195
|
+
function ne(e) {
|
|
2196
|
+
return new Promise((t, i) => {
|
|
2197
|
+
Hn({
|
|
2198
|
+
...e,
|
|
2199
|
+
rowFormat: "object",
|
|
2200
|
+
// force object output
|
|
2201
|
+
onComplete: t
|
|
2202
|
+
}).catch(i);
|
|
2203
|
+
});
|
|
2204
|
+
}
|
|
2205
|
+
async function ei(e) {
|
|
2206
|
+
if (!e.file || !(e.file.byteLength >= 0))
|
|
2207
|
+
throw new Error("parquet expected AsyncBuffer");
|
|
2208
|
+
e.metadata ??= await he(e.file, e);
|
|
2209
|
+
const { metadata: t, rowStart: i = 0, columns: n, orderBy: r, filter: f } = e;
|
|
2210
|
+
if (i < 0) throw new Error("parquet rowStart must be positive");
|
|
2211
|
+
const s = e.rowEnd ?? Number(t.num_rows);
|
|
2212
|
+
if (r && !F(e.metadata).children.map((c) => c.element.name).includes(r))
|
|
2213
|
+
throw new Error(`parquet orderBy column not found: ${r}`);
|
|
2214
|
+
if (f && !r && s < t.num_rows) {
|
|
2215
|
+
const o = [];
|
|
2216
|
+
let c = 0;
|
|
2217
|
+
for (const d of t.row_groups) {
|
|
2218
|
+
const a = c + Number(d.num_rows), l = await ne({
|
|
2219
|
+
...e,
|
|
2220
|
+
rowStart: c,
|
|
2221
|
+
rowEnd: a
|
|
2222
|
+
});
|
|
2223
|
+
if (o.push(...l), o.length >= s) break;
|
|
2224
|
+
c = a;
|
|
2225
|
+
}
|
|
2226
|
+
return o.slice(i, s);
|
|
2227
|
+
} else if (f && r) {
|
|
2228
|
+
const o = n && !n.includes(r) ? [...n, r] : n, c = await ne({
|
|
2229
|
+
...e,
|
|
2230
|
+
rowStart: void 0,
|
|
2231
|
+
rowEnd: void 0,
|
|
2232
|
+
columns: o
|
|
2233
|
+
});
|
|
2234
|
+
if (c.sort((d, a) => ot(d[r], a[r])), o !== n)
|
|
2235
|
+
for (const d of c)
|
|
2236
|
+
delete d[r];
|
|
2237
|
+
return c.slice(i, s);
|
|
2238
|
+
} else {
|
|
2239
|
+
if (f)
|
|
2240
|
+
return (await ne({
|
|
2241
|
+
...e,
|
|
2242
|
+
rowStart: void 0,
|
|
2243
|
+
rowEnd: void 0
|
|
2244
|
+
})).slice(i, s);
|
|
2245
|
+
if (typeof r == "string") {
|
|
2246
|
+
const o = await Kn({
|
|
2247
|
+
...e,
|
|
2248
|
+
rowStart: void 0,
|
|
2249
|
+
rowEnd: void 0,
|
|
2250
|
+
columns: [r]
|
|
2251
|
+
}), c = Array.from(o, (l, u) => u).sort((l, u) => ot(o[l], o[u])).slice(i, s), d = await Zn({ ...e, rows: c });
|
|
2252
|
+
return c.map((l) => d[l]);
|
|
2253
|
+
} else
|
|
2254
|
+
return await ne(e);
|
|
2255
|
+
}
|
|
2256
|
+
}
|
|
2257
|
+
async function Zn(e) {
|
|
2258
|
+
const { file: t, rows: i } = e;
|
|
2259
|
+
e.metadata ??= await he(t, e);
|
|
2260
|
+
const { row_groups: n } = e.metadata, r = Array(n.length).fill(!1);
|
|
2261
|
+
let f = 0;
|
|
2262
|
+
const s = n.map((a) => f += Number(a.num_rows));
|
|
2263
|
+
for (const a of i) {
|
|
2264
|
+
const l = s.findIndex((u) => a < u);
|
|
2265
|
+
r[l] = !0;
|
|
2266
|
+
}
|
|
2267
|
+
const o = [];
|
|
2268
|
+
let c;
|
|
2269
|
+
f = 0;
|
|
2270
|
+
for (let a = 0; a < r.length; a++) {
|
|
2271
|
+
const l = f + Number(n[a].num_rows);
|
|
2272
|
+
r[a] ? c === void 0 && (c = f) : c !== void 0 && (o.push([c, l]), c = void 0), f = l;
|
|
2273
|
+
}
|
|
2274
|
+
c !== void 0 && o.push([c, f]);
|
|
2275
|
+
const d = Array(Number(e.metadata.num_rows));
|
|
2276
|
+
for (const [a, l] of o) {
|
|
2277
|
+
const u = await ne({ ...e, rowStart: a, rowEnd: l });
|
|
2278
|
+
for (let g = a; g < l; g++)
|
|
2279
|
+
d[g] = { __index__: g, ...u[g - a] };
|
|
2280
|
+
}
|
|
2281
|
+
return d;
|
|
2282
|
+
}
|
|
2283
|
+
function ot(e, t) {
|
|
2284
|
+
return e < t ? -1 : e > t ? 1 : 0;
|
|
2285
|
+
}
|
|
2286
|
+
export {
|
|
2287
|
+
Jn as asyncBufferFromUrl,
|
|
2288
|
+
an as byteLengthFromUrl,
|
|
2289
|
+
Xn as cachedAsyncBuffer,
|
|
2290
|
+
pt as flatten,
|
|
2291
|
+
qe as parquetMetadata,
|
|
2292
|
+
he as parquetMetadataAsync,
|
|
2293
|
+
ei as parquetQuery,
|
|
2294
|
+
Hn as parquetRead,
|
|
2295
|
+
ne as parquetReadObjects,
|
|
2296
|
+
F as parquetSchema,
|
|
2297
|
+
nn as readColumnIndex,
|
|
2298
|
+
ve as readOffsetIndex,
|
|
2299
|
+
Fn as snappyUncompress,
|
|
2300
|
+
He as toJson
|
|
2301
|
+
};
|