@gscdump/engine 0.31.10 → 0.32.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/THIRD-PARTY-LICENSES.md +2 -2
- package/dist/_chunks/libs/hyparquet-writer.mjs +186 -2565
- package/dist/_chunks/libs/hyparquet.d.mts +1 -1
- package/dist/_chunks/libs/hyparquet.mjs +206 -500
- package/dist/adapters/hyparquet.mjs +1 -1
- package/dist/rollups.d.mts +1 -1
- package/dist/rollups.mjs +1 -1
- package/package.json +4 -4
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const ParquetTypes
|
|
1
|
+
const ParquetTypes = [
|
|
2
2
|
"BOOLEAN",
|
|
3
3
|
"INT32",
|
|
4
4
|
"INT64",
|
|
@@ -8,7 +8,7 @@ const ParquetTypes$1 = [
|
|
|
8
8
|
"BYTE_ARRAY",
|
|
9
9
|
"FIXED_LEN_BYTE_ARRAY"
|
|
10
10
|
];
|
|
11
|
-
const Encodings
|
|
11
|
+
const Encodings = [
|
|
12
12
|
"PLAIN",
|
|
13
13
|
"GROUP_VAR_INT",
|
|
14
14
|
"PLAIN_DICTIONARY",
|
|
@@ -20,12 +20,12 @@ const Encodings$1 = [
|
|
|
20
20
|
"RLE_DICTIONARY",
|
|
21
21
|
"BYTE_STREAM_SPLIT"
|
|
22
22
|
];
|
|
23
|
-
const FieldRepetitionTypes
|
|
23
|
+
const FieldRepetitionTypes = [
|
|
24
24
|
"REQUIRED",
|
|
25
25
|
"OPTIONAL",
|
|
26
26
|
"REPEATED"
|
|
27
27
|
];
|
|
28
|
-
const ConvertedTypes
|
|
28
|
+
const ConvertedTypes = [
|
|
29
29
|
"UTF8",
|
|
30
30
|
"MAP",
|
|
31
31
|
"MAP_KEY_VALUE",
|
|
@@ -49,7 +49,7 @@ const ConvertedTypes$1 = [
|
|
|
49
49
|
"BSON",
|
|
50
50
|
"INTERVAL"
|
|
51
51
|
];
|
|
52
|
-
const CompressionCodecs
|
|
52
|
+
const CompressionCodecs = [
|
|
53
53
|
"UNCOMPRESSED",
|
|
54
54
|
"SNAPPY",
|
|
55
55
|
"GZIP",
|
|
@@ -59,18 +59,18 @@ const CompressionCodecs$1 = [
|
|
|
59
59
|
"ZSTD",
|
|
60
60
|
"LZ4_RAW"
|
|
61
61
|
];
|
|
62
|
-
const PageTypes
|
|
62
|
+
const PageTypes = [
|
|
63
63
|
"DATA_PAGE",
|
|
64
64
|
"INDEX_PAGE",
|
|
65
65
|
"DICTIONARY_PAGE",
|
|
66
66
|
"DATA_PAGE_V2"
|
|
67
67
|
];
|
|
68
|
-
const BoundaryOrders
|
|
68
|
+
const BoundaryOrders = [
|
|
69
69
|
"UNORDERED",
|
|
70
70
|
"ASCENDING",
|
|
71
71
|
"DESCENDING"
|
|
72
72
|
];
|
|
73
|
-
const EdgeInterpolationAlgorithms
|
|
73
|
+
const EdgeInterpolationAlgorithms = [
|
|
74
74
|
"SPHERICAL",
|
|
75
75
|
"VINCENTY",
|
|
76
76
|
"THOMAS",
|
|
@@ -170,7 +170,7 @@ function readPolygon(reader, flags) {
|
|
|
170
170
|
}
|
|
171
171
|
return rings;
|
|
172
172
|
}
|
|
173
|
-
const decoder$
|
|
173
|
+
const decoder$2 = new TextDecoder();
|
|
174
174
|
const DEFAULT_PARSERS = {
|
|
175
175
|
timestampFromMilliseconds(millis) {
|
|
176
176
|
return new Date(Number(millis));
|
|
@@ -185,7 +185,7 @@ const DEFAULT_PARSERS = {
|
|
|
185
185
|
return /* @__PURE__ */ new Date(days * 864e5);
|
|
186
186
|
},
|
|
187
187
|
stringFromBytes(bytes) {
|
|
188
|
-
return bytes && decoder$
|
|
188
|
+
return bytes && decoder$2.decode(bytes);
|
|
189
189
|
},
|
|
190
190
|
geometryFromBytes(bytes) {
|
|
191
191
|
return bytes && wkbToGeojson({
|
|
@@ -221,7 +221,7 @@ function convert(data, columnDecoder) {
|
|
|
221
221
|
if (ctype === "DECIMAL") {
|
|
222
222
|
const factor = 10 ** -(element.scale || 0);
|
|
223
223
|
const arr = new Array(data.length);
|
|
224
|
-
for (let i = 0; i < arr.length; i++) if (data[i] instanceof Uint8Array) arr[i] = parseDecimal
|
|
224
|
+
for (let i = 0; i < arr.length; i++) if (data[i] instanceof Uint8Array) arr[i] = parseDecimal(data[i]) * factor;
|
|
225
225
|
else arr[i] = Number(data[i]) * factor;
|
|
226
226
|
return arr;
|
|
227
227
|
}
|
|
@@ -229,7 +229,7 @@ function convert(data, columnDecoder) {
|
|
|
229
229
|
if (ctype === "DATE") return Array.from(data).map((v) => parsers.dateFromDays(v));
|
|
230
230
|
if (ctype === "TIMESTAMP_MILLIS") return Array.from(data).map((v) => parsers.timestampFromMilliseconds(v));
|
|
231
231
|
if (ctype === "TIMESTAMP_MICROS") return Array.from(data).map((v) => parsers.timestampFromMicroseconds(v));
|
|
232
|
-
if (ctype === "JSON") return data.map((v) => JSON.parse(decoder$
|
|
232
|
+
if (ctype === "JSON") return data.map((v) => JSON.parse(decoder$2.decode(v)));
|
|
233
233
|
if (ctype === "BSON") throw new Error("parquet bson not supported");
|
|
234
234
|
if (ctype === "INTERVAL") throw new Error("parquet interval not supported");
|
|
235
235
|
if (ltype?.type === "GEOMETRY") return data.map((v) => parsers.geometryFromBytes(v));
|
|
@@ -260,7 +260,7 @@ function convert(data, columnDecoder) {
|
|
|
260
260
|
}
|
|
261
261
|
return data;
|
|
262
262
|
}
|
|
263
|
-
function parseDecimal
|
|
263
|
+
function parseDecimal(bytes) {
|
|
264
264
|
if (!bytes.length) return 0;
|
|
265
265
|
let value = 0n;
|
|
266
266
|
for (const byte of bytes) value = value * 256n + BigInt(byte);
|
|
@@ -283,13 +283,13 @@ function parseFloat16(bytes) {
|
|
|
283
283
|
if (exp === 31) return frac ? NaN : sign * Infinity;
|
|
284
284
|
return sign * 2 ** (exp - 15) * (1 + frac / 1024);
|
|
285
285
|
}
|
|
286
|
-
function schemaTree
|
|
286
|
+
function schemaTree(schema, rootIndex, path) {
|
|
287
287
|
const element = schema[rootIndex];
|
|
288
288
|
const children = [];
|
|
289
289
|
let count = 1;
|
|
290
290
|
if (element.num_children) while (children.length < element.num_children) {
|
|
291
291
|
const childElement = schema[rootIndex + count];
|
|
292
|
-
const child = schemaTree
|
|
292
|
+
const child = schemaTree(schema, rootIndex + count, [...path, childElement.name]);
|
|
293
293
|
count += child.count;
|
|
294
294
|
children.push(child);
|
|
295
295
|
}
|
|
@@ -300,8 +300,8 @@ function schemaTree$1(schema, rootIndex, path) {
|
|
|
300
300
|
path
|
|
301
301
|
};
|
|
302
302
|
}
|
|
303
|
-
function getSchemaPath
|
|
304
|
-
let tree = schemaTree
|
|
303
|
+
function getSchemaPath(schema, name) {
|
|
304
|
+
let tree = schemaTree(schema, 0, []);
|
|
305
305
|
const path = [tree];
|
|
306
306
|
for (const part of name) {
|
|
307
307
|
const child = tree.children.find((child) => child.element.name === part);
|
|
@@ -325,12 +325,12 @@ function getMaxRepetitionLevel(schemaPath) {
|
|
|
325
325
|
for (const { element } of schemaPath) if (element.repetition_type === "REPEATED") maxLevel++;
|
|
326
326
|
return maxLevel;
|
|
327
327
|
}
|
|
328
|
-
function getMaxDefinitionLevel
|
|
328
|
+
function getMaxDefinitionLevel(schemaPath) {
|
|
329
329
|
let maxLevel = 0;
|
|
330
330
|
for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
|
|
331
331
|
return maxLevel;
|
|
332
332
|
}
|
|
333
|
-
function isListLike
|
|
333
|
+
function isListLike(schema) {
|
|
334
334
|
if (!schema) return false;
|
|
335
335
|
if (schema.element.converted_type !== "LIST") return false;
|
|
336
336
|
if (schema.children.length > 1) return false;
|
|
@@ -339,7 +339,7 @@ function isListLike$1(schema) {
|
|
|
339
339
|
if (firstChild.element.repetition_type !== "REPEATED") return false;
|
|
340
340
|
return true;
|
|
341
341
|
}
|
|
342
|
-
function isMapLike
|
|
342
|
+
function isMapLike(schema) {
|
|
343
343
|
if (!schema) return false;
|
|
344
344
|
if (schema.element.converted_type !== "MAP") return false;
|
|
345
345
|
if (schema.children.length > 1) return false;
|
|
@@ -465,12 +465,12 @@ function markGeoColumns(schema, key_value_metadata) {
|
|
|
465
465
|
if (type === "BYTE_ARRAY" && !logical_type) schema[i].logical_type = columns.get(name);
|
|
466
466
|
}
|
|
467
467
|
}
|
|
468
|
-
const defaultInitialFetchSize
|
|
469
|
-
const decoder$
|
|
468
|
+
const defaultInitialFetchSize = 1 << 19;
|
|
469
|
+
const decoder$1 = new TextDecoder();
|
|
470
470
|
function decode(value) {
|
|
471
|
-
return value && decoder$
|
|
471
|
+
return value && decoder$1.decode(value);
|
|
472
472
|
}
|
|
473
|
-
async function parquetMetadataAsync(asyncBuffer, { parsers, initialFetchSize = defaultInitialFetchSize
|
|
473
|
+
async function parquetMetadataAsync(asyncBuffer, { parsers, initialFetchSize = defaultInitialFetchSize, geoparquet = true } = {}) {
|
|
474
474
|
if (!asyncBuffer || !(asyncBuffer.byteLength >= 0)) throw new Error("parquet expected AsyncBuffer");
|
|
475
475
|
const footerOffset = Math.max(0, asyncBuffer.byteLength - initialFetchSize);
|
|
476
476
|
const footerBuffer = await asyncBuffer.slice(footerOffset, asyncBuffer.byteLength);
|
|
@@ -512,12 +512,12 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
|
|
|
512
512
|
});
|
|
513
513
|
const version = metadata.field_1;
|
|
514
514
|
const schema = metadata.field_2.map((field) => ({
|
|
515
|
-
type: ParquetTypes
|
|
515
|
+
type: ParquetTypes[field.field_1],
|
|
516
516
|
type_length: field.field_2,
|
|
517
|
-
repetition_type: FieldRepetitionTypes
|
|
517
|
+
repetition_type: FieldRepetitionTypes[field.field_3],
|
|
518
518
|
name: decode(field.field_4),
|
|
519
519
|
num_children: field.field_5,
|
|
520
|
-
converted_type: ConvertedTypes
|
|
520
|
+
converted_type: ConvertedTypes[field.field_6],
|
|
521
521
|
scale: field.field_7,
|
|
522
522
|
precision: field.field_8,
|
|
523
523
|
field_id: field.field_9,
|
|
@@ -530,10 +530,10 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
|
|
|
530
530
|
file_path: decode(column.field_1),
|
|
531
531
|
file_offset: column.field_2,
|
|
532
532
|
meta_data: column.field_3 && {
|
|
533
|
-
type: ParquetTypes
|
|
534
|
-
encodings: column.field_3.field_2?.map((e) => Encodings
|
|
533
|
+
type: ParquetTypes[column.field_3.field_1],
|
|
534
|
+
encodings: column.field_3.field_2?.map((e) => Encodings[e]),
|
|
535
535
|
path_in_schema: column.field_3.field_3.map(decode),
|
|
536
|
-
codec: CompressionCodecs
|
|
536
|
+
codec: CompressionCodecs[column.field_3.field_4],
|
|
537
537
|
num_values: column.field_3.field_5,
|
|
538
538
|
total_uncompressed_size: column.field_3.field_6,
|
|
539
539
|
total_compressed_size: column.field_3.field_7,
|
|
@@ -546,8 +546,8 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
|
|
|
546
546
|
dictionary_page_offset: column.field_3.field_11,
|
|
547
547
|
statistics: convertStats(column.field_3.field_12, columnSchema[columnIndex], parsers),
|
|
548
548
|
encoding_stats: column.field_3.field_13?.map((encodingStat) => ({
|
|
549
|
-
page_type: PageTypes
|
|
550
|
-
encoding: Encodings
|
|
549
|
+
page_type: PageTypes[encodingStat.field_1],
|
|
550
|
+
encoding: Encodings[encodingStat.field_2],
|
|
551
551
|
count: encodingStat.field_3
|
|
552
552
|
})),
|
|
553
553
|
bloom_filter_offset: column.field_3.field_14,
|
|
@@ -606,7 +606,7 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
|
|
|
606
606
|
};
|
|
607
607
|
}
|
|
608
608
|
function parquetSchema({ schema }) {
|
|
609
|
-
return getSchemaPath
|
|
609
|
+
return getSchemaPath(schema, [])[0];
|
|
610
610
|
}
|
|
611
611
|
function logicalType(logicalType) {
|
|
612
612
|
if (logicalType?.field_1) return { type: "STRING" };
|
|
@@ -650,7 +650,7 @@ function logicalType(logicalType) {
|
|
|
650
650
|
if (logicalType?.field_18) return {
|
|
651
651
|
type: "GEOGRAPHY",
|
|
652
652
|
crs: decode(logicalType.field_18.field_1),
|
|
653
|
-
algorithm: EdgeInterpolationAlgorithms
|
|
653
|
+
algorithm: EdgeInterpolationAlgorithms[logicalType.field_18.field_2]
|
|
654
654
|
};
|
|
655
655
|
return logicalType;
|
|
656
656
|
}
|
|
@@ -688,7 +688,7 @@ function convertMetadata(value, schema, parsers) {
|
|
|
688
688
|
if (type === "INT64" && logical_type?.type === "TIMESTAMP") return parsers.timestampFromMilliseconds(view.getBigInt64(0, true));
|
|
689
689
|
if (type === "INT32" && view.byteLength === 4) return view.getInt32(0, true);
|
|
690
690
|
if (type === "INT64" && view.byteLength === 8) return view.getBigInt64(0, true);
|
|
691
|
-
if (converted_type === "DECIMAL") return parseDecimal
|
|
691
|
+
if (converted_type === "DECIMAL") return parseDecimal(value) * 10 ** -(schema.scale || 0);
|
|
692
692
|
if (logical_type?.type === "FLOAT16") return parseFloat16(value);
|
|
693
693
|
if (logical_type?.type === "UUID") return parsers.uuidFromBytes(value);
|
|
694
694
|
if (type === "FIXED_LEN_BYTE_ARRAY") return value;
|
|
@@ -705,75 +705,75 @@ function readOffsetIndex(reader) {
|
|
|
705
705
|
unencoded_byte_array_data_bytes: thrift.field_2
|
|
706
706
|
};
|
|
707
707
|
}
|
|
708
|
-
const MASK
|
|
709
|
-
const PRIME1
|
|
710
|
-
const PRIME2
|
|
711
|
-
const PRIME3
|
|
712
|
-
const PRIME4
|
|
713
|
-
const PRIME5
|
|
714
|
-
function rotl64
|
|
715
|
-
return (x << r | x >> 64n - r) & MASK
|
|
716
|
-
}
|
|
717
|
-
function round
|
|
718
|
-
acc = acc + val * PRIME2
|
|
719
|
-
acc = rotl64
|
|
720
|
-
return acc * PRIME1
|
|
721
|
-
}
|
|
722
|
-
function mergeRound
|
|
723
|
-
acc ^= round
|
|
724
|
-
return acc * PRIME1
|
|
725
|
-
}
|
|
726
|
-
function xxhash64
|
|
708
|
+
const MASK = 18446744073709551615n;
|
|
709
|
+
const PRIME1 = 11400714785074694791n;
|
|
710
|
+
const PRIME2 = 14029467366897019727n;
|
|
711
|
+
const PRIME3 = 1609587929392839161n;
|
|
712
|
+
const PRIME4 = 9650029242287828579n;
|
|
713
|
+
const PRIME5 = 2870177450012600261n;
|
|
714
|
+
function rotl64(x, r) {
|
|
715
|
+
return (x << r | x >> 64n - r) & MASK;
|
|
716
|
+
}
|
|
717
|
+
function round(acc, val) {
|
|
718
|
+
acc = acc + val * PRIME2 & MASK;
|
|
719
|
+
acc = rotl64(acc, 31n);
|
|
720
|
+
return acc * PRIME1 & MASK;
|
|
721
|
+
}
|
|
722
|
+
function mergeRound(acc, val) {
|
|
723
|
+
acc ^= round(0n, val);
|
|
724
|
+
return acc * PRIME1 + PRIME4 & MASK;
|
|
725
|
+
}
|
|
726
|
+
function xxhash64(input, seed = 0n) {
|
|
727
727
|
const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
|
|
728
728
|
const len = input.byteLength;
|
|
729
729
|
let offset = 0;
|
|
730
730
|
let h64;
|
|
731
731
|
if (len >= 32) {
|
|
732
|
-
let v1 = seed + PRIME1
|
|
733
|
-
let v2 = seed + PRIME2
|
|
732
|
+
let v1 = seed + PRIME1 + PRIME2 & MASK;
|
|
733
|
+
let v2 = seed + PRIME2 & MASK;
|
|
734
734
|
let v3 = seed;
|
|
735
|
-
let v4 = seed - PRIME1
|
|
735
|
+
let v4 = seed - PRIME1 & MASK;
|
|
736
736
|
while (offset + 32 <= len) {
|
|
737
|
-
v1 = round
|
|
737
|
+
v1 = round(v1, view.getBigUint64(offset, true));
|
|
738
738
|
offset += 8;
|
|
739
|
-
v2 = round
|
|
739
|
+
v2 = round(v2, view.getBigUint64(offset, true));
|
|
740
740
|
offset += 8;
|
|
741
|
-
v3 = round
|
|
741
|
+
v3 = round(v3, view.getBigUint64(offset, true));
|
|
742
742
|
offset += 8;
|
|
743
|
-
v4 = round
|
|
743
|
+
v4 = round(v4, view.getBigUint64(offset, true));
|
|
744
744
|
offset += 8;
|
|
745
745
|
}
|
|
746
|
-
h64 = rotl64
|
|
747
|
-
h64 = mergeRound
|
|
748
|
-
h64 = mergeRound
|
|
749
|
-
h64 = mergeRound
|
|
750
|
-
h64 = mergeRound
|
|
751
|
-
} else h64 = seed + PRIME5
|
|
752
|
-
h64 = h64 + BigInt(len) & MASK
|
|
746
|
+
h64 = rotl64(v1, 1n) + rotl64(v2, 7n) + rotl64(v3, 12n) + rotl64(v4, 18n) & MASK;
|
|
747
|
+
h64 = mergeRound(h64, v1);
|
|
748
|
+
h64 = mergeRound(h64, v2);
|
|
749
|
+
h64 = mergeRound(h64, v3);
|
|
750
|
+
h64 = mergeRound(h64, v4);
|
|
751
|
+
} else h64 = seed + PRIME5 & MASK;
|
|
752
|
+
h64 = h64 + BigInt(len) & MASK;
|
|
753
753
|
while (offset + 8 <= len) {
|
|
754
|
-
h64 ^= round
|
|
755
|
-
h64 = rotl64
|
|
754
|
+
h64 ^= round(0n, view.getBigUint64(offset, true));
|
|
755
|
+
h64 = rotl64(h64, 27n) * PRIME1 + PRIME4 & MASK;
|
|
756
756
|
offset += 8;
|
|
757
757
|
}
|
|
758
758
|
if (offset + 4 <= len) {
|
|
759
|
-
h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1
|
|
760
|
-
h64 = rotl64
|
|
759
|
+
h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1 & MASK;
|
|
760
|
+
h64 = rotl64(h64, 23n) * PRIME2 + PRIME3 & MASK;
|
|
761
761
|
offset += 4;
|
|
762
762
|
}
|
|
763
763
|
while (offset < len) {
|
|
764
|
-
h64 ^= BigInt(view.getUint8(offset)) * PRIME5
|
|
765
|
-
h64 = rotl64
|
|
764
|
+
h64 ^= BigInt(view.getUint8(offset)) * PRIME5 & MASK;
|
|
765
|
+
h64 = rotl64(h64, 11n) * PRIME1 & MASK;
|
|
766
766
|
offset += 1;
|
|
767
767
|
}
|
|
768
768
|
h64 ^= h64 >> 33n;
|
|
769
|
-
h64 = h64 * PRIME2
|
|
769
|
+
h64 = h64 * PRIME2 & MASK;
|
|
770
770
|
h64 ^= h64 >> 29n;
|
|
771
|
-
h64 = h64 * PRIME3
|
|
771
|
+
h64 = h64 * PRIME3 & MASK;
|
|
772
772
|
h64 ^= h64 >> 32n;
|
|
773
773
|
return h64;
|
|
774
774
|
}
|
|
775
|
-
const textEncoder
|
|
776
|
-
const SALT
|
|
775
|
+
const textEncoder = new TextEncoder();
|
|
776
|
+
const SALT = new Uint32Array([
|
|
777
777
|
1203114875,
|
|
778
778
|
1150766481,
|
|
779
779
|
2284105051,
|
|
@@ -789,7 +789,7 @@ function blockIndex(hash, numBlocks) {
|
|
|
789
789
|
function blockMask(hash) {
|
|
790
790
|
const m = /* @__PURE__ */ new Uint32Array(8);
|
|
791
791
|
const low = Number(hash & 4294967295n) | 0;
|
|
792
|
-
for (let i = 0; i < 8; i++) m[i] = 1 << (Math.imul(low, SALT
|
|
792
|
+
for (let i = 0; i < 8; i++) m[i] = 1 << (Math.imul(low, SALT[i]) >>> 27);
|
|
793
793
|
return m;
|
|
794
794
|
}
|
|
795
795
|
function sbbfContains(blocks, hash) {
|
|
@@ -815,24 +815,24 @@ function readBloomFilter(reader) {
|
|
|
815
815
|
blocks
|
|
816
816
|
};
|
|
817
817
|
}
|
|
818
|
-
function hashParquetValue
|
|
818
|
+
function hashParquetValue(value, element) {
|
|
819
819
|
if (value === null || value === void 0) return void 0;
|
|
820
820
|
const { type, converted_type, logical_type } = element;
|
|
821
821
|
if (type === "BOOLEAN") {
|
|
822
822
|
if (typeof value !== "boolean") return void 0;
|
|
823
|
-
return xxhash64
|
|
823
|
+
return xxhash64(new Uint8Array([value ? 1 : 0]));
|
|
824
824
|
}
|
|
825
825
|
if (type === "FLOAT") {
|
|
826
826
|
if (typeof value !== "number") return void 0;
|
|
827
827
|
const buf = /* @__PURE__ */ new ArrayBuffer(4);
|
|
828
828
|
new DataView(buf).setFloat32(0, value, true);
|
|
829
|
-
return xxhash64
|
|
829
|
+
return xxhash64(new Uint8Array(buf));
|
|
830
830
|
}
|
|
831
831
|
if (type === "DOUBLE") {
|
|
832
832
|
if (typeof value !== "number") return void 0;
|
|
833
833
|
const buf = /* @__PURE__ */ new ArrayBuffer(8);
|
|
834
834
|
new DataView(buf).setFloat64(0, value, true);
|
|
835
|
-
return xxhash64
|
|
835
|
+
return xxhash64(new Uint8Array(buf));
|
|
836
836
|
}
|
|
837
837
|
if (type === "INT32") {
|
|
838
838
|
if (converted_type === "DATE" || converted_type === "DECIMAL" || converted_type === "TIME_MILLIS") return void 0;
|
|
@@ -840,7 +840,7 @@ function hashParquetValue$1(value, element) {
|
|
|
840
840
|
if (typeof value !== "number" || !Number.isInteger(value)) return void 0;
|
|
841
841
|
const buf = /* @__PURE__ */ new ArrayBuffer(4);
|
|
842
842
|
new DataView(buf).setInt32(0, value | 0, true);
|
|
843
|
-
return xxhash64
|
|
843
|
+
return xxhash64(new Uint8Array(buf));
|
|
844
844
|
}
|
|
845
845
|
if (type === "INT64") {
|
|
846
846
|
if (converted_type === "TIMESTAMP_MILLIS" || converted_type === "TIMESTAMP_MICROS") return void 0;
|
|
@@ -852,21 +852,21 @@ function hashParquetValue$1(value, element) {
|
|
|
852
852
|
else return void 0;
|
|
853
853
|
const buf = /* @__PURE__ */ new ArrayBuffer(8);
|
|
854
854
|
new DataView(buf).setBigUint64(0, BigInt.asUintN(64, bigValue), true);
|
|
855
|
-
return xxhash64
|
|
855
|
+
return xxhash64(new Uint8Array(buf));
|
|
856
856
|
}
|
|
857
857
|
if (type === "BYTE_ARRAY") {
|
|
858
858
|
if (converted_type === "JSON" || converted_type === "BSON" || converted_type === "DECIMAL") return void 0;
|
|
859
859
|
if (logical_type?.type === "JSON" || logical_type?.type === "BSON" || logical_type?.type === "VARIANT") return void 0;
|
|
860
860
|
if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
|
|
861
|
-
if (typeof value === "string") return xxhash64
|
|
862
|
-
if (value instanceof Uint8Array) return xxhash64
|
|
861
|
+
if (typeof value === "string") return xxhash64(textEncoder.encode(value));
|
|
862
|
+
if (value instanceof Uint8Array) return xxhash64(value);
|
|
863
863
|
return;
|
|
864
864
|
}
|
|
865
865
|
if (type === "FIXED_LEN_BYTE_ARRAY") {
|
|
866
866
|
if (converted_type === "DECIMAL" || converted_type === "INTERVAL") return void 0;
|
|
867
867
|
if (logical_type?.type === "DECIMAL" || logical_type?.type === "UUID" || logical_type?.type === "FLOAT16") return void 0;
|
|
868
868
|
if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
|
|
869
|
-
if (value instanceof Uint8Array) return xxhash64
|
|
869
|
+
if (value instanceof Uint8Array) return xxhash64(value);
|
|
870
870
|
return;
|
|
871
871
|
}
|
|
872
872
|
}
|
|
@@ -893,18 +893,18 @@ function walkBloomEligible(filter, out) {
|
|
|
893
893
|
} else out.add(field);
|
|
894
894
|
}
|
|
895
895
|
}
|
|
896
|
-
function toJson
|
|
896
|
+
function toJson(obj) {
|
|
897
897
|
if (obj === void 0) return null;
|
|
898
898
|
if (typeof obj === "bigint") return Number(obj);
|
|
899
899
|
if (Object.is(obj, -0)) return 0;
|
|
900
|
-
if (Array.isArray(obj)) return obj.map(toJson
|
|
900
|
+
if (Array.isArray(obj)) return obj.map(toJson);
|
|
901
901
|
if (obj instanceof Uint8Array) return Array.from(obj);
|
|
902
902
|
if (obj instanceof Date) return obj.toISOString();
|
|
903
903
|
if (obj instanceof Object) {
|
|
904
904
|
const newObj = {};
|
|
905
905
|
for (const key of Object.keys(obj)) {
|
|
906
906
|
if (obj[key] === void 0) continue;
|
|
907
|
-
newObj[key] = toJson
|
|
907
|
+
newObj[key] = toJson(obj[key]);
|
|
908
908
|
}
|
|
909
909
|
return newObj;
|
|
910
910
|
}
|
|
@@ -932,6 +932,102 @@ function equals(a, b, strict = true) {
|
|
|
932
932
|
for (const k of aKeys) if (!equals(a[k], b[k], strict)) return false;
|
|
933
933
|
return true;
|
|
934
934
|
}
|
|
935
|
+
async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
|
|
936
|
+
const controller = new AbortController();
|
|
937
|
+
const headers = new Headers(requestInit.headers);
|
|
938
|
+
headers.set("Range", "bytes=0-0");
|
|
939
|
+
const res = await fetchFn(url, {
|
|
940
|
+
...requestInit,
|
|
941
|
+
headers,
|
|
942
|
+
signal: controller.signal
|
|
943
|
+
});
|
|
944
|
+
if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
|
|
945
|
+
if (res.status === 206) {
|
|
946
|
+
const contentRange = res.headers.get("Content-Range");
|
|
947
|
+
if (!contentRange) throw new Error("missing content-range header");
|
|
948
|
+
const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
|
|
949
|
+
if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
|
|
950
|
+
return parseInt(match[1]);
|
|
951
|
+
}
|
|
952
|
+
if (res.status === 200) {
|
|
953
|
+
const contentLength = res.headers.get("Content-Length");
|
|
954
|
+
controller.abort();
|
|
955
|
+
if (contentLength) return parseInt(contentLength);
|
|
956
|
+
}
|
|
957
|
+
throw new Error("server does not support range requests and missing content-length");
|
|
958
|
+
}
|
|
959
|
+
async function byteLengthFromUrl(url, requestInit, customFetch) {
|
|
960
|
+
const fetch = customFetch ?? globalThis.fetch;
|
|
961
|
+
const res = await fetch(url, {
|
|
962
|
+
...requestInit,
|
|
963
|
+
method: "HEAD"
|
|
964
|
+
});
|
|
965
|
+
if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
966
|
+
if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
|
|
967
|
+
const length = res.headers.get("Content-Length");
|
|
968
|
+
if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
969
|
+
return parseInt(length);
|
|
970
|
+
}
|
|
971
|
+
async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
|
|
972
|
+
if (!url) throw new Error("missing url");
|
|
973
|
+
const fetch = customFetch ?? globalThis.fetch;
|
|
974
|
+
byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
|
|
975
|
+
let buffer = void 0;
|
|
976
|
+
const init = requestInit || {};
|
|
977
|
+
return {
|
|
978
|
+
byteLength,
|
|
979
|
+
async slice(start, end) {
|
|
980
|
+
if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
|
|
981
|
+
const headers = new Headers(init.headers);
|
|
982
|
+
const endStr = end === void 0 ? "" : end - 1;
|
|
983
|
+
headers.set("Range", `bytes=${start}-${endStr}`);
|
|
984
|
+
const res = await fetch(url, {
|
|
985
|
+
...init,
|
|
986
|
+
headers
|
|
987
|
+
});
|
|
988
|
+
if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
|
|
989
|
+
if (res.status === 200) {
|
|
990
|
+
buffer = res.arrayBuffer();
|
|
991
|
+
return buffer.then((buffer) => buffer.slice(start, end));
|
|
992
|
+
} else if (res.status === 206) return res.arrayBuffer();
|
|
993
|
+
else throw new Error(`fetch received unexpected status code ${res.status}`);
|
|
994
|
+
}
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
|
|
998
|
+
if (byteLength < minSize) {
|
|
999
|
+
const buffer = slice(0, byteLength);
|
|
1000
|
+
return {
|
|
1001
|
+
byteLength,
|
|
1002
|
+
async slice(start, end) {
|
|
1003
|
+
return (await buffer).slice(start, end);
|
|
1004
|
+
}
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
const cache = /* @__PURE__ */ new Map();
|
|
1008
|
+
return {
|
|
1009
|
+
byteLength,
|
|
1010
|
+
slice(start, end) {
|
|
1011
|
+
const key = cacheKey(start, end, byteLength);
|
|
1012
|
+
const cached = cache.get(key);
|
|
1013
|
+
if (cached) return cached;
|
|
1014
|
+
const promise = slice(start, end);
|
|
1015
|
+
cache.set(key, promise);
|
|
1016
|
+
return promise;
|
|
1017
|
+
}
|
|
1018
|
+
};
|
|
1019
|
+
}
|
|
1020
|
+
function cacheKey(start, end, size) {
|
|
1021
|
+
if (start < 0) {
|
|
1022
|
+
if (end !== void 0) throw new Error(`invalid suffix range [${start}, ${end}]`);
|
|
1023
|
+
if (size === void 0) return `${start},`;
|
|
1024
|
+
return `${size + start},${size}`;
|
|
1025
|
+
} else if (end !== void 0) {
|
|
1026
|
+
if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`);
|
|
1027
|
+
return `${start},${end}`;
|
|
1028
|
+
} else if (size === void 0) return `${start},`;
|
|
1029
|
+
else return `${start},${size}`;
|
|
1030
|
+
}
|
|
935
1031
|
function flatten(chunks) {
|
|
936
1032
|
if (!chunks) return [];
|
|
937
1033
|
if (chunks.length === 1) return chunks[0];
|
|
@@ -1010,13 +1106,13 @@ function canSkipRowGroup({ rowGroup, physicalColumns, filter, strict = true, blo
|
|
|
1010
1106
|
}
|
|
1011
1107
|
if (bloom && element) {
|
|
1012
1108
|
if (operator === "$eq") {
|
|
1013
|
-
const hash = hashParquetValue
|
|
1109
|
+
const hash = hashParquetValue(target, element);
|
|
1014
1110
|
if (hash !== void 0 && !sbbfContains(bloom.blocks, hash)) return true;
|
|
1015
1111
|
}
|
|
1016
1112
|
if (operator === "$in" && Array.isArray(target) && target.length > 0) {
|
|
1017
1113
|
let allAbsent = true;
|
|
1018
1114
|
for (const v of target) {
|
|
1019
|
-
const h = hashParquetValue
|
|
1115
|
+
const h = hashParquetValue(v, element);
|
|
1020
1116
|
if (h === void 0 || sbbfContains(bloom.blocks, h)) {
|
|
1021
1117
|
allAbsent = false;
|
|
1022
1118
|
break;
|
|
@@ -1176,7 +1272,7 @@ function prefetchAsyncBuffer(file, { fetches }) {
|
|
|
1176
1272
|
}
|
|
1177
1273
|
};
|
|
1178
1274
|
}
|
|
1179
|
-
const decoder
|
|
1275
|
+
const decoder = new TextDecoder();
|
|
1180
1276
|
const metadataCache = /* @__PURE__ */ new WeakMap();
|
|
1181
1277
|
function decodeVariantColumn(value, parsers = DEFAULT_PARSERS) {
|
|
1182
1278
|
if (Array.isArray(value)) return value.map((entry) => decodeVariantColumn(entry, parsers));
|
|
@@ -1240,7 +1336,7 @@ function parseVariantMetadata(bytes) {
|
|
|
1240
1336
|
const start = offsets[i];
|
|
1241
1337
|
const end = offsets[i + 1];
|
|
1242
1338
|
const strBytes = new Uint8Array(bytes.buffer, bytes.byteOffset + base + start, end - start);
|
|
1243
|
-
dictionary[i] = decoder
|
|
1339
|
+
dictionary[i] = decoder.decode(strBytes);
|
|
1244
1340
|
}
|
|
1245
1341
|
const metadata = {
|
|
1246
1342
|
dictionary,
|
|
@@ -1264,7 +1360,7 @@ function readVariant(reader, metadata, parsers) {
|
|
|
1264
1360
|
if (basicType === 3) return readVariantArray(reader, header, metadata, parsers);
|
|
1265
1361
|
const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, header);
|
|
1266
1362
|
reader.offset += header;
|
|
1267
|
-
return decoder
|
|
1363
|
+
return decoder.decode(bytes);
|
|
1268
1364
|
}
|
|
1269
1365
|
function readVariantPrimitive(reader, typeId, parsers) {
|
|
1270
1366
|
switch (typeId) {
|
|
@@ -1318,7 +1414,7 @@ function readVariantPrimitive(reader, typeId, parsers) {
|
|
|
1318
1414
|
case 15: return readVariantBinary(reader);
|
|
1319
1415
|
case 16: {
|
|
1320
1416
|
const bytes = readVariantBinary(reader);
|
|
1321
|
-
return decoder
|
|
1417
|
+
return decoder.decode(bytes);
|
|
1322
1418
|
}
|
|
1323
1419
|
case 17: {
|
|
1324
1420
|
const value = reader.view.getBigInt64(reader.offset, true);
|
|
@@ -1400,7 +1496,7 @@ function readVariantBinary(reader) {
|
|
|
1400
1496
|
return bytes;
|
|
1401
1497
|
}
|
|
1402
1498
|
function assembleLists(output, definitionLevels, repetitionLevels, values, schemaPath) {
|
|
1403
|
-
const maxDefinitionLevel = getMaxDefinitionLevel
|
|
1499
|
+
const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
|
|
1404
1500
|
if (!definitionLevels?.length && !repetitionLevels.length) {
|
|
1405
1501
|
if (!maxDefinitionLevel || !values.length) return values;
|
|
1406
1502
|
definitionLevels = new Array(values.length).fill(maxDefinitionLevel);
|
|
@@ -1460,7 +1556,7 @@ function assembleNested(subcolumnData, schema, parsers, depth = 0) {
|
|
|
1460
1556
|
const path = schema.path.join(".");
|
|
1461
1557
|
const optional = schema.element.repetition_type === "OPTIONAL";
|
|
1462
1558
|
const nextDepth = optional ? depth + 1 : depth;
|
|
1463
|
-
if (isListLike
|
|
1559
|
+
if (isListLike(schema)) {
|
|
1464
1560
|
let sublist = schema.children[0];
|
|
1465
1561
|
let subDepth = nextDepth;
|
|
1466
1562
|
if (sublist.children.length === 1) {
|
|
@@ -1476,7 +1572,7 @@ function assembleNested(subcolumnData, schema, parsers, depth = 0) {
|
|
|
1476
1572
|
subcolumnData.delete(subcolumn);
|
|
1477
1573
|
return;
|
|
1478
1574
|
}
|
|
1479
|
-
if (isMapLike
|
|
1575
|
+
if (isMapLike(schema)) {
|
|
1480
1576
|
const mapName = schema.children[0].element.name;
|
|
1481
1577
|
assembleNested(subcolumnData, schema.children[0].children[0], parsers, nextDepth + 1);
|
|
1482
1578
|
assembleNested(subcolumnData, schema.children[0].children[1], parsers, nextDepth + 1);
|
|
@@ -1869,7 +1965,7 @@ function readRepetitionLevels(reader, daph, schemaPath) {
|
|
|
1869
1965
|
return [];
|
|
1870
1966
|
}
|
|
1871
1967
|
function readDefinitionLevels(reader, daph, schemaPath) {
|
|
1872
|
-
const maxDefinitionLevel = getMaxDefinitionLevel
|
|
1968
|
+
const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
|
|
1873
1969
|
if (!maxDefinitionLevel) return {
|
|
1874
1970
|
definitionLevels: [],
|
|
1875
1971
|
numNulls: 0
|
|
@@ -1951,7 +2047,7 @@ function readRepetitionLevelsV2(reader, daph2, schemaPath) {
|
|
|
1951
2047
|
return values;
|
|
1952
2048
|
}
|
|
1953
2049
|
function readDefinitionLevelsV2(reader, daph2, schemaPath) {
|
|
1954
|
-
const maxDefinitionLevel = getMaxDefinitionLevel
|
|
2050
|
+
const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
|
|
1955
2051
|
if (maxDefinitionLevel) {
|
|
1956
2052
|
const values = new Array(daph2.num_values);
|
|
1957
2053
|
readRleBitPackedHybrid(reader, bitWidth(maxDefinitionLevel), values, daph2.definition_levels_byte_length);
|
|
@@ -2044,15 +2140,15 @@ function readPage(reader, header, columnDecoder, dictionary, previousChunk, page
|
|
|
2044
2140
|
function parquetHeader(reader) {
|
|
2045
2141
|
const header = deserializeTCompactProtocol(reader);
|
|
2046
2142
|
return {
|
|
2047
|
-
type: PageTypes
|
|
2143
|
+
type: PageTypes[header.field_1],
|
|
2048
2144
|
uncompressed_page_size: header.field_2,
|
|
2049
2145
|
compressed_page_size: header.field_3,
|
|
2050
2146
|
crc: header.field_4,
|
|
2051
2147
|
data_page_header: header.field_5 && {
|
|
2052
2148
|
num_values: header.field_5.field_1,
|
|
2053
|
-
encoding: Encodings
|
|
2054
|
-
definition_level_encoding: Encodings
|
|
2055
|
-
repetition_level_encoding: Encodings
|
|
2149
|
+
encoding: Encodings[header.field_5.field_2],
|
|
2150
|
+
definition_level_encoding: Encodings[header.field_5.field_3],
|
|
2151
|
+
repetition_level_encoding: Encodings[header.field_5.field_4],
|
|
2056
2152
|
statistics: header.field_5.field_5 && {
|
|
2057
2153
|
max: header.field_5.field_5.field_1,
|
|
2058
2154
|
min: header.field_5.field_5.field_2,
|
|
@@ -2065,14 +2161,14 @@ function parquetHeader(reader) {
|
|
|
2065
2161
|
index_page_header: header.field_6,
|
|
2066
2162
|
dictionary_page_header: header.field_7 && {
|
|
2067
2163
|
num_values: header.field_7.field_1,
|
|
2068
|
-
encoding: Encodings
|
|
2164
|
+
encoding: Encodings[header.field_7.field_2],
|
|
2069
2165
|
is_sorted: header.field_7.field_3
|
|
2070
2166
|
},
|
|
2071
2167
|
data_page_header_v2: header.field_8 && {
|
|
2072
2168
|
num_values: header.field_8.field_1,
|
|
2073
2169
|
num_nulls: header.field_8.field_2,
|
|
2074
2170
|
num_rows: header.field_8.field_3,
|
|
2075
|
-
encoding: Encodings
|
|
2171
|
+
encoding: Encodings[header.field_8.field_4],
|
|
2076
2172
|
definition_levels_byte_length: header.field_8.field_5,
|
|
2077
2173
|
repetition_levels_byte_length: header.field_8.field_6,
|
|
2078
2174
|
is_compressed: header.field_8.field_7 === void 0 ? true : header.field_8.field_7,
|
|
@@ -2084,7 +2180,7 @@ function readRowGroup(options, { metadata }, groupPlan) {
|
|
|
2084
2180
|
const asyncColumns = [];
|
|
2085
2181
|
for (const chunk of groupPlan.chunks) {
|
|
2086
2182
|
const { data_page_offset, dictionary_page_offset, path_in_schema: pathInSchema } = chunk.columnMetadata;
|
|
2087
|
-
const schemaPath = getSchemaPath
|
|
2183
|
+
const schemaPath = getSchemaPath(metadata.schema, pathInSchema);
|
|
2088
2184
|
const columnDecoder = {
|
|
2089
2185
|
pathInSchema,
|
|
2090
2186
|
element: schemaPath[schemaPath.length - 1].element,
|
|
@@ -2327,394 +2423,4 @@ function parquetReadObjects(options) {
|
|
|
2327
2423
|
}).catch(reject);
|
|
2328
2424
|
});
|
|
2329
2425
|
}
|
|
2330
|
-
|
|
2331
|
-
"BOOLEAN",
|
|
2332
|
-
"INT32",
|
|
2333
|
-
"INT64",
|
|
2334
|
-
"INT96",
|
|
2335
|
-
"FLOAT",
|
|
2336
|
-
"DOUBLE",
|
|
2337
|
-
"BYTE_ARRAY",
|
|
2338
|
-
"FIXED_LEN_BYTE_ARRAY"
|
|
2339
|
-
];
|
|
2340
|
-
const Encodings = [
|
|
2341
|
-
"PLAIN",
|
|
2342
|
-
"GROUP_VAR_INT",
|
|
2343
|
-
"PLAIN_DICTIONARY",
|
|
2344
|
-
"RLE",
|
|
2345
|
-
"BIT_PACKED",
|
|
2346
|
-
"DELTA_BINARY_PACKED",
|
|
2347
|
-
"DELTA_LENGTH_BYTE_ARRAY",
|
|
2348
|
-
"DELTA_BYTE_ARRAY",
|
|
2349
|
-
"RLE_DICTIONARY",
|
|
2350
|
-
"BYTE_STREAM_SPLIT"
|
|
2351
|
-
];
|
|
2352
|
-
const FieldRepetitionTypes = [
|
|
2353
|
-
"REQUIRED",
|
|
2354
|
-
"OPTIONAL",
|
|
2355
|
-
"REPEATED"
|
|
2356
|
-
];
|
|
2357
|
-
const ConvertedTypes = [
|
|
2358
|
-
"UTF8",
|
|
2359
|
-
"MAP",
|
|
2360
|
-
"MAP_KEY_VALUE",
|
|
2361
|
-
"LIST",
|
|
2362
|
-
"ENUM",
|
|
2363
|
-
"DECIMAL",
|
|
2364
|
-
"DATE",
|
|
2365
|
-
"TIME_MILLIS",
|
|
2366
|
-
"TIME_MICROS",
|
|
2367
|
-
"TIMESTAMP_MILLIS",
|
|
2368
|
-
"TIMESTAMP_MICROS",
|
|
2369
|
-
"UINT_8",
|
|
2370
|
-
"UINT_16",
|
|
2371
|
-
"UINT_32",
|
|
2372
|
-
"UINT_64",
|
|
2373
|
-
"INT_8",
|
|
2374
|
-
"INT_16",
|
|
2375
|
-
"INT_32",
|
|
2376
|
-
"INT_64",
|
|
2377
|
-
"JSON",
|
|
2378
|
-
"BSON",
|
|
2379
|
-
"INTERVAL"
|
|
2380
|
-
];
|
|
2381
|
-
const CompressionCodecs = [
|
|
2382
|
-
"UNCOMPRESSED",
|
|
2383
|
-
"SNAPPY",
|
|
2384
|
-
"GZIP",
|
|
2385
|
-
"LZO",
|
|
2386
|
-
"BROTLI",
|
|
2387
|
-
"LZ4",
|
|
2388
|
-
"ZSTD",
|
|
2389
|
-
"LZ4_RAW"
|
|
2390
|
-
];
|
|
2391
|
-
const PageTypes = [
|
|
2392
|
-
"DATA_PAGE",
|
|
2393
|
-
"INDEX_PAGE",
|
|
2394
|
-
"DICTIONARY_PAGE",
|
|
2395
|
-
"DATA_PAGE_V2"
|
|
2396
|
-
];
|
|
2397
|
-
const BoundaryOrders = [
|
|
2398
|
-
"UNORDERED",
|
|
2399
|
-
"ASCENDING",
|
|
2400
|
-
"DESCENDING"
|
|
2401
|
-
];
|
|
2402
|
-
const EdgeInterpolationAlgorithms = [
|
|
2403
|
-
"SPHERICAL",
|
|
2404
|
-
"VINCENTY",
|
|
2405
|
-
"THOMAS",
|
|
2406
|
-
"ANDOYER",
|
|
2407
|
-
"KARNEY"
|
|
2408
|
-
];
|
|
2409
|
-
new TextDecoder();
|
|
2410
|
-
function parseDecimal(bytes) {
|
|
2411
|
-
if (!bytes.length) return 0;
|
|
2412
|
-
let value = 0n;
|
|
2413
|
-
for (const byte of bytes) value = value * 256n + BigInt(byte);
|
|
2414
|
-
const bits = bytes.length * 8;
|
|
2415
|
-
if (value >= 2n ** BigInt(bits - 1)) value -= 2n ** BigInt(bits);
|
|
2416
|
-
return Number(value);
|
|
2417
|
-
}
|
|
2418
|
-
function schemaTree(schema, rootIndex, path) {
|
|
2419
|
-
const element = schema[rootIndex];
|
|
2420
|
-
const children = [];
|
|
2421
|
-
let count = 1;
|
|
2422
|
-
if (element.num_children) while (children.length < element.num_children) {
|
|
2423
|
-
const childElement = schema[rootIndex + count];
|
|
2424
|
-
const child = schemaTree(schema, rootIndex + count, [...path, childElement.name]);
|
|
2425
|
-
count += child.count;
|
|
2426
|
-
children.push(child);
|
|
2427
|
-
}
|
|
2428
|
-
return {
|
|
2429
|
-
count,
|
|
2430
|
-
element,
|
|
2431
|
-
children,
|
|
2432
|
-
path
|
|
2433
|
-
};
|
|
2434
|
-
}
|
|
2435
|
-
function getSchemaPath(schema, name) {
|
|
2436
|
-
let tree = schemaTree(schema, 0, []);
|
|
2437
|
-
const path = [tree];
|
|
2438
|
-
for (const part of name) {
|
|
2439
|
-
const child = tree.children.find((child) => child.element.name === part);
|
|
2440
|
-
if (!child) throw new Error(`parquet schema element not found: ${name}`);
|
|
2441
|
-
path.push(child);
|
|
2442
|
-
tree = child;
|
|
2443
|
-
}
|
|
2444
|
-
return path;
|
|
2445
|
-
}
|
|
2446
|
-
function getMaxDefinitionLevel(schemaPath) {
|
|
2447
|
-
let maxLevel = 0;
|
|
2448
|
-
for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
|
|
2449
|
-
return maxLevel;
|
|
2450
|
-
}
|
|
2451
|
-
function isListLike(schema) {
|
|
2452
|
-
if (!schema) return false;
|
|
2453
|
-
if (schema.element.converted_type !== "LIST") return false;
|
|
2454
|
-
if (schema.children.length > 1) return false;
|
|
2455
|
-
const firstChild = schema.children[0];
|
|
2456
|
-
if (firstChild.children.length > 1) return false;
|
|
2457
|
-
if (firstChild.element.repetition_type !== "REPEATED") return false;
|
|
2458
|
-
return true;
|
|
2459
|
-
}
|
|
2460
|
-
function isMapLike(schema) {
|
|
2461
|
-
if (!schema) return false;
|
|
2462
|
-
if (schema.element.converted_type !== "MAP") return false;
|
|
2463
|
-
if (schema.children.length > 1) return false;
|
|
2464
|
-
const firstChild = schema.children[0];
|
|
2465
|
-
if (firstChild.children.length !== 2) return false;
|
|
2466
|
-
if (firstChild.element.repetition_type !== "REPEATED") return false;
|
|
2467
|
-
if (firstChild.children.find((child) => child.element.name === "key")?.element.repetition_type === "REPEATED") return false;
|
|
2468
|
-
if (firstChild.children.find((child) => child.element.name === "value")?.element.repetition_type === "REPEATED") return false;
|
|
2469
|
-
return true;
|
|
2470
|
-
}
|
|
2471
|
-
const defaultInitialFetchSize = 1 << 19;
|
|
2472
|
-
new TextDecoder();
|
|
2473
|
-
const MASK = 18446744073709551615n;
|
|
2474
|
-
const PRIME1 = 11400714785074694791n;
|
|
2475
|
-
const PRIME2 = 14029467366897019727n;
|
|
2476
|
-
const PRIME3 = 1609587929392839161n;
|
|
2477
|
-
const PRIME4 = 9650029242287828579n;
|
|
2478
|
-
const PRIME5 = 2870177450012600261n;
|
|
2479
|
-
function rotl64(x, r) {
|
|
2480
|
-
return (x << r | x >> 64n - r) & MASK;
|
|
2481
|
-
}
|
|
2482
|
-
function round(acc, val) {
|
|
2483
|
-
acc = acc + val * PRIME2 & MASK;
|
|
2484
|
-
acc = rotl64(acc, 31n);
|
|
2485
|
-
return acc * PRIME1 & MASK;
|
|
2486
|
-
}
|
|
2487
|
-
function mergeRound(acc, val) {
|
|
2488
|
-
acc ^= round(0n, val);
|
|
2489
|
-
return acc * PRIME1 + PRIME4 & MASK;
|
|
2490
|
-
}
|
|
2491
|
-
function xxhash64(input, seed = 0n) {
|
|
2492
|
-
const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
|
|
2493
|
-
const len = input.byteLength;
|
|
2494
|
-
let offset = 0;
|
|
2495
|
-
let h64;
|
|
2496
|
-
if (len >= 32) {
|
|
2497
|
-
let v1 = seed + PRIME1 + PRIME2 & MASK;
|
|
2498
|
-
let v2 = seed + PRIME2 & MASK;
|
|
2499
|
-
let v3 = seed;
|
|
2500
|
-
let v4 = seed - PRIME1 & MASK;
|
|
2501
|
-
while (offset + 32 <= len) {
|
|
2502
|
-
v1 = round(v1, view.getBigUint64(offset, true));
|
|
2503
|
-
offset += 8;
|
|
2504
|
-
v2 = round(v2, view.getBigUint64(offset, true));
|
|
2505
|
-
offset += 8;
|
|
2506
|
-
v3 = round(v3, view.getBigUint64(offset, true));
|
|
2507
|
-
offset += 8;
|
|
2508
|
-
v4 = round(v4, view.getBigUint64(offset, true));
|
|
2509
|
-
offset += 8;
|
|
2510
|
-
}
|
|
2511
|
-
h64 = rotl64(v1, 1n) + rotl64(v2, 7n) + rotl64(v3, 12n) + rotl64(v4, 18n) & MASK;
|
|
2512
|
-
h64 = mergeRound(h64, v1);
|
|
2513
|
-
h64 = mergeRound(h64, v2);
|
|
2514
|
-
h64 = mergeRound(h64, v3);
|
|
2515
|
-
h64 = mergeRound(h64, v4);
|
|
2516
|
-
} else h64 = seed + PRIME5 & MASK;
|
|
2517
|
-
h64 = h64 + BigInt(len) & MASK;
|
|
2518
|
-
while (offset + 8 <= len) {
|
|
2519
|
-
h64 ^= round(0n, view.getBigUint64(offset, true));
|
|
2520
|
-
h64 = rotl64(h64, 27n) * PRIME1 + PRIME4 & MASK;
|
|
2521
|
-
offset += 8;
|
|
2522
|
-
}
|
|
2523
|
-
if (offset + 4 <= len) {
|
|
2524
|
-
h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1 & MASK;
|
|
2525
|
-
h64 = rotl64(h64, 23n) * PRIME2 + PRIME3 & MASK;
|
|
2526
|
-
offset += 4;
|
|
2527
|
-
}
|
|
2528
|
-
while (offset < len) {
|
|
2529
|
-
h64 ^= BigInt(view.getUint8(offset)) * PRIME5 & MASK;
|
|
2530
|
-
h64 = rotl64(h64, 11n) * PRIME1 & MASK;
|
|
2531
|
-
offset += 1;
|
|
2532
|
-
}
|
|
2533
|
-
h64 ^= h64 >> 33n;
|
|
2534
|
-
h64 = h64 * PRIME2 & MASK;
|
|
2535
|
-
h64 ^= h64 >> 29n;
|
|
2536
|
-
h64 = h64 * PRIME3 & MASK;
|
|
2537
|
-
h64 ^= h64 >> 32n;
|
|
2538
|
-
return h64;
|
|
2539
|
-
}
|
|
2540
|
-
const textEncoder = new TextEncoder();
|
|
2541
|
-
new Uint32Array([
|
|
2542
|
-
1203114875,
|
|
2543
|
-
1150766481,
|
|
2544
|
-
2284105051,
|
|
2545
|
-
2729912477,
|
|
2546
|
-
1884591559,
|
|
2547
|
-
770785867,
|
|
2548
|
-
2667333959,
|
|
2549
|
-
1550580529
|
|
2550
|
-
]);
|
|
2551
|
-
function hashParquetValue(value, element) {
|
|
2552
|
-
if (value === null || value === void 0) return void 0;
|
|
2553
|
-
const { type, converted_type, logical_type } = element;
|
|
2554
|
-
if (type === "BOOLEAN") {
|
|
2555
|
-
if (typeof value !== "boolean") return void 0;
|
|
2556
|
-
return xxhash64(new Uint8Array([value ? 1 : 0]));
|
|
2557
|
-
}
|
|
2558
|
-
if (type === "FLOAT") {
|
|
2559
|
-
if (typeof value !== "number") return void 0;
|
|
2560
|
-
const buf = /* @__PURE__ */ new ArrayBuffer(4);
|
|
2561
|
-
new DataView(buf).setFloat32(0, value, true);
|
|
2562
|
-
return xxhash64(new Uint8Array(buf));
|
|
2563
|
-
}
|
|
2564
|
-
if (type === "DOUBLE") {
|
|
2565
|
-
if (typeof value !== "number") return void 0;
|
|
2566
|
-
const buf = /* @__PURE__ */ new ArrayBuffer(8);
|
|
2567
|
-
new DataView(buf).setFloat64(0, value, true);
|
|
2568
|
-
return xxhash64(new Uint8Array(buf));
|
|
2569
|
-
}
|
|
2570
|
-
if (type === "INT32") {
|
|
2571
|
-
if (converted_type === "DATE" || converted_type === "DECIMAL" || converted_type === "TIME_MILLIS") return void 0;
|
|
2572
|
-
if (logical_type?.type === "DATE" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
|
|
2573
|
-
if (typeof value !== "number" || !Number.isInteger(value)) return void 0;
|
|
2574
|
-
const buf = /* @__PURE__ */ new ArrayBuffer(4);
|
|
2575
|
-
new DataView(buf).setInt32(0, value | 0, true);
|
|
2576
|
-
return xxhash64(new Uint8Array(buf));
|
|
2577
|
-
}
|
|
2578
|
-
if (type === "INT64") {
|
|
2579
|
-
if (converted_type === "TIMESTAMP_MILLIS" || converted_type === "TIMESTAMP_MICROS") return void 0;
|
|
2580
|
-
if (converted_type === "TIME_MICROS" || converted_type === "DECIMAL") return void 0;
|
|
2581
|
-
if (logical_type?.type === "TIMESTAMP" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
|
|
2582
|
-
let bigValue;
|
|
2583
|
-
if (typeof value === "bigint") bigValue = value;
|
|
2584
|
-
else if (typeof value === "number" && Number.isSafeInteger(value)) bigValue = BigInt(value);
|
|
2585
|
-
else return void 0;
|
|
2586
|
-
const buf = /* @__PURE__ */ new ArrayBuffer(8);
|
|
2587
|
-
new DataView(buf).setBigUint64(0, BigInt.asUintN(64, bigValue), true);
|
|
2588
|
-
return xxhash64(new Uint8Array(buf));
|
|
2589
|
-
}
|
|
2590
|
-
if (type === "BYTE_ARRAY") {
|
|
2591
|
-
if (converted_type === "JSON" || converted_type === "BSON" || converted_type === "DECIMAL") return void 0;
|
|
2592
|
-
if (logical_type?.type === "JSON" || logical_type?.type === "BSON" || logical_type?.type === "VARIANT") return void 0;
|
|
2593
|
-
if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
|
|
2594
|
-
if (typeof value === "string") return xxhash64(textEncoder.encode(value));
|
|
2595
|
-
if (value instanceof Uint8Array) return xxhash64(value);
|
|
2596
|
-
return;
|
|
2597
|
-
}
|
|
2598
|
-
if (type === "FIXED_LEN_BYTE_ARRAY") {
|
|
2599
|
-
if (converted_type === "DECIMAL" || converted_type === "INTERVAL") return void 0;
|
|
2600
|
-
if (logical_type?.type === "DECIMAL" || logical_type?.type === "UUID" || logical_type?.type === "FLOAT16") return void 0;
|
|
2601
|
-
if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
|
|
2602
|
-
if (value instanceof Uint8Array) return xxhash64(value);
|
|
2603
|
-
return;
|
|
2604
|
-
}
|
|
2605
|
-
}
|
|
2606
|
-
function toJson(obj) {
|
|
2607
|
-
if (obj === void 0) return null;
|
|
2608
|
-
if (typeof obj === "bigint") return Number(obj);
|
|
2609
|
-
if (Object.is(obj, -0)) return 0;
|
|
2610
|
-
if (Array.isArray(obj)) return obj.map(toJson);
|
|
2611
|
-
if (obj instanceof Uint8Array) return Array.from(obj);
|
|
2612
|
-
if (obj instanceof Date) return obj.toISOString();
|
|
2613
|
-
if (obj instanceof Object) {
|
|
2614
|
-
const newObj = {};
|
|
2615
|
-
for (const key of Object.keys(obj)) {
|
|
2616
|
-
if (obj[key] === void 0) continue;
|
|
2617
|
-
newObj[key] = toJson(obj[key]);
|
|
2618
|
-
}
|
|
2619
|
-
return newObj;
|
|
2620
|
-
}
|
|
2621
|
-
return obj;
|
|
2622
|
-
}
|
|
2623
|
-
async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
|
|
2624
|
-
const controller = new AbortController();
|
|
2625
|
-
const headers = new Headers(requestInit.headers);
|
|
2626
|
-
headers.set("Range", "bytes=0-0");
|
|
2627
|
-
const res = await fetchFn(url, {
|
|
2628
|
-
...requestInit,
|
|
2629
|
-
headers,
|
|
2630
|
-
signal: controller.signal
|
|
2631
|
-
});
|
|
2632
|
-
if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
|
|
2633
|
-
if (res.status === 206) {
|
|
2634
|
-
const contentRange = res.headers.get("Content-Range");
|
|
2635
|
-
if (!contentRange) throw new Error("missing content-range header");
|
|
2636
|
-
const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
|
|
2637
|
-
if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
|
|
2638
|
-
return parseInt(match[1]);
|
|
2639
|
-
}
|
|
2640
|
-
if (res.status === 200) {
|
|
2641
|
-
const contentLength = res.headers.get("Content-Length");
|
|
2642
|
-
controller.abort();
|
|
2643
|
-
if (contentLength) return parseInt(contentLength);
|
|
2644
|
-
}
|
|
2645
|
-
throw new Error("server does not support range requests and missing content-length");
|
|
2646
|
-
}
|
|
2647
|
-
async function byteLengthFromUrl(url, requestInit, customFetch) {
|
|
2648
|
-
const fetch = customFetch ?? globalThis.fetch;
|
|
2649
|
-
const res = await fetch(url, {
|
|
2650
|
-
...requestInit,
|
|
2651
|
-
method: "HEAD"
|
|
2652
|
-
});
|
|
2653
|
-
if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
2654
|
-
if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
|
|
2655
|
-
const length = res.headers.get("Content-Length");
|
|
2656
|
-
if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
2657
|
-
return parseInt(length);
|
|
2658
|
-
}
|
|
2659
|
-
async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
|
|
2660
|
-
if (!url) throw new Error("missing url");
|
|
2661
|
-
const fetch = customFetch ?? globalThis.fetch;
|
|
2662
|
-
byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
|
|
2663
|
-
let buffer = void 0;
|
|
2664
|
-
const init = requestInit || {};
|
|
2665
|
-
return {
|
|
2666
|
-
byteLength,
|
|
2667
|
-
async slice(start, end) {
|
|
2668
|
-
if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
|
|
2669
|
-
const headers = new Headers(init.headers);
|
|
2670
|
-
const endStr = end === void 0 ? "" : end - 1;
|
|
2671
|
-
headers.set("Range", `bytes=${start}-${endStr}`);
|
|
2672
|
-
const res = await fetch(url, {
|
|
2673
|
-
...init,
|
|
2674
|
-
headers
|
|
2675
|
-
});
|
|
2676
|
-
if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
|
|
2677
|
-
if (res.status === 200) {
|
|
2678
|
-
buffer = res.arrayBuffer();
|
|
2679
|
-
return buffer.then((buffer) => buffer.slice(start, end));
|
|
2680
|
-
} else if (res.status === 206) return res.arrayBuffer();
|
|
2681
|
-
else throw new Error(`fetch received unexpected status code ${res.status}`);
|
|
2682
|
-
}
|
|
2683
|
-
};
|
|
2684
|
-
}
|
|
2685
|
-
function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
|
|
2686
|
-
if (byteLength < minSize) {
|
|
2687
|
-
const buffer = slice(0, byteLength);
|
|
2688
|
-
return {
|
|
2689
|
-
byteLength,
|
|
2690
|
-
async slice(start, end) {
|
|
2691
|
-
return (await buffer).slice(start, end);
|
|
2692
|
-
}
|
|
2693
|
-
};
|
|
2694
|
-
}
|
|
2695
|
-
const cache = /* @__PURE__ */ new Map();
|
|
2696
|
-
return {
|
|
2697
|
-
byteLength,
|
|
2698
|
-
slice(start, end) {
|
|
2699
|
-
const key = cacheKey(start, end, byteLength);
|
|
2700
|
-
const cached = cache.get(key);
|
|
2701
|
-
if (cached) return cached;
|
|
2702
|
-
const promise = slice(start, end);
|
|
2703
|
-
cache.set(key, promise);
|
|
2704
|
-
return promise;
|
|
2705
|
-
}
|
|
2706
|
-
};
|
|
2707
|
-
}
|
|
2708
|
-
function cacheKey(start, end, size) {
|
|
2709
|
-
if (start < 0) {
|
|
2710
|
-
if (end !== void 0) throw new Error(`invalid suffix range [${start}, ${end}]`);
|
|
2711
|
-
if (size === void 0) return `${start},`;
|
|
2712
|
-
return `${size + start},${size}`;
|
|
2713
|
-
} else if (end !== void 0) {
|
|
2714
|
-
if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`);
|
|
2715
|
-
return `${start},${end}`;
|
|
2716
|
-
} else if (size === void 0) return `${start},`;
|
|
2717
|
-
else return `${start},${size}`;
|
|
2718
|
-
}
|
|
2719
|
-
new TextDecoder();
|
|
2720
|
-
export { BoundaryOrders, BoundaryOrders$1, CompressionCodecs, CompressionCodecs$1, ConvertedTypes, ConvertedTypes$1, EdgeInterpolationAlgorithms, EdgeInterpolationAlgorithms$1, Encodings, Encodings$1, FieldRepetitionTypes, FieldRepetitionTypes$1, PageTypes, PageTypes$1, ParquetTypes, ParquetTypes$1, asyncBufferFromUrl, cachedAsyncBuffer, getMaxDefinitionLevel, getMaxDefinitionLevel$1, getSchemaPath, getSchemaPath$1, hashParquetValue, hashParquetValue$1, isListLike, isListLike$1, isMapLike, isMapLike$1, parquetReadObjects, parseDecimal, snappyUncompress, toJson, toJson$1 };
|
|
2426
|
+
export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes, asyncBufferFromUrl, cachedAsyncBuffer, getMaxDefinitionLevel, getSchemaPath, hashParquetValue, isListLike, isMapLike, parquetReadObjects, parseDecimal, snappyUncompress, toJson };
|