@gscdump/lakehouse 0.35.3 → 0.35.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/THIRD-PARTY-LICENSES.md +2 -2
- package/dist/_chunks/libs/hyparquet.mjs +260 -246
- package/dist/_chunks/libs/icebird.mjs +19 -28
- package/package.json +4 -4
|
@@ -304,11 +304,11 @@ Repository: https://github.com/101arrowz/fzstd
|
|
|
304
304
|
|
|
305
305
|
---------------------------------------
|
|
306
306
|
|
|
307
|
-
## hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
|
|
307
|
+
## hyparquet, hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
|
|
308
308
|
|
|
309
309
|
License: MIT
|
|
310
310
|
By: Hyperparam
|
|
311
|
-
Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
|
|
311
|
+
Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
|
|
312
312
|
|
|
313
313
|
> The MIT License (MIT)
|
|
314
314
|
>
|
|
@@ -1,82 +1,3 @@
|
|
|
1
|
-
const ParquetTypes = [
|
|
2
|
-
"BOOLEAN",
|
|
3
|
-
"INT32",
|
|
4
|
-
"INT64",
|
|
5
|
-
"INT96",
|
|
6
|
-
"FLOAT",
|
|
7
|
-
"DOUBLE",
|
|
8
|
-
"BYTE_ARRAY",
|
|
9
|
-
"FIXED_LEN_BYTE_ARRAY"
|
|
10
|
-
];
|
|
11
|
-
const Encodings = [
|
|
12
|
-
"PLAIN",
|
|
13
|
-
"GROUP_VAR_INT",
|
|
14
|
-
"PLAIN_DICTIONARY",
|
|
15
|
-
"RLE",
|
|
16
|
-
"BIT_PACKED",
|
|
17
|
-
"DELTA_BINARY_PACKED",
|
|
18
|
-
"DELTA_LENGTH_BYTE_ARRAY",
|
|
19
|
-
"DELTA_BYTE_ARRAY",
|
|
20
|
-
"RLE_DICTIONARY",
|
|
21
|
-
"BYTE_STREAM_SPLIT"
|
|
22
|
-
];
|
|
23
|
-
const FieldRepetitionTypes = [
|
|
24
|
-
"REQUIRED",
|
|
25
|
-
"OPTIONAL",
|
|
26
|
-
"REPEATED"
|
|
27
|
-
];
|
|
28
|
-
const ConvertedTypes = [
|
|
29
|
-
"UTF8",
|
|
30
|
-
"MAP",
|
|
31
|
-
"MAP_KEY_VALUE",
|
|
32
|
-
"LIST",
|
|
33
|
-
"ENUM",
|
|
34
|
-
"DECIMAL",
|
|
35
|
-
"DATE",
|
|
36
|
-
"TIME_MILLIS",
|
|
37
|
-
"TIME_MICROS",
|
|
38
|
-
"TIMESTAMP_MILLIS",
|
|
39
|
-
"TIMESTAMP_MICROS",
|
|
40
|
-
"UINT_8",
|
|
41
|
-
"UINT_16",
|
|
42
|
-
"UINT_32",
|
|
43
|
-
"UINT_64",
|
|
44
|
-
"INT_8",
|
|
45
|
-
"INT_16",
|
|
46
|
-
"INT_32",
|
|
47
|
-
"INT_64",
|
|
48
|
-
"JSON",
|
|
49
|
-
"BSON",
|
|
50
|
-
"INTERVAL"
|
|
51
|
-
];
|
|
52
|
-
const CompressionCodecs = [
|
|
53
|
-
"UNCOMPRESSED",
|
|
54
|
-
"SNAPPY",
|
|
55
|
-
"GZIP",
|
|
56
|
-
"LZO",
|
|
57
|
-
"BROTLI",
|
|
58
|
-
"LZ4",
|
|
59
|
-
"ZSTD",
|
|
60
|
-
"LZ4_RAW"
|
|
61
|
-
];
|
|
62
|
-
const PageTypes = [
|
|
63
|
-
"DATA_PAGE",
|
|
64
|
-
"INDEX_PAGE",
|
|
65
|
-
"DICTIONARY_PAGE",
|
|
66
|
-
"DATA_PAGE_V2"
|
|
67
|
-
];
|
|
68
|
-
const BoundaryOrders = [
|
|
69
|
-
"UNORDERED",
|
|
70
|
-
"ASCENDING",
|
|
71
|
-
"DESCENDING"
|
|
72
|
-
];
|
|
73
|
-
const EdgeInterpolationAlgorithms = [
|
|
74
|
-
"SPHERICAL",
|
|
75
|
-
"VINCENTY",
|
|
76
|
-
"THOMAS",
|
|
77
|
-
"ANDOYER",
|
|
78
|
-
"KARNEY"
|
|
79
|
-
];
|
|
80
1
|
new TextDecoder();
|
|
81
2
|
function parseDecimal(bytes) {
|
|
82
3
|
if (!bytes.length) return 0;
|
|
@@ -86,6 +7,185 @@ function parseDecimal(bytes) {
|
|
|
86
7
|
if (value >= 2n ** BigInt(bits - 1)) value -= 2n ** BigInt(bits);
|
|
87
8
|
return Number(value);
|
|
88
9
|
}
|
|
10
|
+
const defaultInitialFetchSize = 1 << 19;
|
|
11
|
+
new TextDecoder();
|
|
12
|
+
new TextEncoder();
|
|
13
|
+
new Uint32Array([
|
|
14
|
+
1203114875,
|
|
15
|
+
1150766481,
|
|
16
|
+
2284105051,
|
|
17
|
+
2729912477,
|
|
18
|
+
1884591559,
|
|
19
|
+
770785867,
|
|
20
|
+
2667333959,
|
|
21
|
+
1550580529
|
|
22
|
+
]);
|
|
23
|
+
async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
|
|
24
|
+
const controller = new AbortController();
|
|
25
|
+
const headers = new Headers(requestInit.headers);
|
|
26
|
+
headers.set("Range", "bytes=0-0");
|
|
27
|
+
const res = await fetchFn(url, {
|
|
28
|
+
...requestInit,
|
|
29
|
+
headers,
|
|
30
|
+
signal: controller.signal
|
|
31
|
+
});
|
|
32
|
+
if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
|
|
33
|
+
if (res.status === 206) {
|
|
34
|
+
const contentRange = res.headers.get("Content-Range");
|
|
35
|
+
if (!contentRange) throw new Error("missing content-range header");
|
|
36
|
+
const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
|
|
37
|
+
if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
|
|
38
|
+
return parseInt(match[1]);
|
|
39
|
+
}
|
|
40
|
+
if (res.status === 200) {
|
|
41
|
+
const contentLength = res.headers.get("Content-Length");
|
|
42
|
+
controller.abort();
|
|
43
|
+
if (contentLength) return parseInt(contentLength);
|
|
44
|
+
}
|
|
45
|
+
throw new Error("server does not support range requests and missing content-length");
|
|
46
|
+
}
|
|
47
|
+
async function byteLengthFromUrl(url, requestInit, customFetch) {
|
|
48
|
+
const fetch = customFetch ?? globalThis.fetch;
|
|
49
|
+
const res = await fetch(url, {
|
|
50
|
+
...requestInit,
|
|
51
|
+
method: "HEAD"
|
|
52
|
+
});
|
|
53
|
+
if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
54
|
+
if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
|
|
55
|
+
const length = res.headers.get("Content-Length");
|
|
56
|
+
if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
57
|
+
return parseInt(length);
|
|
58
|
+
}
|
|
59
|
+
async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
|
|
60
|
+
if (!url) throw new Error("missing url");
|
|
61
|
+
const fetch = customFetch ?? globalThis.fetch;
|
|
62
|
+
byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
|
|
63
|
+
let buffer = void 0;
|
|
64
|
+
const init = requestInit || {};
|
|
65
|
+
return {
|
|
66
|
+
byteLength,
|
|
67
|
+
async slice(start, end) {
|
|
68
|
+
if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
|
|
69
|
+
const headers = new Headers(init.headers);
|
|
70
|
+
const endStr = end === void 0 ? "" : end - 1;
|
|
71
|
+
headers.set("Range", `bytes=${start}-${endStr}`);
|
|
72
|
+
const res = await fetch(url, {
|
|
73
|
+
...init,
|
|
74
|
+
headers
|
|
75
|
+
});
|
|
76
|
+
if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
|
|
77
|
+
if (res.status === 200) {
|
|
78
|
+
buffer = res.arrayBuffer();
|
|
79
|
+
return buffer.then((buffer) => buffer.slice(start, end));
|
|
80
|
+
} else if (res.status === 206) return res.arrayBuffer();
|
|
81
|
+
else throw new Error(`fetch received unexpected status code ${res.status}`);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
|
|
86
|
+
if (byteLength < minSize) {
|
|
87
|
+
const buffer = slice(0, byteLength);
|
|
88
|
+
return {
|
|
89
|
+
byteLength,
|
|
90
|
+
async slice(start, end) {
|
|
91
|
+
return (await buffer).slice(start, end);
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
const cache = /* @__PURE__ */ new Map();
|
|
96
|
+
return {
|
|
97
|
+
byteLength,
|
|
98
|
+
slice(start, end) {
|
|
99
|
+
const key = cacheKey(start, end, byteLength);
|
|
100
|
+
const cached = cache.get(key);
|
|
101
|
+
if (cached) return cached;
|
|
102
|
+
const promise = slice(start, end);
|
|
103
|
+
cache.set(key, promise);
|
|
104
|
+
return promise;
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function cacheKey(start, end, size) {
|
|
109
|
+
if (start < 0) {
|
|
110
|
+
if (end !== void 0) throw new Error(`invalid suffix range [${start}, ${end}]`);
|
|
111
|
+
if (size === void 0) return `${start},`;
|
|
112
|
+
return `${size + start},${size}`;
|
|
113
|
+
} else if (end !== void 0) {
|
|
114
|
+
if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`);
|
|
115
|
+
return `${start},${end}`;
|
|
116
|
+
} else if (size === void 0) return `${start},`;
|
|
117
|
+
else return `${start},${size}`;
|
|
118
|
+
}
|
|
119
|
+
new TextDecoder();
|
|
120
|
+
const WORD_MASK = [
|
|
121
|
+
0,
|
|
122
|
+
255,
|
|
123
|
+
65535,
|
|
124
|
+
16777215,
|
|
125
|
+
4294967295
|
|
126
|
+
];
|
|
127
|
+
function copyBytes(fromArray, fromPos, toArray, toPos, length) {
|
|
128
|
+
for (let i = 0; i < length; i++) toArray[toPos + i] = fromArray[fromPos + i];
|
|
129
|
+
}
|
|
130
|
+
function snappyUncompress(input, output) {
|
|
131
|
+
const inputLength = input.byteLength;
|
|
132
|
+
const outputLength = output.byteLength;
|
|
133
|
+
let pos = 0;
|
|
134
|
+
let outPos = 0;
|
|
135
|
+
while (pos < inputLength) {
|
|
136
|
+
const c = input[pos];
|
|
137
|
+
pos++;
|
|
138
|
+
if (c < 128) break;
|
|
139
|
+
}
|
|
140
|
+
if (outputLength && pos >= inputLength) throw new Error("invalid snappy length header");
|
|
141
|
+
while (pos < inputLength) {
|
|
142
|
+
const c = input[pos];
|
|
143
|
+
let len = 0;
|
|
144
|
+
pos++;
|
|
145
|
+
if (pos >= inputLength) throw new Error("missing eof marker");
|
|
146
|
+
if ((c & 3) === 0) {
|
|
147
|
+
let len = (c >>> 2) + 1;
|
|
148
|
+
if (len > 60) {
|
|
149
|
+
if (pos + 3 >= inputLength) throw new Error("snappy error literal pos + 3 >= inputLength");
|
|
150
|
+
const lengthSize = len - 60;
|
|
151
|
+
len = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
|
|
152
|
+
len = (len & WORD_MASK[lengthSize]) + 1;
|
|
153
|
+
pos += lengthSize;
|
|
154
|
+
}
|
|
155
|
+
if (pos + len > inputLength) throw new Error("snappy error literal exceeds input length");
|
|
156
|
+
copyBytes(input, pos, output, outPos, len);
|
|
157
|
+
pos += len;
|
|
158
|
+
outPos += len;
|
|
159
|
+
} else {
|
|
160
|
+
let offset = 0;
|
|
161
|
+
switch (c & 3) {
|
|
162
|
+
case 1:
|
|
163
|
+
len = (c >>> 2 & 7) + 4;
|
|
164
|
+
offset = input[pos] + (c >>> 5 << 8);
|
|
165
|
+
pos++;
|
|
166
|
+
break;
|
|
167
|
+
case 2:
|
|
168
|
+
if (inputLength <= pos + 1) throw new Error("snappy error end of input");
|
|
169
|
+
len = (c >>> 2) + 1;
|
|
170
|
+
offset = input[pos] + (input[pos + 1] << 8);
|
|
171
|
+
pos += 2;
|
|
172
|
+
break;
|
|
173
|
+
case 3:
|
|
174
|
+
if (inputLength <= pos + 3) throw new Error("snappy error end of input");
|
|
175
|
+
len = (c >>> 2) + 1;
|
|
176
|
+
offset = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
|
|
177
|
+
pos += 4;
|
|
178
|
+
break;
|
|
179
|
+
default: break;
|
|
180
|
+
}
|
|
181
|
+
if (offset === 0 || isNaN(offset)) throw new Error(`invalid offset ${offset} pos ${pos} inputLength ${inputLength}`);
|
|
182
|
+
if (offset > outPos) throw new Error("cannot copy from before start of buffer");
|
|
183
|
+
copyBytes(output, outPos - offset, output, outPos, len);
|
|
184
|
+
outPos += len;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (outPos !== outputLength) throw new Error("premature end of input");
|
|
188
|
+
}
|
|
89
189
|
function schemaTree(schema, rootIndex, path) {
|
|
90
190
|
const element = schema[rootIndex];
|
|
91
191
|
const children = [];
|
|
@@ -139,8 +239,6 @@ function isMapLike(schema) {
|
|
|
139
239
|
if (firstChild.children.find((child) => child.element.name === "value")?.element.repetition_type === "REPEATED") return false;
|
|
140
240
|
return true;
|
|
141
241
|
}
|
|
142
|
-
const defaultInitialFetchSize = 1 << 19;
|
|
143
|
-
new TextDecoder();
|
|
144
242
|
const MASK = 18446744073709551615n;
|
|
145
243
|
const PRIME1 = 11400714785074694791n;
|
|
146
244
|
const PRIME2 = 14029467366897019727n;
|
|
@@ -274,6 +372,87 @@ function hashParquetValue(value, element) {
|
|
|
274
372
|
return;
|
|
275
373
|
}
|
|
276
374
|
}
|
|
375
|
+
const ParquetTypes = [
|
|
376
|
+
"BOOLEAN",
|
|
377
|
+
"INT32",
|
|
378
|
+
"INT64",
|
|
379
|
+
"INT96",
|
|
380
|
+
"FLOAT",
|
|
381
|
+
"DOUBLE",
|
|
382
|
+
"BYTE_ARRAY",
|
|
383
|
+
"FIXED_LEN_BYTE_ARRAY"
|
|
384
|
+
];
|
|
385
|
+
const Encodings = [
|
|
386
|
+
"PLAIN",
|
|
387
|
+
"GROUP_VAR_INT",
|
|
388
|
+
"PLAIN_DICTIONARY",
|
|
389
|
+
"RLE",
|
|
390
|
+
"BIT_PACKED",
|
|
391
|
+
"DELTA_BINARY_PACKED",
|
|
392
|
+
"DELTA_LENGTH_BYTE_ARRAY",
|
|
393
|
+
"DELTA_BYTE_ARRAY",
|
|
394
|
+
"RLE_DICTIONARY",
|
|
395
|
+
"BYTE_STREAM_SPLIT"
|
|
396
|
+
];
|
|
397
|
+
const FieldRepetitionTypes = [
|
|
398
|
+
"REQUIRED",
|
|
399
|
+
"OPTIONAL",
|
|
400
|
+
"REPEATED"
|
|
401
|
+
];
|
|
402
|
+
const ConvertedTypes = [
|
|
403
|
+
"UTF8",
|
|
404
|
+
"MAP",
|
|
405
|
+
"MAP_KEY_VALUE",
|
|
406
|
+
"LIST",
|
|
407
|
+
"ENUM",
|
|
408
|
+
"DECIMAL",
|
|
409
|
+
"DATE",
|
|
410
|
+
"TIME_MILLIS",
|
|
411
|
+
"TIME_MICROS",
|
|
412
|
+
"TIMESTAMP_MILLIS",
|
|
413
|
+
"TIMESTAMP_MICROS",
|
|
414
|
+
"UINT_8",
|
|
415
|
+
"UINT_16",
|
|
416
|
+
"UINT_32",
|
|
417
|
+
"UINT_64",
|
|
418
|
+
"INT_8",
|
|
419
|
+
"INT_16",
|
|
420
|
+
"INT_32",
|
|
421
|
+
"INT_64",
|
|
422
|
+
"JSON",
|
|
423
|
+
"BSON",
|
|
424
|
+
"INTERVAL"
|
|
425
|
+
];
|
|
426
|
+
const CompressionCodecs = [
|
|
427
|
+
"UNCOMPRESSED",
|
|
428
|
+
"SNAPPY",
|
|
429
|
+
"GZIP",
|
|
430
|
+
"LZO",
|
|
431
|
+
"BROTLI",
|
|
432
|
+
"LZ4",
|
|
433
|
+
"ZSTD",
|
|
434
|
+
"LZ4_RAW"
|
|
435
|
+
];
|
|
436
|
+
const PageTypes = [
|
|
437
|
+
"DATA_PAGE",
|
|
438
|
+
"INDEX_PAGE",
|
|
439
|
+
"DICTIONARY_PAGE",
|
|
440
|
+
"DATA_PAGE_V2"
|
|
441
|
+
];
|
|
442
|
+
const BoundaryOrders = [
|
|
443
|
+
"UNORDERED",
|
|
444
|
+
"ASCENDING",
|
|
445
|
+
"DESCENDING"
|
|
446
|
+
];
|
|
447
|
+
const EdgeInterpolationAlgorithms = [
|
|
448
|
+
"SPHERICAL",
|
|
449
|
+
"VINCENTY",
|
|
450
|
+
"THOMAS",
|
|
451
|
+
"ANDOYER",
|
|
452
|
+
"KARNEY"
|
|
453
|
+
];
|
|
454
|
+
new TextDecoder();
|
|
455
|
+
new TextDecoder();
|
|
277
456
|
function toJson(obj) {
|
|
278
457
|
if (obj === void 0) return null;
|
|
279
458
|
if (typeof obj === "bigint") return Number(obj);
|
|
@@ -291,170 +470,5 @@ function toJson(obj) {
|
|
|
291
470
|
}
|
|
292
471
|
return obj;
|
|
293
472
|
}
|
|
294
|
-
async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
|
|
295
|
-
const controller = new AbortController();
|
|
296
|
-
const headers = new Headers(requestInit.headers);
|
|
297
|
-
headers.set("Range", "bytes=0-0");
|
|
298
|
-
const res = await fetchFn(url, {
|
|
299
|
-
...requestInit,
|
|
300
|
-
headers,
|
|
301
|
-
signal: controller.signal
|
|
302
|
-
});
|
|
303
|
-
if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
|
|
304
|
-
if (res.status === 206) {
|
|
305
|
-
const contentRange = res.headers.get("Content-Range");
|
|
306
|
-
if (!contentRange) throw new Error("missing content-range header");
|
|
307
|
-
const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
|
|
308
|
-
if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
|
|
309
|
-
return parseInt(match[1]);
|
|
310
|
-
}
|
|
311
|
-
if (res.status === 200) {
|
|
312
|
-
const contentLength = res.headers.get("Content-Length");
|
|
313
|
-
controller.abort();
|
|
314
|
-
if (contentLength) return parseInt(contentLength);
|
|
315
|
-
}
|
|
316
|
-
throw new Error("server does not support range requests and missing content-length");
|
|
317
|
-
}
|
|
318
|
-
async function byteLengthFromUrl(url, requestInit, customFetch) {
|
|
319
|
-
const fetch = customFetch ?? globalThis.fetch;
|
|
320
|
-
const res = await fetch(url, {
|
|
321
|
-
...requestInit,
|
|
322
|
-
method: "HEAD"
|
|
323
|
-
});
|
|
324
|
-
if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
325
|
-
if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
|
|
326
|
-
const length = res.headers.get("Content-Length");
|
|
327
|
-
if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
|
|
328
|
-
return parseInt(length);
|
|
329
|
-
}
|
|
330
|
-
async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
|
|
331
|
-
if (!url) throw new Error("missing url");
|
|
332
|
-
const fetch = customFetch ?? globalThis.fetch;
|
|
333
|
-
byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
|
|
334
|
-
let buffer = void 0;
|
|
335
|
-
const init = requestInit || {};
|
|
336
|
-
return {
|
|
337
|
-
byteLength,
|
|
338
|
-
async slice(start, end) {
|
|
339
|
-
if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
|
|
340
|
-
const headers = new Headers(init.headers);
|
|
341
|
-
const endStr = end === void 0 ? "" : end - 1;
|
|
342
|
-
headers.set("Range", `bytes=${start}-${endStr}`);
|
|
343
|
-
const res = await fetch(url, {
|
|
344
|
-
...init,
|
|
345
|
-
headers
|
|
346
|
-
});
|
|
347
|
-
if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
|
|
348
|
-
if (res.status === 200) {
|
|
349
|
-
buffer = res.arrayBuffer();
|
|
350
|
-
return buffer.then((buffer) => buffer.slice(start, end));
|
|
351
|
-
} else if (res.status === 206) return res.arrayBuffer();
|
|
352
|
-
else throw new Error(`fetch received unexpected status code ${res.status}`);
|
|
353
|
-
}
|
|
354
|
-
};
|
|
355
|
-
}
|
|
356
|
-
function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
|
|
357
|
-
if (byteLength < minSize) {
|
|
358
|
-
const buffer = slice(0, byteLength);
|
|
359
|
-
return {
|
|
360
|
-
byteLength,
|
|
361
|
-
async slice(start, end) {
|
|
362
|
-
return (await buffer).slice(start, end);
|
|
363
|
-
}
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
const cache = /* @__PURE__ */ new Map();
|
|
367
|
-
return {
|
|
368
|
-
byteLength,
|
|
369
|
-
slice(start, end) {
|
|
370
|
-
const key = cacheKey(start, end, byteLength);
|
|
371
|
-
const cached = cache.get(key);
|
|
372
|
-
if (cached) return cached;
|
|
373
|
-
const promise = slice(start, end);
|
|
374
|
-
cache.set(key, promise);
|
|
375
|
-
return promise;
|
|
376
|
-
}
|
|
377
|
-
};
|
|
378
|
-
}
|
|
379
|
-
function cacheKey(start, end, size) {
|
|
380
|
-
if (start < 0) {
|
|
381
|
-
if (end !== void 0) throw new Error(`invalid suffix range [${start}, ${end}]`);
|
|
382
|
-
if (size === void 0) return `${start},`;
|
|
383
|
-
return `${size + start},${size}`;
|
|
384
|
-
} else if (end !== void 0) {
|
|
385
|
-
if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`);
|
|
386
|
-
return `${start},${end}`;
|
|
387
|
-
} else if (size === void 0) return `${start},`;
|
|
388
|
-
else return `${start},${size}`;
|
|
389
|
-
}
|
|
390
473
|
new TextDecoder();
|
|
391
|
-
const WORD_MASK = [
|
|
392
|
-
0,
|
|
393
|
-
255,
|
|
394
|
-
65535,
|
|
395
|
-
16777215,
|
|
396
|
-
4294967295
|
|
397
|
-
];
|
|
398
|
-
function copyBytes(fromArray, fromPos, toArray, toPos, length) {
|
|
399
|
-
for (let i = 0; i < length; i++) toArray[toPos + i] = fromArray[fromPos + i];
|
|
400
|
-
}
|
|
401
|
-
function snappyUncompress(input, output) {
|
|
402
|
-
const inputLength = input.byteLength;
|
|
403
|
-
const outputLength = output.byteLength;
|
|
404
|
-
let pos = 0;
|
|
405
|
-
let outPos = 0;
|
|
406
|
-
while (pos < inputLength) {
|
|
407
|
-
const c = input[pos];
|
|
408
|
-
pos++;
|
|
409
|
-
if (c < 128) break;
|
|
410
|
-
}
|
|
411
|
-
if (outputLength && pos >= inputLength) throw new Error("invalid snappy length header");
|
|
412
|
-
while (pos < inputLength) {
|
|
413
|
-
const c = input[pos];
|
|
414
|
-
let len = 0;
|
|
415
|
-
pos++;
|
|
416
|
-
if (pos >= inputLength) throw new Error("missing eof marker");
|
|
417
|
-
if ((c & 3) === 0) {
|
|
418
|
-
let len = (c >>> 2) + 1;
|
|
419
|
-
if (len > 60) {
|
|
420
|
-
if (pos + 3 >= inputLength) throw new Error("snappy error literal pos + 3 >= inputLength");
|
|
421
|
-
const lengthSize = len - 60;
|
|
422
|
-
len = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
|
|
423
|
-
len = (len & WORD_MASK[lengthSize]) + 1;
|
|
424
|
-
pos += lengthSize;
|
|
425
|
-
}
|
|
426
|
-
if (pos + len > inputLength) throw new Error("snappy error literal exceeds input length");
|
|
427
|
-
copyBytes(input, pos, output, outPos, len);
|
|
428
|
-
pos += len;
|
|
429
|
-
outPos += len;
|
|
430
|
-
} else {
|
|
431
|
-
let offset = 0;
|
|
432
|
-
switch (c & 3) {
|
|
433
|
-
case 1:
|
|
434
|
-
len = (c >>> 2 & 7) + 4;
|
|
435
|
-
offset = input[pos] + (c >>> 5 << 8);
|
|
436
|
-
pos++;
|
|
437
|
-
break;
|
|
438
|
-
case 2:
|
|
439
|
-
if (inputLength <= pos + 1) throw new Error("snappy error end of input");
|
|
440
|
-
len = (c >>> 2) + 1;
|
|
441
|
-
offset = input[pos] + (input[pos + 1] << 8);
|
|
442
|
-
pos += 2;
|
|
443
|
-
break;
|
|
444
|
-
case 3:
|
|
445
|
-
if (inputLength <= pos + 3) throw new Error("snappy error end of input");
|
|
446
|
-
len = (c >>> 2) + 1;
|
|
447
|
-
offset = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
|
|
448
|
-
pos += 4;
|
|
449
|
-
break;
|
|
450
|
-
default: break;
|
|
451
|
-
}
|
|
452
|
-
if (offset === 0 || isNaN(offset)) throw new Error(`invalid offset ${offset} pos ${pos} inputLength ${inputLength}`);
|
|
453
|
-
if (offset > outPos) throw new Error("cannot copy from before start of buffer");
|
|
454
|
-
copyBytes(output, outPos - offset, output, outPos, len);
|
|
455
|
-
outPos += len;
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
if (outPos !== outputLength) throw new Error("premature end of input");
|
|
459
|
-
}
|
|
460
474
|
export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes, asyncBufferFromUrl, cachedAsyncBuffer, getMaxDefinitionLevel, getSchemaPath, hashParquetValue, isListLike, isMapLike, parseDecimal, snappyUncompress, toJson };
|
|
@@ -345,32 +345,6 @@ async function fetchAvroRecords(url, resolver, byteLength) {
|
|
|
345
345
|
});
|
|
346
346
|
}
|
|
347
347
|
const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
|
|
348
|
-
function stringifyIcebergJson(value, indent) {
|
|
349
|
-
const sp = indent ? " ".repeat(indent) : "";
|
|
350
|
-
function emit(v, depth) {
|
|
351
|
-
if (typeof v === "bigint") return v.toString();
|
|
352
|
-
if (v === null) return "null";
|
|
353
|
-
if (typeof v === "string") return JSON.stringify(v);
|
|
354
|
-
if (typeof v === "number" || typeof v === "boolean") return JSON.stringify(v);
|
|
355
|
-
if (Array.isArray(v)) {
|
|
356
|
-
if (v.length === 0) return "[]";
|
|
357
|
-
const inner = v.map((x) => emit(x, depth + 1));
|
|
358
|
-
if (!sp) return "[" + inner.join(",") + "]";
|
|
359
|
-
const pad = sp.repeat(depth + 1), close = sp.repeat(depth);
|
|
360
|
-
return "[\n" + pad + inner.join(",\n" + pad) + "\n" + close + "]";
|
|
361
|
-
}
|
|
362
|
-
if (typeof v === "object") {
|
|
363
|
-
const keys = Object.keys(v).filter((k) => v[k] !== void 0);
|
|
364
|
-
if (keys.length === 0) return "{}";
|
|
365
|
-
const inner = keys.map((k) => JSON.stringify(k) + (sp ? ": " : ":") + emit(v[k], depth + 1));
|
|
366
|
-
if (!sp) return "{" + inner.join(",") + "}";
|
|
367
|
-
const pad = sp.repeat(depth + 1), close = sp.repeat(depth);
|
|
368
|
-
return "{\n" + pad + inner.join(",\n" + pad) + "\n" + close + "}";
|
|
369
|
-
}
|
|
370
|
-
return JSON.stringify(v);
|
|
371
|
-
}
|
|
372
|
-
return emit(value, 0);
|
|
373
|
-
}
|
|
374
348
|
function parseIcebergJson(text) {
|
|
375
349
|
let i = 0;
|
|
376
350
|
function skipWs() {
|
|
@@ -499,6 +473,23 @@ function parseIcebergJson(text) {
|
|
|
499
473
|
if (i !== text.length) throw new Error(`unexpected trailing input at ${i}`);
|
|
500
474
|
return value;
|
|
501
475
|
}
|
|
476
|
+
function stringifyIcebergJson(value, indent = 2) {
|
|
477
|
+
const pad = " ".repeat(indent);
|
|
478
|
+
function serialize(val, depth) {
|
|
479
|
+
if (typeof val === "bigint") return val.toString();
|
|
480
|
+
if (val === null || typeof val !== "object") return JSON.stringify(val);
|
|
481
|
+
const inner = pad.repeat(depth + 1);
|
|
482
|
+
const outer = pad.repeat(depth);
|
|
483
|
+
if (Array.isArray(val)) {
|
|
484
|
+
if (val.length === 0) return "[]";
|
|
485
|
+
return `[\n${val.map((v) => inner + serialize(v === void 0 ? null : v, depth + 1)).join(",\n")}\n${outer}]`;
|
|
486
|
+
}
|
|
487
|
+
const keys = Object.keys(val).filter((k) => val[k] !== void 0);
|
|
488
|
+
if (keys.length === 0) return "{}";
|
|
489
|
+
return `{\n${keys.map((k) => `${inner}${JSON.stringify(k)}: ${serialize(val[k], depth + 1)}`).join(",\n")}\n${outer}}`;
|
|
490
|
+
}
|
|
491
|
+
return serialize(value, 0);
|
|
492
|
+
}
|
|
502
493
|
function metadataFileVersionNumber(file) {
|
|
503
494
|
const match = file.match(/^(?:v(\d+)|(\d+)-.+)(?:\.metadata\.json|\.gz\.metadata\.json|\.metadata\.json\.gz)$/);
|
|
504
495
|
if (!match) return void 0;
|
|
@@ -1411,7 +1402,7 @@ async function icebergCreate({ tableUrl, resolver, schema, formatVersion, partit
|
|
|
1411
1402
|
if (formatVersion >= 3) metadata["next-row-id"] = 0;
|
|
1412
1403
|
if (!resolver.writer) throw new Error("resolver.writer is required");
|
|
1413
1404
|
const metadataWriter = conditionalCommits ? resolver.writer(metadataUrl, { ifNoneMatch: "*" }) : resolver.writer(metadataUrl);
|
|
1414
|
-
const metadataBytes = new TextEncoder().encode(stringifyIcebergJson(metadata
|
|
1405
|
+
const metadataBytes = new TextEncoder().encode(stringifyIcebergJson(metadata));
|
|
1415
1406
|
metadataWriter.appendBytes(metadataBytes);
|
|
1416
1407
|
await metadataWriter.finish();
|
|
1417
1408
|
const versionHintUrl = `${tableUrl}/metadata/version-hint.text`;
|
|
@@ -1455,7 +1446,7 @@ async function fileCatalogCommit({ tableUrl, metadata, metadataFileName, current
|
|
|
1455
1446
|
"metadata-log": trimmedLog
|
|
1456
1447
|
};
|
|
1457
1448
|
const metaWriter = conditionalCommits ? resolver.writer(newMetadataPath, { ifNoneMatch: "*" }) : resolver.writer(newMetadataPath);
|
|
1458
|
-
metaWriter.appendBytes(new TextEncoder().encode(stringifyIcebergJson(newMetadata
|
|
1449
|
+
metaWriter.appendBytes(new TextEncoder().encode(stringifyIcebergJson(newMetadata)));
|
|
1459
1450
|
await metaWriter.finish();
|
|
1460
1451
|
try {
|
|
1461
1452
|
const hintWriter = resolver.writer(`${tableUrl}/metadata/version-hint.text`);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/lakehouse",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.35.
|
|
4
|
+
"version": "0.35.5",
|
|
5
5
|
"description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -46,9 +46,9 @@
|
|
|
46
46
|
"node": ">=18"
|
|
47
47
|
},
|
|
48
48
|
"devDependencies": {
|
|
49
|
-
"@types/node": "^26.0
|
|
50
|
-
"hyparquet": "^1.26.
|
|
51
|
-
"icebird": "^0.8.
|
|
49
|
+
"@types/node": "^26.1.0",
|
|
50
|
+
"hyparquet": "^1.26.2",
|
|
51
|
+
"icebird": "^0.8.12",
|
|
52
52
|
"unstorage": "^1.17.5",
|
|
53
53
|
"vitest": "^4.1.9"
|
|
54
54
|
},
|