@queria-io/fdl-js 0.10.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/parquet.d.ts +3 -1
- package/dist/parquet.js +39 -9
- package/dist/parquet.js.map +1 -1
- package/package.json +1 -1
- package/src/parquet.ts +47 -10
package/dist/parquet.d.ts
CHANGED
|
@@ -18,12 +18,14 @@ export declare const cacheApiFooterCache: (cache: Cache, base?: string) => Foote
|
|
|
18
18
|
export declare const memoryFooterCache: () => FooterCache;
|
|
19
19
|
/**
|
|
20
20
|
* A table's column as one data file holds it: its name in the file (undefined when the file was
|
|
21
|
-
* written before the column was added)
|
|
21
|
+
* written before the column was added), the value rows of such a file hold, and the cast from the
|
|
22
|
+
* file's values to the column's type (an int32 file of a column widened to int64 gives numbers)
|
|
22
23
|
*/
|
|
23
24
|
export type FileColumn = {
|
|
24
25
|
name: string;
|
|
25
26
|
physical: string | undefined;
|
|
26
27
|
fill: () => unknown;
|
|
28
|
+
cast: (value: unknown) => unknown;
|
|
27
29
|
};
|
|
28
30
|
export type ParquetFile = {
|
|
29
31
|
key: string;
|
package/dist/parquet.js
CHANGED
|
@@ -63,13 +63,19 @@ const topLevel = (metadata) => {
|
|
|
63
63
|
return out;
|
|
64
64
|
};
|
|
65
65
|
const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
|
|
66
|
+
const BIGINT = new Set(["int64", "uint64"]);
|
|
67
|
+
/** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
|
|
68
|
+
const SPECIAL = { inf: Infinity, "-inf": -Infinity, nan: NaN };
|
|
69
|
+
const float = (text) => SPECIAL[text.toLowerCase()] ?? Number(text);
|
|
66
70
|
/** The value a column's default text stands for, typed as hyparquet returns the column */
|
|
67
71
|
const defaultValue = ({ name, type, default: text }) => {
|
|
68
72
|
if (text === undefined)
|
|
69
73
|
return () => null;
|
|
70
|
-
if (INTEGER.has(type)
|
|
74
|
+
if (INTEGER.has(type))
|
|
71
75
|
return () => Number(text);
|
|
72
|
-
if (type === "
|
|
76
|
+
if (type === "float32" || type === "float64")
|
|
77
|
+
return () => float(text);
|
|
78
|
+
if (BIGINT.has(type))
|
|
73
79
|
return () => BigInt(text);
|
|
74
80
|
if (type === "boolean")
|
|
75
81
|
return () => text === "true";
|
|
@@ -87,6 +93,8 @@ const fileColumns = (metadata, columns) => {
|
|
|
87
93
|
physical: (inFile.find((f) => f.id === c.id) ??
|
|
88
94
|
inFile.find((f) => f.id === undefined && f.name === c.name))?.name,
|
|
89
95
|
fill: defaultValue(c),
|
|
96
|
+
// A column widened from a narrower integer type keeps numbers in the files written before
|
|
97
|
+
cast: BIGINT.has(c.type) ? (v) => (typeof v === "number" ? BigInt(v) : v) : (v) => v,
|
|
90
98
|
}));
|
|
91
99
|
};
|
|
92
100
|
const deletedPositions = async (store, deletes, options) => {
|
|
@@ -122,12 +130,32 @@ export const openParquet = async (store, { key, bytes, footerBytes, deletes, col
|
|
|
122
130
|
...(deleted ? { deleted } : {}),
|
|
123
131
|
};
|
|
124
132
|
};
|
|
133
|
+
/** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
|
|
125
134
|
const columnIndex = (metadata, column) => {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
135
|
+
let leaf = 0;
|
|
136
|
+
let i = 1;
|
|
137
|
+
while (i < metadata.schema.length) {
|
|
138
|
+
const s = metadata.schema[i];
|
|
139
|
+
if (!s.num_children) {
|
|
140
|
+
if (s.name === column)
|
|
141
|
+
return leaf;
|
|
142
|
+
leaf++;
|
|
143
|
+
i++;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (s.name === column)
|
|
147
|
+
throw new Error(`${column} is not a leaf column`);
|
|
148
|
+
// Skip the group's subtree, counting its leaves
|
|
149
|
+
let pending = s.num_children;
|
|
150
|
+
i++;
|
|
151
|
+
while (pending > 0) {
|
|
152
|
+
const child = metadata.schema[i++].num_children ?? 0;
|
|
153
|
+
if (child === 0)
|
|
154
|
+
leaf++;
|
|
155
|
+
pending += child - 1;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
throw new Error(`no column ${column}`);
|
|
131
159
|
};
|
|
132
160
|
// A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
|
|
133
161
|
// and above) after U+E000-U+FFFF
|
|
@@ -195,8 +223,10 @@ export const readRows = async ({ file, metadata, columns: table, deleted }, { co
|
|
|
195
223
|
const physical = wanted.flatMap((c) => (c.physical === undefined ? [] : [c.physical]));
|
|
196
224
|
const rows = physical.length > 0
|
|
197
225
|
? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
|
|
198
|
-
: Array.from({
|
|
199
|
-
|
|
226
|
+
: Array.from({
|
|
227
|
+
length: Math.max(0, Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start),
|
|
228
|
+
}, () => ({}));
|
|
229
|
+
return keep(rows).map((row) => Object.fromEntries(wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.cast(row[c.physical])])));
|
|
200
230
|
};
|
|
201
231
|
// INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
|
|
202
232
|
// by value, and unlike BigInt() it does not throw for 1.5
|
package/dist/parquet.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AAoBF,kFAAkF;AAClF,MAAM,QAAQ,GAAG,CAAC,QAAsB,EAAmC,EAAE;IAC3E,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,6BAA6B;QAC7B,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;QAClC,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEjF,0FAA0F;AAC1F,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAU,EAAmB,EAAE;IAC9E,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7F,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC;IACrD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7D,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CACR,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAC5D,EAAE,IAAI;QACP,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;KACtB,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,0EAA0E;IAC1E,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,EAAE,EAC/D,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CACnF,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AA0BF,kFAAkF;AAClF,MAAM,QAAQ,GAAG,CAAC,QAAsB,EAAmC,EAAE;IAC3E,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,6BAA6B;QAC7B,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;QAClC,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5C,0FAA0F;AAC1F,MAAM,OAAO,GAA2B,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACvF,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;AAE5E,0FAA0F;AAC1F,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAU,EAAmB,EAAE;IAC9E,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC;IACrD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7D,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CACR,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAC5D,EAAE,IAAI;QACP,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;QACrB,0FAA0F;QAC1F,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;KACrF,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,2FAA2F;AAC3F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;gBAAE,OAAO,IAAI,CAAC;YACnC,IAAI,EAAE,CAAC;YACP,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uBAAuB,CAAC,CAAC;QACzE,gDAAgD;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,CAAC;QAC7B,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC;YACtD,IAAI,KAAK,KAAK,CAAC;gBAAE,IAAI,EAAE,CAAC;YACxB,OAAO,IAAI,KAAK,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;AACzC,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR;YACE,MAAM,EAAE,IAAI,CAAC,GAAG,CACd,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CACtE;SACF,EACD,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAC3F,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@queria-io/fdl-js",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare-workers",
|
package/src/parquet.ts
CHANGED
|
@@ -83,9 +83,15 @@ const readFooter = async (file: AsyncBuffer, footerBytes?: number): Promise<Arra
|
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
85
|
* A table's column as one data file holds it: its name in the file (undefined when the file was
|
|
86
|
-
* written before the column was added)
|
|
86
|
+
* written before the column was added), the value rows of such a file hold, and the cast from the
|
|
87
|
+
* file's values to the column's type (an int32 file of a column widened to int64 gives numbers)
|
|
87
88
|
*/
|
|
88
|
-
export type FileColumn = {
|
|
89
|
+
export type FileColumn = {
|
|
90
|
+
name: string;
|
|
91
|
+
physical: string | undefined;
|
|
92
|
+
fill: () => unknown;
|
|
93
|
+
cast: (value: unknown) => unknown;
|
|
94
|
+
};
|
|
89
95
|
|
|
90
96
|
export type ParquetFile = {
|
|
91
97
|
key: string;
|
|
@@ -115,12 +121,18 @@ const topLevel = (metadata: FileMetaData): { name: string; id?: number }[] => {
|
|
|
115
121
|
};
|
|
116
122
|
|
|
117
123
|
const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
|
|
124
|
+
const BIGINT = new Set(["int64", "uint64"]);
|
|
125
|
+
|
|
126
|
+
/** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
|
|
127
|
+
const SPECIAL: Record<string, number> = { inf: Infinity, "-inf": -Infinity, nan: NaN };
|
|
128
|
+
const float = (text: string) => SPECIAL[text.toLowerCase()] ?? Number(text);
|
|
118
129
|
|
|
119
130
|
/** The value a column's default text stands for, typed as hyparquet returns the column */
|
|
120
131
|
const defaultValue = ({ name, type, default: text }: Column): (() => unknown) => {
|
|
121
132
|
if (text === undefined) return () => null;
|
|
122
|
-
if (INTEGER.has(type)
|
|
123
|
-
if (type === "
|
|
133
|
+
if (INTEGER.has(type)) return () => Number(text);
|
|
134
|
+
if (type === "float32" || type === "float64") return () => float(text);
|
|
135
|
+
if (BIGINT.has(type)) return () => BigInt(text);
|
|
124
136
|
if (type === "boolean") return () => text === "true";
|
|
125
137
|
if (type === "varchar" || type === "json") return () => text;
|
|
126
138
|
return () => {
|
|
@@ -138,6 +150,8 @@ const fileColumns = (metadata: FileMetaData, columns: Column[]): FileColumn[] =>
|
|
|
138
150
|
inFile.find((f) => f.id === undefined && f.name === c.name)
|
|
139
151
|
)?.name,
|
|
140
152
|
fill: defaultValue(c),
|
|
153
|
+
// A column widened from a narrower integer type keeps numbers in the files written before
|
|
154
|
+
cast: BIGINT.has(c.type) ? (v) => (typeof v === "number" ? BigInt(v) : v) : (v) => v,
|
|
141
155
|
}));
|
|
142
156
|
};
|
|
143
157
|
|
|
@@ -186,11 +200,29 @@ export const openParquet = async (
|
|
|
186
200
|
export type RowRange = { rowStart: number; rowEnd: number };
|
|
187
201
|
type Value = string | number | bigint;
|
|
188
202
|
|
|
203
|
+
/** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
|
|
189
204
|
const columnIndex = (metadata: FileMetaData, column: string): number => {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
205
|
+
let leaf = 0;
|
|
206
|
+
let i = 1;
|
|
207
|
+
while (i < metadata.schema.length) {
|
|
208
|
+
const s = metadata.schema[i]!;
|
|
209
|
+
if (!s.num_children) {
|
|
210
|
+
if (s.name === column) return leaf;
|
|
211
|
+
leaf++;
|
|
212
|
+
i++;
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
if (s.name === column) throw new Error(`${column} is not a leaf column`);
|
|
216
|
+
// Skip the group's subtree, counting its leaves
|
|
217
|
+
let pending = s.num_children;
|
|
218
|
+
i++;
|
|
219
|
+
while (pending > 0) {
|
|
220
|
+
const child = metadata.schema[i++]!.num_children ?? 0;
|
|
221
|
+
if (child === 0) leaf++;
|
|
222
|
+
pending += child - 1;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
throw new Error(`no column ${column}`);
|
|
194
226
|
};
|
|
195
227
|
|
|
196
228
|
// A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
|
|
@@ -271,12 +303,17 @@ export const readRows = async <T = Record<string, unknown>>(
|
|
|
271
303
|
physical.length > 0
|
|
272
304
|
? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
|
|
273
305
|
: Array.from(
|
|
274
|
-
{
|
|
306
|
+
{
|
|
307
|
+
length: Math.max(
|
|
308
|
+
0,
|
|
309
|
+
Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start,
|
|
310
|
+
),
|
|
311
|
+
},
|
|
275
312
|
(): Record<string, unknown> => ({}),
|
|
276
313
|
);
|
|
277
314
|
return keep(rows).map((row) =>
|
|
278
315
|
Object.fromEntries(
|
|
279
|
-
wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : row[c.physical]]),
|
|
316
|
+
wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.cast(row[c.physical])]),
|
|
280
317
|
),
|
|
281
318
|
) as T[];
|
|
282
319
|
};
|